Remove legacy ascii data from EMTF

- now get this from MTH5
- relates to issue #320

aurora/general_helper_functions.py

@@ -1,5 +1,6 @@
 import inspect
 import os
+import pathlib
 import scipy.io as sio
 import subprocess
 
@@ -8,6 +9,8 @@
 
 import aurora
 import mt_metadata
+import mth5
+
 
 init_file = inspect.getfile(aurora)
 AURORA_PATH = Path(init_file).parent.parent
@@ -17,6 +20,18 @@
 BAND_SETUP_PATH = CONFIG_PATH.joinpath("emtf_band_setup")
 
 
+def get_mth5_ascii_data_path():
+    """
+    Get the path to the
+    Returns
+    -------
+    mth5_data_path: pathlib.Path
+        This is the place where the legacy test files (ascii MT data from EMTF) are archived
+    """
+    mth5_data_path = pathlib.Path(mth5.__file__).parent.joinpath("data")
+    return mth5_data_path
+
+
 def get_test_path():
     test_path = AURORA_PATH.joinpath("tests")
     if not test_path.exists():

aurora/test_utils/synthetic/paths.py

@@ -1,39 +1,62 @@
 """
 Sets up paths for synthetic data testing.
 
+The DATA_PATH from general_helper_functions has traditionally had the
+synthetic ascii data, but this is now stored in MTH5.
 """
 import pathlib
 
 from aurora.general_helper_functions import DATA_PATH
+from aurora.general_helper_functions import get_mth5_ascii_data_path
 from loguru import logger
 
+DEFAULT_SANDBOX_PATH = DATA_PATH.joinpath("synthetic")
+
 
 class SyntheticTestPaths:
-    def __init__(self, sandbox_path=None):
-        """
-        ivars:
-        - ascii_data_path, where the ascii data are stored
-        - mth5_path: this is where the mth5 files get written to.
-        - config folder: this is where the config files get saved while tests are running
-        - aurora_results folder: this is where the processing results get saved during test execution
+    """
+    sandbox path must be a place that has write access.  This class was created because on some
+    installations we only have read access.  Originally there was a data/ folder with the synthetic
+    ascii data stored there, and we created the mth5 and other data products in the same place.
+
+    Here we have a ascii_data_path which points at the ascii files (which may be read only), but we
+    also accept a kwarg for "sandbox_path" which is writable and this is where the mth5 and etc. will
+    get built.
 
+    TODO: consider creating a symlink in aurora's legacy data path that points at the mth5 ascii files.
 
+    """
+
+    def __init__(self, sandbox_path=None, ascii_data_path=None):
+        """
 
         Parameters
         ----------
-        sandbox_path: None or pathlib.Path
-        """
+        sandbox_path: pathlib.Path
+            A writable path where test results are stored
 
-        self._root_path = DATA_PATH.joinpath("synthetic")
-        if sandbox_path is None:
-            logger.debug(f"synthetic sandbox path is being set to {self._root_path}")
-            self._sandbox_path = self._root_path
+        ivars:
+        - ascii_data_path: where the ascii data are stored
+        - mth5_path: this is where the mth5 files get written to.
+        - config_path: this is where the config files get saved while tests are running
+        - aurora_results_path: this is where the processing results get saved during test execution
+        - emtf_results_path: stores some legacy results from EMTF processing for tests/comparison.
 
+        """
         # READ ONLY OK
-        self.ascii_data_path = self._root_path.joinpath("ascii")
+        if ascii_data_path is None:
+            self.ascii_data_path = get_mth5_ascii_data_path()
 
         # NEED WRITE ACCESS
         # Consider using an environment variable for sandbox_path
+        if sandbox_path is None:
+            logger.debug(
+                f"synthetic sandbox path is being set to {DEFAULT_SANDBOX_PATH}"
+            )
+            self._sandbox_path = DEFAULT_SANDBOX_PATH
+        else:
+            self._sandbox_path = sandbox_path
+
         self.mth5_path = self._sandbox_path.joinpath("mth5")
         self.aurora_results_path = self._sandbox_path.joinpath("aurora_results")
         self.emtf_results_path = self._sandbox_path.joinpath("emtf_results")
@@ -52,6 +75,10 @@ def writability_check(self):
         pass
 
     def mkdirs(self):
+        """
+        Makes the directories that the tests will write results to.
+
+        """
         self.aurora_results_path.mkdir(parents=True, exist_ok=True)
         self.config_path.mkdir(parents=True, exist_ok=True)
         self.emtf_results_path.mkdir(parents=True, exist_ok=True)

aurora/test_utils/synthetic/station_config.py

@@ -15,14 +15,14 @@
 Run level: 'sample_rate', 1.0
 
 """
+from aurora.general_helper_functions import get_mth5_ascii_data_path
 from aurora.test_utils.synthetic.paths import SyntheticTestPaths
 from mt_metadata.timeseries.filters.helper_functions import make_coefficient_filter
 from mt_metadata.timeseries import Run
 from mt_metadata.transfer_functions.processing.aurora import ChannelNomenclature
 
-
+ASCII_DATA_PATH = get_mth5_ascii_data_path()
 synthetic_test_paths = SyntheticTestPaths()
-DATA_PATH = synthetic_test_paths.ascii_data_path
 
 
 def make_filters(as_list=False):
@@ -121,7 +121,7 @@ def make_station_01(channel_nomenclature="default"):
 
     run_001 = SyntheticRun(
         "001",
-        raw_data_path=DATA_PATH.joinpath("test1.asc"),
+        raw_data_path=ASCII_DATA_PATH.joinpath("test1.asc"),
         channel_nomenclature=channel_nomenclature,
         start=None,
     )
@@ -156,7 +156,7 @@ def make_station_02(channel_nomenclature="default"):
     test2 = make_station_01(channel_nomenclature=channel_nomenclature)
     test2.id = "test2"
     test2.mth5_name = "test2.h5"
-    test2.runs[0].raw_data_path = DATA_PATH.joinpath("test2.asc")
+    test2.runs[0].raw_data_path = ASCII_DATA_PATH.joinpath("test2.asc")
     nan_indices = {}
     for channel in test2.runs[0].channels:
         nan_indices[channel] = []
@@ -202,7 +202,7 @@ def make_station_03(channel_nomenclature="default"):
 
     run_001 = SyntheticRun(
         "001",
-        raw_data_path=DATA_PATH.joinpath("test1.asc"),
+        raw_data_path=ASCII_DATA_PATH.joinpath("test1.asc"),
         nan_indices=nan_indices,
         filters=filters,
         channel_nomenclature=channel_nomenclature,
@@ -214,7 +214,7 @@ def make_station_03(channel_nomenclature="default"):
         noise_scalars[ch] = 2.0
     run_002 = SyntheticRun(
         "002",
-        raw_data_path=DATA_PATH.joinpath("test1.asc"),
+        raw_data_path=ASCII_DATA_PATH.joinpath("test1.asc"),
         noise_scalars=noise_scalars,
         nan_indices=nan_indices,
         filters=filters,
@@ -226,7 +226,7 @@ def make_station_03(channel_nomenclature="default"):
         noise_scalars[ch] = 5.0
     run_003 = SyntheticRun(
         "003",
-        raw_data_path=DATA_PATH.joinpath("test1.asc"),
+        raw_data_path=ASCII_DATA_PATH.joinpath("test1.asc"),
         noise_scalars=noise_scalars,
         nan_indices=nan_indices,
         filters=filters,
@@ -238,7 +238,7 @@ def make_station_03(channel_nomenclature="default"):
         noise_scalars[ch] = 10.0
     run_004 = SyntheticRun(
         "004",
-        raw_data_path=DATA_PATH.joinpath("test1.asc"),
+        raw_data_path=ASCII_DATA_PATH.joinpath("test1.asc"),
         noise_scalars=noise_scalars,
         nan_indices=nan_indices,
         filters=filters,
@@ -266,7 +266,7 @@ def make_station_04(channel_nomenclature="default"):
 
     run_001 = SyntheticRun(
         "001",
-        raw_data_path=DATA_PATH.joinpath("test1.asc"),
+        raw_data_path=ASCII_DATA_PATH.joinpath("test1.asc"),
         channel_nomenclature=channel_nomenclature,
         start=None,
         sample_rate=8.0,