From b6d2a2115b21252c3335b4dd0091d4ea042d93c1 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Thu, 7 Sep 2023 17:08:37 -0700 Subject: [PATCH] remove old handling of remote references --- .../earthscope/01_test_load_spud_tfs.py | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/aurora/test_utils/earthscope/01_test_load_spud_tfs.py b/aurora/test_utils/earthscope/01_test_load_spud_tfs.py index 5948df7f..f21d8792 100644 --- a/aurora/test_utils/earthscope/01_test_load_spud_tfs.py +++ b/aurora/test_utils/earthscope/01_test_load_spud_tfs.py @@ -4,11 +4,6 @@ There are two possible places to access an xml in each row, called emtf_xml_path and data_xml_path. -It has been asserted that -(df.data_remotes.astype(str)==df.data_remotes_2.astype(str)).all() -(df.emtf_remotes.astype(str)==df.emtf_remotes_2.astype(str)).all() -(df.emtf_remotes.astype(str) == df.data_remotes_2.astype(str)).all() -which basically means we can deprecate one of get_remotes_from_tf, get_remotes_from_tf_2 Dask Notes: - 0 partitions 720s @@ -39,9 +34,6 @@ from aurora.test_utils.earthscope.helpers import SUMMARY_TABLES_PATH from aurora.test_utils.earthscope.helpers import load_xml_tf from aurora.test_utils.earthscope.helpers import get_most_recent_summary_filepath -from aurora.test_utils.earthscope.helpers import get_remotes_from_tf -from aurora.test_utils.earthscope.helpers import get_remotes_from_tf_2 -from aurora.test_utils.earthscope.helpers import get_rr_type from aurora.test_utils.earthscope.helpers import get_summary_table_filename from aurora.test_utils.earthscope.helpers import get_summary_table_schema from aurora.test_utils.earthscope.helpers import load_most_recent_summary @@ -54,6 +46,7 @@ XML_SOURCES = ["emtf", "data"] N_PARTITIONS = 1 + def prepare_dataframe_for_scraping(restrict_to_first_n_rows=False,): """ Define the data structure that is output from this stage of processing @@ -87,20 +80,12 @@ def enrich_row(row): xml_path = SPUD_XML_PATHS[xml_source].joinpath(row[f"{xml_source}_xml_filebase"]) try: tf = load_xml_tf(xml_path) - # OLD - rr_type = get_rr_type(tf) + + remotes = tf.station_metadata.transfer_function.remote_references + rr_type = tf.station_metadata.transfer_function.processing_type row[f"{xml_source}_remote_ref_type"] = rr_type - remotes = get_remotes_from_tf(tf) row[f"{xml_source}_remotes"] = ",".join(remotes) - # NEW - # remotes = tf.station_metadata.transfer_function.remote_references - # remotes = [x for x in remotes if x != tf.station] - # Do we want the "self" station being returned in remotes - # rr_type = tf.station_metadata.transfer_function.processing_type - # row[f"{xml_source}_remote_ref_type"] = rr_type - # row[f"{xml_source}_remotes"] = ",".join(remotes) - except Exception as e: row[f"{xml_source}_error"] = True row[f"{xml_source}_exception"] = e.__class__.__name__