#29: Handle exceptions when files need to be skipped [subdaily CDM lite]

* #5: removed unspecified except and checked for output paths

* #5: handled missing input files

PYTHON_CDM_Conversion_code/hourly_qff_to_cdm_lite_v1.py

@@ -397,10 +397,15 @@ def main(station="", subset="", run_all=False, clobber=False):
 
     # To start at begining of files
     for filename in all_filenames:
-        print(f"Processing {filename}")
+
+        if not os.path.exists(os.path.join(utils.SUBDAILY_QFF_IN_DIR, filename)):
+            print("Input QFF file missing: {}".format(os.path.join(utils.SUBDAILY_QFF_IN_DIR, filename)))
+            continue
+        else:
+            print("Processing {}".format(os.path.join(utils.SUBDAILY_QFF_IN_DIR, filename)))
 
         # Read in the dataframe
-        df=pd.read_csv(os.path.join(utils.SUBDAILY_QFF_IN_DIR, filename), sep="|",low_memory=False)
+        df=pd.read_csv(os.path.join(utils.SUBDAILY_QFF_IN_DIR, filename), sep="|", low_memory=False)
 
         # Set up the output filenames, and check if they exist
         station_id=df.iloc[1]["Station_ID"] # NOTE: this is renamed below to "primary_station_id"
@@ -774,11 +779,14 @@ def main(station="", subset="", run_all=False, clobber=False):
             qc_merged_df.to_csv(qc_outfile, index=False, sep="|")
             print(f"   {qc_outfile}")
             print("    Done")
-        except:
-            # Continue to next iteration.
-            continue
+        except IOError:
+            # something wrong with file paths, despite checking
+            print(f"Cannot save datafile: {cdmlite_outfile}")
+        except RuntimeError:
+            print("Runtime error")
+        # TODO add logging for these errors
 
-#    return # main
+    return # main
 
 #****************************************
 if __name__ == "__main__":

PYTHON_CDM_Conversion_code/utils.py

@@ -35,6 +35,22 @@
 MONTHLY_CDM_LITE_OUT_DIR = config.get("Paths", "monthly_cdmlite_outdir")
 MONTHLY_CDM_OBS_OUT_DIR = config.get("Paths", "monthly_cdmobs_outdir")
 MONTHLY_CDM_HEAD_OUT_DIR = config.get("Paths", "monthly_cdmhead_outdir")
+
+# make directories if they do not exist
+for path in (SUBDAILY_QC_OUT_DIR,
+             SUBDAILY_CDM_LITE_OUT_DIR,
+             SUBDAILY_CDM_OBS_OUT_DIR,
+             SUBDAILY_CDM_HEAD_OUT_DIR,
+             DAILY_CDM_LITE_OUT_DIR,
+             DAILY_QC_OUT_DIR,
+             DAILY_CDM_OBS_OUT_DIR,
+             MONTHLY_CDM_LITE_OUT_DIR,
+             MONTHLY_CDM_OBS_OUT_DIR,
+             MONTHLY_CDM_HEAD_OUT_DIR,
+             ):
+    os.makedirs(path, exist_ok=True)
+
+
 # Files
 SUBDAILY_CDM_LITE_FILE_ROOT = config.get("Filenames", "subdaily_cdmlite_file")
 SUBDAILY_QC_FILE_ROOT = config.get("Filenames", "subdaily_cdmqc_file")
@@ -54,5 +70,5 @@
 DAILY_STATION_RECORD_ENTRIES_OBS_LITE = config.get("Records", "daily_station_records_obs_lite")
 DAILY_STATION_RECORD_ENTRIES_HEADER = config.get("Records", "daily_station_records_header")
 MONTHLY_STATION_RECORD_ENTRIES_OBS_LITE = config.get("Records", "monthly_station_records_obs_lite")
-MONTHLY_STATION_RECORD_ENTRIES_HEADER = config.get("Records", "monthly_station_records_header"):wq
+MONTHLY_STATION_RECORD_ENTRIES_HEADER = config.get("Records", "monthly_station_records_header")