Merge pull request #37 from joergschultzelutter/36-add-messaging-in-c…

…ase-external-dependencies-cannot-be-downloaded

36 add messaging in case external dependencies cannot be downloaded

docs/INSTALLATION.md

@@ -21,7 +21,7 @@ Currently, MPAD uses various APIs and access keys for its purposes:
 - aprs.fi
 - DAPNET API
 - SMTP/IMAP username and password
-- optional: file name reference to an [Apprise](https://github.com/caronc/apprise) YAML config [file](https://github.com/caronc/apprise/wiki/config_yaml). If a reference to an Apprise YML config file is specified and a program crash occurs, MPAD will send a message (along with the stack trace) to the user hosting the program.
+- optional: file name reference to an [Apprise](https://github.com/caronc/apprise) YAML config [file](https://github.com/caronc/apprise/wiki/config_yaml). If a reference to an Apprise YML config file is specified and a program crash occurs, MPAD will send a message (along with the stack trace) to the user hosting the program. Additionally, your local MPAD installation will send you notifications whenever external resources (e.g. TLE database, repeater data etc) cannot be downloaded. This will make it easier for MPAD admins to identify deprecated / changed resource URLs.
 
 If you want to host your own MPAD instance, you need to acquire your personal API access keys (e.g. for aprs.fi) and add these to MPAD's API config file (```mpad_api_access_keys.cfg```). An empty config template file is part of the repository. If you are not a registered DAPNET user, set the DAPNET callsign in the config file to N0CALL. When MPAD encounters this DAPNET user, it will refrain from sending content to DAPNET.
 

src/airport_data_modules.py

@@ -27,6 +27,7 @@
 import logging
 from utility_modules import check_if_file_exists, build_full_pathname
 from mpad_config import mpad_airport_stations_filename
+from messaging_modules import send_apprise_message
 
 # icao https://www.aviationweather.gov/docs/metar/stations.txt
 
@@ -327,6 +328,7 @@ def validate_icao(icao_code: str):
 
 def update_local_airport_stations_file(
     airport_stations_filename: str = mpad_airport_stations_filename,
+    apprise_config_file: str = None,
 ):
     """
     Imports the ICAO/IATA data from the web and saves it to a local file.
@@ -336,6 +338,10 @@ def update_local_airport_stations_file(
     airport_stations_filename : 'str'
         This local file will hold the content
         from https://www.aviationweather.gov/docs/metar/stations.txt.
+    apprise_config_file: 'str'
+        Optional Apprise config file name which will be used in case
+        of errors, telling MPAD's host that the file could not get downloaded
+        (e.g. URL change, URL down, ...)
 
     Returns
     =======
@@ -366,6 +372,18 @@ def update_local_airport_stations_file(
                 logger.debug(
                     msg=f"Cannot update airport data to local file '{absolute_path_filename}'"
                 )
+
+    # Generate an Apprise message in case we were unable to download the file
+    if not success and apprise_config_file:
+        # send_apprise_message will check again if the file exists or not
+        # Therefore, we can skip any further detection steps here
+        send_apprise_message(
+            message_header="MPAD External Dependency Error",
+            message_body=f"Unable to download airport data file from '{file_url}'",
+            apprise_config_file=apprise_config_file,
+            message_attachment=None,
+        )
+
     return success
 
 

src/aprs_listener.py

@@ -529,15 +529,15 @@ def mycallback(raw_aprs_packet: dict):
     #
     # Refresh the local "airport stations" file
     logger.info(msg="Updating airport database ...")
-    update_local_airport_stations_file()
+    update_local_airport_stations_file(apprise_config_file=apprise_config_file)
     #
     # Refresh the local "repeatermap" file
     logger.info(msg="Updating repeater database ...")
-    update_local_repeatermap_file()
+    update_local_repeatermap_file(apprise_config_file=apprise_config_file)
     #
     # Update the satellite TLE file
     logger.info(msg="Updating satellite TLE and frequency database ...")
-    update_local_mpad_satellite_data()
+    update_local_mpad_satellite_data(apprise_config_file=apprise_config_file)
 
     # Now let's set up schedulers for the refresh process
     # These schedulers will download the file(s) every x days
@@ -552,7 +552,7 @@ def mycallback(raw_aprs_packet: dict):
         "interval",
         id="airport_data",
         days=30,
-        args=[],
+        kwargs={"apprise_config_file": apprise_config_file},
     )
 
     # Set up task for repeater data download every 7 days
@@ -561,7 +561,7 @@ def mycallback(raw_aprs_packet: dict):
         "interval",
         id="repeatermap_data",
         days=7,
-        args=[],
+        kwargs={"apprise_config_file": apprise_config_file},
     )
 
     # Set up task for satellite TLE / frequency data
@@ -571,7 +571,7 @@ def mycallback(raw_aprs_packet: dict):
         "interval",
         id="tle_and_satfreq_data",
         days=2,
-        args=[],
+        kwargs={"apprise_config_file": apprise_config_file},
     )
 
     # Set up task for the IMAP garbage collector - which will delete

src/messaging_modules.py

@@ -60,7 +60,7 @@ def send_apprise_message(
     logger.debug(msg="Starting Apprise message processing")
 
     if not apprise_config_file or apprise_config_file == "NOT_CONFIGURED":
-        logger.debug(msg="Skipping post-mortem dump; message file is not configured")
+        logger.debug(msg="Skipping Apprise messaging; message file is not configured")
         return success
 
     if not check_if_file_exists(apprise_config_file):
@@ -69,7 +69,7 @@ def send_apprise_message(
         )
         return success
 
-    if not check_if_file_exists(message_attachment):
+    if message_attachment and not check_if_file_exists(message_attachment):
         logger.debug("Attachment file missing; disabling attachments")
         message_attachment = None
 

src/mpad_config.py

@@ -22,7 +22,7 @@
 #
 # Program version
 #
-mpad_version: str = "0.61"
+mpad_version: str = "0.62"
 #
 ###########################
 # Constants, do not change#

src/parser_test.py

@@ -88,35 +88,65 @@ def testcall(message_text: str, from_callsign: str):
         logger.info(msg=pformat(response_parameters))
 
 
-def download_data_files_if_missing():
+def download_data_files_if_missing(force_download: bool = False):
     # if the user has never ever run the actual bot, some files might be missing, thus preventing us from
     # simulating the bot's actual behavior in real life. As a workaround, check if the files are missing
     # and download them, if necessary
     #
+
+    # Read the config
+    # we only need the Apprise file name
+    (
+        success,
+        aprsdotfi_api_key,
+        aprsis_callsign,
+        aprsis_passcode,
+        dapnet_login_callsign,
+        dapnet_login_passcode,
+        smtpimap_email_address,
+        smtpimap_email_password,
+        apprise_config_file,
+    ) = read_program_config()
+    assert success
+
     # check if airport data file is present
-    if not check_if_file_exists(
-        build_full_pathname(mpad_config.mpad_airport_stations_filename)
+    if (
+        not check_if_file_exists(
+            build_full_pathname(mpad_config.mpad_airport_stations_filename)
+        )
+        or force_download
     ):
         logger.info("Updating local airport data file")
-        update_local_airport_stations_file(mpad_config.mpad_airport_stations_filename)
+        update_local_airport_stations_file(
+            airport_stations_filename=mpad_config.mpad_airport_stations_filename,
+            apprise_config_file=apprise_config_file,
+        )
 
     # check if the satellite data is present
-    if not check_if_file_exists(
-        build_full_pathname(mpad_config.mpad_satellite_frequencies_filename)
-    ) or not check_if_file_exists(
-        build_full_pathname(mpad_config.mpad_tle_amateur_satellites_filename)
+    if (
+        not check_if_file_exists(
+            build_full_pathname(mpad_config.mpad_satellite_frequencies_filename)
+        )
+        or not check_if_file_exists(
+            build_full_pathname(mpad_config.mpad_tle_amateur_satellites_filename)
+        )
+        or force_download
     ):
         logger.info("Updating local satellite data files")
-        update_local_mpad_satellite_data()
+        update_local_mpad_satellite_data(apprise_config_file=apprise_config_file)
 
     # check if the repeater data is present
-    if not check_if_file_exists(
-        build_full_pathname(mpad_config.mpad_repeatermap_raw_data_filename)
-    ) or not check_if_file_exists(
-        build_full_pathname(mpad_config.mpad_hearham_raw_data_filename)
+    if (
+        not check_if_file_exists(
+            build_full_pathname(mpad_config.mpad_repeatermap_raw_data_filename)
+        )
+        or not check_if_file_exists(
+            build_full_pathname(mpad_config.mpad_hearham_raw_data_filename)
+        )
+        or force_download
     ):
         logger.info("Updating local repeater data files")
-        update_local_repeatermap_file()
+        update_local_repeatermap_file(apprise_config_file=apprise_config_file)
 
 
 def mpad_exception_handler():
@@ -195,7 +225,7 @@ def handle_exception(exc_type, exc_value, exc_traceback):
 if __name__ == "__main__":
     # Check if the local database files exist and
     # create them, if necessary
-    download_data_files_if_missing()
+    download_data_files_if_missing(force_download=True)
 
     # Register the on_exit function to be called on program exit
     atexit.register(mpad_exception_handler)

src/repeater_modules.py

@@ -32,6 +32,7 @@
 from geo_conversion_modules import haversine
 import logging
 import operator
+from messaging_modules import send_apprise_message
 
 from mpad_config import (
     mpad_hearham_raw_data_filename,
@@ -46,8 +47,9 @@
 
 
 def download_repeatermap_raw_data_to_local_file(
-    url: str = "http://www.repeatermap.de/apinew.php",
+    file_url: str = "http://www.repeatermap.de/apinew.php",
     repeatermap_raw_data_file: str = mpad_repeatermap_raw_data_filename,
+    apprise_config_file: str = None,
 ):
     """
     Downloads the repeatermap.de data and write it to a file 'as is'
@@ -59,6 +61,10 @@ def download_repeatermap_raw_data_to_local_file(
         Source URL where we will get the data from.
     repeatermap_raw_data_file: 'str'
         Filename of the target file which will hold the raw data
+    apprise_config_file: 'str'
+        Optional Apprise config file name which will be used in case
+        of errors, telling MPAD's host that the file could not get downloaded
+        (e.g. URL change, URL down, ...)
 
     Returns
     =======
@@ -68,7 +74,7 @@ def download_repeatermap_raw_data_to_local_file(
     success = False
     absolute_path_filename = build_full_pathname(file_name=repeatermap_raw_data_file)
     try:
-        resp = requests.get(url)
+        resp = requests.get(file_url)
     except requests.exceptions.RequestException as e:
         logger.error(msg="{e}")
         resp = None
@@ -84,6 +90,18 @@ def download_repeatermap_raw_data_to_local_file(
                 logger.info(
                     msg=f"Cannot write repeatermap.de data to local disc file '{absolute_path_filename}'"
                 )
+
+    # Generate an Apprise message in case we were unable to download the file
+    if not success and apprise_config_file:
+        # send_apprise_message will check again if the file exists or not
+        # Therefore, we can skip any further detection steps here
+        send_apprise_message(
+            message_header="MPAD External Dependency Error",
+            message_body=f"Unable to download repeatermap.de data file from '{file_url}'",
+            apprise_config_file=apprise_config_file,
+            message_attachment=None,
+        )
+
     return success
 
 
@@ -528,23 +546,27 @@ def read_mpad_repeatermap_data_from_disc(
     return success, mpad_repeatermap
 
 
-def update_local_repeatermap_file():
+def update_local_repeatermap_file(apprise_config_file: str = None):
     """
     Wrapper method for importing the raw data from repeatermap.de,
     postprocessing the the data and finally writing the enriched data
     back to a local file.
 
     Parameters
     ==========
+    apprise_config_file: 'str'
+        Optional Apprise config file name which will be used in case
+        of errors, telling MPAD's host that the file could not get downloaded
+        (e.g. URL change, URL down, ...)
 
     Returns
     =======
     success: 'bool'
         True if request was successful
     """
 
-    download_repeatermap_raw_data_to_local_file()
-    download_hearham_raw_data_to_local_file()
+    download_repeatermap_raw_data_to_local_file(apprise_config_file=apprise_config_file)
+    download_hearham_raw_data_to_local_file(apprise_config_file=apprise_config_file)
     success, local_repeatermap_json = create_native_mpad_repeater_data()
     if success:
         success = write_mpad_repeater_data_to_disc(
@@ -740,8 +762,9 @@ def get_nearest_repeater(
 
 
 def download_hearham_raw_data_to_local_file(
-    url: str = "https://hearham.com/api/repeaters/v1",
+    file_url: str = "https://hearham.com/api/repeaters/v1",
     hearham_raw_data_file: str = mpad_hearham_raw_data_filename,
+    apprise_config_file: str = None,
 ):
     """
     Downloads the repeatermap.de data and write it to a file 'as is'
@@ -762,7 +785,7 @@ def download_hearham_raw_data_to_local_file(
     success = False
     absolute_path_filename = build_full_pathname(file_name=hearham_raw_data_file)
     try:
-        resp = requests.get(url)
+        resp = requests.get(file_url)
     except requests.exceptions.RequestException as e:
         logger.error(msg="{e}")
         resp = None
@@ -778,6 +801,18 @@ def download_hearham_raw_data_to_local_file(
                 logger.info(
                     msg=f"Cannot write hearham.com data to local disc file '{absolute_path_filename}'"
                 )
+
+    # Generate an Apprise message in case we were unable to download the file
+    if not success and apprise_config_file:
+        # send_apprise_message will check again if the file exists or not
+        # Therefore, we can skip any further detection steps here
+        send_apprise_message(
+            message_header="MPAD External Dependency Error",
+            message_body=f"Unable to download HearHam data file from '{file_url}'",
+            apprise_config_file=apprise_config_file,
+            message_attachment=None,
+        )
+
     return success