From ccb95670b37427ff4b5806339a469ad979df0e00 Mon Sep 17 00:00:00 2001 From: Joerg Schultze-Lutter Date: Thu, 26 Sep 2024 22:40:45 +0200 Subject: [PATCH 1/3] Initial modifications --- src/airport_data_modules.py | 18 +++++++++++ src/aprs_listener.py | 9 ++++-- src/messaging_modules.py | 4 +-- src/mpad_config.py | 2 +- src/parser_test.py | 60 +++++++++++++++++++++++++++---------- src/repeater_modules.py | 49 +++++++++++++++++++++++++----- src/skyfield_modules.py | 46 ++++++++++++++++++++++++---- 7 files changed, 154 insertions(+), 34 deletions(-) diff --git a/src/airport_data_modules.py b/src/airport_data_modules.py index 81eddb9..9188027 100644 --- a/src/airport_data_modules.py +++ b/src/airport_data_modules.py @@ -27,6 +27,7 @@ import logging from utility_modules import check_if_file_exists, build_full_pathname from mpad_config import mpad_airport_stations_filename +from messaging_modules import send_apprise_message # icao https://www.aviationweather.gov/docs/metar/stations.txt @@ -327,6 +328,7 @@ def validate_icao(icao_code: str): def update_local_airport_stations_file( airport_stations_filename: str = mpad_airport_stations_filename, + apprise_config_file: str = None, ): """ Imports the ICAO/IATA data from the web and saves it to a local file. @@ -336,6 +338,10 @@ def update_local_airport_stations_file( airport_stations_filename : 'str' This local file will hold the content from https://www.aviationweather.gov/docs/metar/stations.txt. + apprise_config_file: 'str' + Optional Apprise config file name which will be used in case + of errors, telling MPAD's host that the file could not get downloaded + (e.g. URL change, URL down, ...) Returns ======= @@ -366,6 +372,18 @@ def update_local_airport_stations_file( logger.debug( msg=f"Cannot update airport data to local file '{absolute_path_filename}'" ) + + # Generate an Apprise message in case we were unable to download the file + if not success and apprise_config_file: + # send_apprise_message will check again if the file exists or not + # Therefore, we can skip any further detection steps here + send_apprise_message( + message_header="MPAD External Dependency Error", + message_body=f"Unable to download airport data file from '{file_url}'", + apprise_config_file=apprise_config_file, + message_attachment=None, + ) + return success diff --git a/src/aprs_listener.py b/src/aprs_listener.py index e18f7eb..6c68151 100644 --- a/src/aprs_listener.py +++ b/src/aprs_listener.py @@ -529,15 +529,15 @@ def mycallback(raw_aprs_packet: dict): # # Refresh the local "airport stations" file logger.info(msg="Updating airport database ...") - update_local_airport_stations_file() + update_local_airport_stations_file(apprise_config_file=apprise_config_file) # # Refresh the local "repeatermap" file logger.info(msg="Updating repeater database ...") - update_local_repeatermap_file() + update_local_repeatermap_file(apprise_config_file=apprise_config_file) # # Update the satellite TLE file logger.info(msg="Updating satellite TLE and frequency database ...") - update_local_mpad_satellite_data() + update_local_mpad_satellite_data(apprise_config_file=apprise_config_file) # Now let's set up schedulers for the refresh process # These schedulers will download the file(s) every x days @@ -553,6 +553,7 @@ def mycallback(raw_aprs_packet: dict): id="airport_data", days=30, args=[], + kwargs={"apprise_config_file": apprise_config_file}, ) # Set up task for repeater data download every 7 days @@ -562,6 +563,7 @@ def mycallback(raw_aprs_packet: dict): id="repeatermap_data", days=7, args=[], + kwargs={"apprise_config_file": apprise_config_file}, ) # Set up task for satellite TLE / frequency data @@ -572,6 +574,7 @@ def mycallback(raw_aprs_packet: dict): id="tle_and_satfreq_data", days=2, args=[], + kwargs={"apprise_config_file": apprise_config_file}, ) # Set up task for the IMAP garbage collector - which will delete diff --git a/src/messaging_modules.py b/src/messaging_modules.py index 42ef9a7..abb1cba 100644 --- a/src/messaging_modules.py +++ b/src/messaging_modules.py @@ -60,7 +60,7 @@ def send_apprise_message( logger.debug(msg="Starting Apprise message processing") if not apprise_config_file or apprise_config_file == "NOT_CONFIGURED": - logger.debug(msg="Skipping post-mortem dump; message file is not configured") + logger.debug(msg="Skipping Apprise messaging; message file is not configured") return success if not check_if_file_exists(apprise_config_file): @@ -69,7 +69,7 @@ def send_apprise_message( ) return success - if not check_if_file_exists(message_attachment): + if message_attachment and not check_if_file_exists(message_attachment): logger.debug("Attachment file missing; disabling attachments") message_attachment = None diff --git a/src/mpad_config.py b/src/mpad_config.py index 5984f8d..1f10b66 100644 --- a/src/mpad_config.py +++ b/src/mpad_config.py @@ -22,7 +22,7 @@ # # Program version # -mpad_version: str = "0.61" +mpad_version: str = "0.62" # ########################### # Constants, do not change# diff --git a/src/parser_test.py b/src/parser_test.py index d6f67db..626c604 100644 --- a/src/parser_test.py +++ b/src/parser_test.py @@ -88,35 +88,65 @@ def testcall(message_text: str, from_callsign: str): logger.info(msg=pformat(response_parameters)) -def download_data_files_if_missing(): +def download_data_files_if_missing(force_download: bool = False): # if the user has never ever run the actual bot, some files might be missing, thus preventing us from # simulating the bot's actual behavior in real life. As a workaround, check if the files are missing # and download them, if necessary # + + # Read the config + # we only need the Apprise file name + ( + success, + aprsdotfi_api_key, + aprsis_callsign, + aprsis_passcode, + dapnet_login_callsign, + dapnet_login_passcode, + smtpimap_email_address, + smtpimap_email_password, + apprise_config_file, + ) = read_program_config() + assert success + # check if airport data file is present - if not check_if_file_exists( - build_full_pathname(mpad_config.mpad_airport_stations_filename) + if ( + not check_if_file_exists( + build_full_pathname(mpad_config.mpad_airport_stations_filename) + ) + or force_download ): logger.info("Updating local airport data file") - update_local_airport_stations_file(mpad_config.mpad_airport_stations_filename) + update_local_airport_stations_file( + airport_stations_filename=mpad_config.mpad_airport_stations_filename, + apprise_config_file=apprise_config_file, + ) # check if the satellite data is present - if not check_if_file_exists( - build_full_pathname(mpad_config.mpad_satellite_frequencies_filename) - ) or not check_if_file_exists( - build_full_pathname(mpad_config.mpad_tle_amateur_satellites_filename) + if ( + not check_if_file_exists( + build_full_pathname(mpad_config.mpad_satellite_frequencies_filename) + ) + or not check_if_file_exists( + build_full_pathname(mpad_config.mpad_tle_amateur_satellites_filename) + ) + or force_download ): logger.info("Updating local satellite data files") - update_local_mpad_satellite_data() + update_local_mpad_satellite_data(apprise_config_file=apprise_config_file) # check if the repeater data is present - if not check_if_file_exists( - build_full_pathname(mpad_config.mpad_repeatermap_raw_data_filename) - ) or not check_if_file_exists( - build_full_pathname(mpad_config.mpad_hearham_raw_data_filename) + if ( + not check_if_file_exists( + build_full_pathname(mpad_config.mpad_repeatermap_raw_data_filename) + ) + or not check_if_file_exists( + build_full_pathname(mpad_config.mpad_hearham_raw_data_filename) + ) + or force_download ): logger.info("Updating local repeater data files") - update_local_repeatermap_file() + update_local_repeatermap_file(apprise_config_file=apprise_config_file) def mpad_exception_handler(): @@ -195,7 +225,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): if __name__ == "__main__": # Check if the local database files exist and # create them, if necessary - download_data_files_if_missing() + download_data_files_if_missing(force_download=True) # Register the on_exit function to be called on program exit atexit.register(mpad_exception_handler) diff --git a/src/repeater_modules.py b/src/repeater_modules.py index 50aaf07..78e13ed 100644 --- a/src/repeater_modules.py +++ b/src/repeater_modules.py @@ -32,6 +32,7 @@ from geo_conversion_modules import haversine import logging import operator +from messaging_modules import send_apprise_message from mpad_config import ( mpad_hearham_raw_data_filename, @@ -46,8 +47,9 @@ def download_repeatermap_raw_data_to_local_file( - url: str = "http://www.repeatermap.de/apinew.php", + file_url: str = "http://www.repeatermap.de/apinew.php", repeatermap_raw_data_file: str = mpad_repeatermap_raw_data_filename, + apprise_config_file: str = None, ): """ Downloads the repeatermap.de data and write it to a file 'as is' @@ -59,6 +61,10 @@ def download_repeatermap_raw_data_to_local_file( Source URL where we will get the data from. repeatermap_raw_data_file: 'str' Filename of the target file which will hold the raw data + apprise_config_file: 'str' + Optional Apprise config file name which will be used in case + of errors, telling MPAD's host that the file could not get downloaded + (e.g. URL change, URL down, ...) Returns ======= @@ -68,7 +74,7 @@ def download_repeatermap_raw_data_to_local_file( success = False absolute_path_filename = build_full_pathname(file_name=repeatermap_raw_data_file) try: - resp = requests.get(url) + resp = requests.get(file_url) except requests.exceptions.RequestException as e: logger.error(msg="{e}") resp = None @@ -84,6 +90,18 @@ def download_repeatermap_raw_data_to_local_file( logger.info( msg=f"Cannot write repeatermap.de data to local disc file '{absolute_path_filename}'" ) + + # Generate an Apprise message in case we were unable to download the file + if not success and apprise_config_file: + # send_apprise_message will check again if the file exists or not + # Therefore, we can skip any further detection steps here + send_apprise_message( + message_header="MPAD External Dependency Error", + message_body=f"Unable to download repeatermap.de data file from '{file_url}'", + apprise_config_file=apprise_config_file, + message_attachment=None, + ) + return success @@ -528,7 +546,7 @@ def read_mpad_repeatermap_data_from_disc( return success, mpad_repeatermap -def update_local_repeatermap_file(): +def update_local_repeatermap_file(apprise_config_file: str = None): """ Wrapper method for importing the raw data from repeatermap.de, postprocessing the the data and finally writing the enriched data @@ -536,6 +554,10 @@ def update_local_repeatermap_file(): Parameters ========== + apprise_config_file: 'str' + Optional Apprise config file name which will be used in case + of errors, telling MPAD's host that the file could not get downloaded + (e.g. URL change, URL down, ...) Returns ======= @@ -543,8 +565,8 @@ def update_local_repeatermap_file(): True if request was successful """ - download_repeatermap_raw_data_to_local_file() - download_hearham_raw_data_to_local_file() + download_repeatermap_raw_data_to_local_file(apprise_config_file=apprise_config_file) + download_hearham_raw_data_to_local_file(apprise_config_file=apprise_config_file) success, local_repeatermap_json = create_native_mpad_repeater_data() if success: success = write_mpad_repeater_data_to_disc( @@ -740,8 +762,9 @@ def get_nearest_repeater( def download_hearham_raw_data_to_local_file( - url: str = "https://hearham.com/api/repeaters/v1", + file_url: str = "https://hearham.com/api/repeaters/v1", hearham_raw_data_file: str = mpad_hearham_raw_data_filename, + apprise_config_file: str = None, ): """ Downloads the repeatermap.de data and write it to a file 'as is' @@ -762,7 +785,7 @@ def download_hearham_raw_data_to_local_file( success = False absolute_path_filename = build_full_pathname(file_name=hearham_raw_data_file) try: - resp = requests.get(url) + resp = requests.get(file_url) except requests.exceptions.RequestException as e: logger.error(msg="{e}") resp = None @@ -778,6 +801,18 @@ def download_hearham_raw_data_to_local_file( logger.info( msg=f"Cannot write hearham.com data to local disc file '{absolute_path_filename}'" ) + + # Generate an Apprise message in case we were unable to download the file + if not success and apprise_config_file: + # send_apprise_message will check again if the file exists or not + # Therefore, we can skip any further detection steps here + send_apprise_message( + message_header="MPAD External Dependency Error", + message_body=f"Unable to download HearHam data file from '{file_url}'", + apprise_config_file=apprise_config_file, + message_attachment=None, + ) + return success diff --git a/src/skyfield_modules.py b/src/skyfield_modules.py index 54ba4df..71d9bf5 100644 --- a/src/skyfield_modules.py +++ b/src/skyfield_modules.py @@ -36,6 +36,7 @@ mpad_satellite_frequencies_filename, mpad_satellite_data_filename, ) +from messaging_modules import send_apprise_message logging.basicConfig( level=logging.INFO, format="%(asctime)s %(module)s -%(levelname)s- %(message)s" @@ -45,6 +46,7 @@ def download_and_write_local_tle_file( tle_filename: str = mpad_tle_amateur_satellites_filename, + apprise_config_file: str = None, ): """ Download the amateur radio satellite TLE data @@ -56,6 +58,10 @@ def download_and_write_local_tle_file( This local file will hold the content from http://www.celestrak.com/NORAD/elements/amateur.txt Default is "tle_amateur_satellites.txt" + apprise_config_file: 'str' + Optional Apprise config file name which will be used in case + of errors, telling MPAD's host that the file could not get downloaded + (e.g. URL change, URL down, ...) Returns ======= @@ -64,15 +70,15 @@ def download_and_write_local_tle_file( """ # This is the fixed name of the file that we are going to download - tle_data_file_url = "http://www.celestrak.com/NORAD/elements/amateur.txt" + file_url = "http://www.celestrak.com/NORAD/elements/amateur.txt" absolute_path_filename = build_full_pathname(file_name=tle_filename) success: bool = False # try to get the file try: - r = requests.get(tle_data_file_url) + r = requests.get(file_url) except: - logger.info(msg=f"Cannot download TLE data from {tle_data_file_url}") + logger.info(msg=f"Cannot download TLE data from {file_url}") r = None if r: if r.status_code == 200: @@ -85,11 +91,23 @@ def download_and_write_local_tle_file( logger.info( msg=f"Cannot update TLE data to file {absolute_path_filename}" ) + + # Generate an Apprise message in case we were unable to download the file + if not success and apprise_config_file: + # send_apprise_message will check again if the file exists or not + # Therefore, we can skip any further detection steps here + send_apprise_message( + message_header="MPAD External Dependency Error", + message_body=f"Unable to download TLE file file from '{file_url}'", + apprise_config_file=apprise_config_file, + message_attachment=None, + ) return success def download_and_write_local_satfreq_file( satfreq_filename: str = mpad_satellite_frequencies_filename, + apprise_config_file: str = None, ): """ Download the amateur radio satellite frequency data @@ -101,6 +119,10 @@ def download_and_write_local_satfreq_file( This local CSV file will hold the content from http://www.ne.jp/asahi/hamradio/je9pel/satslist.csv Default name is "satellite_frequencies.csv" + apprise_config_file: 'str' + Optional Apprise config file name which will be used in case + of errors, telling MPAD's host that the file could not get downloaded + (e.g. URL change, URL down, ...) Returns ======= @@ -128,6 +150,18 @@ def download_and_write_local_satfreq_file( logger.info( msg=f"Cannot write satellite frequency csv file {absolute_path_filename} to disc" ) + + # Generate an Apprise message in case we were unable to download the file + if not success and apprise_config_file: + # send_apprise_message will check again if the file exists or not + # Therefore, we can skip any further detection steps here + send_apprise_message( + message_header="MPAD External Dependency Error", + message_body=f"Unable to download satellite frequency csv file from '{file_url}'", + apprise_config_file=apprise_config_file, + message_attachment=None, + ) + return success @@ -330,7 +364,7 @@ def read_local_satfreq_file( return success, satellite_dictionary -def update_local_mpad_satellite_data(): +def update_local_mpad_satellite_data(apprise_config_file: str = None): """ Wrapper method for importing both TLE and Satellite Frequency data, blending the data records (whereas possible) and write the JSON content @@ -352,9 +386,9 @@ def update_local_mpad_satellite_data(): """ logger.info(msg="Generating local satellite frequencies database") - download_and_write_local_satfreq_file() + download_and_write_local_satfreq_file(apprise_config_file=apprise_config_file) logger.info(msg="Generating local TLE database") - download_and_write_local_tle_file() + download_and_write_local_tle_file(apprise_config_file=apprise_config_file) logger.info(msg="Creating blended satellite database version") success, json_satellite_data = create_native_satellite_data() if success: From a4e79bc4d55f5fcfde066ebb11119ba89fd2d32d Mon Sep 17 00:00:00 2001 From: Joerg Schultze-Lutter Date: Fri, 27 Sep 2024 22:33:33 +0200 Subject: [PATCH 2/3] Scheduler updates --- .github/workflows/dependabot.yml | 10 ---------- src/aprs_listener.py | 3 --- 2 files changed, 13 deletions(-) delete mode 100644 .github/workflows/dependabot.yml diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml deleted file mode 100644 index dfd0e30..0000000 --- a/.github/workflows/dependabot.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Set update schedule for GitHub Actions - -version: 2 -updates: - - - package-ecosystem: "github-actions" - directory: "/" - schedule: - # Check for updates to GitHub Actions every week - interval: "weekly" diff --git a/src/aprs_listener.py b/src/aprs_listener.py index 6c68151..0709230 100644 --- a/src/aprs_listener.py +++ b/src/aprs_listener.py @@ -552,7 +552,6 @@ def mycallback(raw_aprs_packet: dict): "interval", id="airport_data", days=30, - args=[], kwargs={"apprise_config_file": apprise_config_file}, ) @@ -562,7 +561,6 @@ def mycallback(raw_aprs_packet: dict): "interval", id="repeatermap_data", days=7, - args=[], kwargs={"apprise_config_file": apprise_config_file}, ) @@ -573,7 +571,6 @@ def mycallback(raw_aprs_packet: dict): "interval", id="tle_and_satfreq_data", days=2, - args=[], kwargs={"apprise_config_file": apprise_config_file}, ) From d1926871c9669fc7686b68b175f7298b16510938 Mon Sep 17 00:00:00 2001 From: Joerg Schultze-Lutter Date: Sun, 29 Sep 2024 21:30:43 +0200 Subject: [PATCH 3/3] Documentation updates --- docs/INSTALLATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index 44998e1..c6ef7c7 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -21,7 +21,7 @@ Currently, MPAD uses various APIs and access keys for its purposes: - aprs.fi - DAPNET API - SMTP/IMAP username and password -- optional: file name reference to an [Apprise](https://github.com/caronc/apprise) YAML config [file](https://github.com/caronc/apprise/wiki/config_yaml). If a reference to an Apprise YML config file is specified and a program crash occurs, MPAD will send a message (along with the stack trace) to the user hosting the program. +- optional: file name reference to an [Apprise](https://github.com/caronc/apprise) YAML config [file](https://github.com/caronc/apprise/wiki/config_yaml). If a reference to an Apprise YML config file is specified and a program crash occurs, MPAD will send a message (along with the stack trace) to the user hosting the program. Additionally, your local MPAD installation will send you notifications whenever external resources (e.g. TLE database, repeater data etc) cannot be downloaded. This will make it easier for MPAD admins to identify deprecated / changed resource URLs. If you want to host your own MPAD instance, you need to acquire your personal API access keys (e.g. for aprs.fi) and add these to MPAD's API config file (```mpad_api_access_keys.cfg```). An empty config template file is part of the repository. If you are not a registered DAPNET user, set the DAPNET callsign in the config file to N0CALL. When MPAD encounters this DAPNET user, it will refrain from sending content to DAPNET.