From 7e4c6a3830e35e62c127bb8039492b905f411c0b Mon Sep 17 00:00:00 2001 From: Joerg Schultze-Lutter Date: Fri, 18 Feb 2022 22:20:19 +0100 Subject: [PATCH] Several bug fixes --- src/mowas-pwb.py | 10 ++++- src/mowas.py | 95 ++++++++++++++++++++++++++++++++---------- src/outputgenerator.py | 7 ++++ src/utils.py | 18 ++++++++ 4 files changed, 108 insertions(+), 22 deletions(-) diff --git a/src/mowas-pwb.py b/src/mowas-pwb.py index 8751dc1..2ce7ba2 100644 --- a/src/mowas-pwb.py +++ b/src/mowas-pwb.py @@ -71,6 +71,7 @@ mowas_email_recipient, mowas_enable_covid_content, mowas_target_language, + mowas_localfile, ) = get_command_line_params() # User wants to disable both DAPNET and Telegram? @@ -318,6 +319,7 @@ mowas_dapnet_high_prio_level=mowas_dapnet_high_prio_level, mowas_active_categories=mowas_active_categories, enable_covid_messaging=mowas_enable_covid_content, + local_file_name=mowas_localfile, ) # Did we find some new message updates that we need to send to the user? @@ -354,18 +356,24 @@ # Finally, check if we need to send something via Email if mowas_email_enabled: - logger.info(msg="Preparing Email notifications") + logger.info(msg="Generating Email notifications") success = generate_email_messages( mowas_messages_to_send=mowas_messages_to_send, warncell_data=warncell_data, smtpimap_email_address=mowas_smtpimap_email_address, smtpimap_email_password=mowas_smtpimap_email_password, mail_recipient=mowas_email_recipient, + smtp_server_address=mowas_smtp_server_address, + smtp_server_port=mowas_smtp_server_port, ) logger.info(msg=f"Email message success: {success}") else: logger.debug(msg="No new messages found") + # Exit the loop if we were testing with a local file + if mowas_localfile: + break + # Finally, go to sleep logger.debug(msg=f"Entering sleep mode for {mowas_run_interval} mins...") time.sleep(mowas_run_interval * 60) diff --git a/src/mowas.py b/src/mowas.py index 3c467dd..0cfc7e4 100644 --- a/src/mowas.py +++ b/src/mowas.py @@ -21,7 +21,7 @@ import numpy as np from shapely.geometry import Point, Polygon from expiringdict import ExpiringDict -from utils import remove_html_content, get_program_config_from_file +from utils import remove_html_content from translate import translate_text_list from geodata import ( get_reverse_geopy_data, @@ -32,6 +32,7 @@ import requests import sys from pprint import pformat +import json # Set up the global logger variable logging.basicConfig( @@ -96,6 +97,7 @@ def process_mowas_data( deepl_api_key: str = None, aprs_latitude: float = None, aprs_longitude: float = None, + local_file_name: str = None, ): """ Process our MOWAS data and return a dictionary with messages that are to be sent to the user @@ -116,8 +118,10 @@ def process_mowas_data( mowas_active_categories: 'list' List of active categories (from the program's config file) enable_covid_messaging: 'bool' - - + Enables Covid messages (usually, they get suppressed) + local_file_name: 'str" + For local testing; digests local file instead of online content + (if filename has been supplied) target_language: 'str' If not 'None', this is the language that we need to supply in addition to the German data deepl_api_key: 'str' @@ -186,17 +190,39 @@ def process_mowas_data( # Alert or Update message got_alert_or_update = False + # Boolean marker for quick-and-dirty file handler + processed_our_file = False + # For each of our own categories, try to download the MOWAS data for mowas_category in mowas_dictionary: # Only process this category if it is set as "active" # in the program config file if mowas_category in mowas_active_categories: - # OK, let's try to get that data from the government server - success, json_data = download_mowas_data( - base_url="https://warnung.bund.de", - url_path=mowas_dictionary[mowas_category], - ) - logger.debug(msg=f"Processing mowas_category {mowas_category}: {success}") + # Check if we have a local file name for testing + if local_file_name: + # VERY quick and dirty handler - needs refactoring :) + # did we already get through this loop? + # yes => exit loop + if processed_our_file: + break + logger.info( + msg=f"Entering local file test mode; file '{local_file_name}'" + ) + with open(f"{local_file_name}", "r") as f: + if f.mode == "r": + json_data = json.load(f) + processed_our_file = True + success = True + else: + # do the real thing + # OK, let's try to get that data from the government server + success, json_data = download_mowas_data( + base_url="https://warnung.bund.de", + url_path=mowas_dictionary[mowas_category], + ) + logger.debug( + msg=f"Processing mowas_category {mowas_category}: {success}" + ) if success: for element in json_data: # general marker which tells us whether we should send this message @@ -364,8 +390,16 @@ def process_mowas_data( longitude = coord[1] # Let's create our coordinate that we want to check + area_match = False try: p = Point(latitude, longitude) + + # Check if we are either inside of the polygon or + # touch its borders + area_match = p.within(poly) or p.intersects( + poly + ) + except Exception as ex: exc_type, exc_value, exc_tb = sys.exc_info() logger.info( @@ -383,10 +417,6 @@ def process_mowas_data( logger.info(msg=pformat(locals())) sys.exit(0) - # Check if we are either inside of the polygon or - # touch its borders - area_match = p.within(poly) or p.intersects(poly) - # and set our global marker if we have found something area_matches_with_user_latlon = ( True @@ -490,8 +520,13 @@ def process_mowas_data( "utm": utm, "aprs_coordinates": aprs, } - if mowas_coordinates not in coords_matching_latlon: - coords_matching_latlon.append(mowas_coordinates) + if ( + mowas_coordinates + not in coords_matching_latlon + ): + coords_matching_latlon.append( + mowas_coordinates + ) # We went through all areas - now let's see of we found something if area_matches_with_user_latlon: @@ -502,15 +537,33 @@ def process_mowas_data( # Check if the message contains Covid content and flag the # message as "not to be added" if related content has been found if not enable_covid_messaging: + # some of these field values can have None data type + # We only use this data temporarily so if the field value is None, + # we replace this value with "" for our quick check + _headline = ( + mowas_headline.lower() if mowas_headline else "" + ) + _description = ( + mowas_description.lower() + if mowas_description + else "" + ) + _instruction = ( + mowas_instruction.lower() + if mowas_instruction + else "" + ) content = [ - mowas_headline.lower(), - mowas_description.lower(), - mowas_instruction.lower(), + _headline, + _description, + _instruction, ] - # fmt: off - if any("covid" in s for s in content) or any("corona" in s for s in content): + if ( + any("covid" in s for s in content) + or any("corona" in s for s in content) + or any("impfung" in s for s in content) + ): add_data = False - # fmt: on # Add to the expiring dict unless it is a "Cancel" msg if mowas_msgtype != "Cancel": diff --git a/src/outputgenerator.py b/src/outputgenerator.py index dd25cc2..313aada 100644 --- a/src/outputgenerator.py +++ b/src/outputgenerator.py @@ -455,7 +455,9 @@ def generate_email_messages( severity = mowas_messages_to_send[mowas_message_id]["severity"] description = mowas_messages_to_send[mowas_message_id]["description"] instruction = mowas_messages_to_send[mowas_message_id]["instruction"] + instruction = "" if not instruction else instruction contact = mowas_messages_to_send[mowas_message_id]["contact"] + contact = "" if not contact else contact sent = mowas_messages_to_send[mowas_message_id]["sent"] msgtype = mowas_messages_to_send[mowas_message_id]["msgtype"] areas = mowas_messages_to_send[mowas_message_id]["areas"] @@ -468,6 +470,7 @@ def generate_email_messages( lang_headline = mowas_messages_to_send[mowas_message_id]["lang_headline"] lang_description = mowas_messages_to_send[mowas_message_id]["lang_description"] lang_instruction = mowas_messages_to_send[mowas_message_id]["lang_instruction"] + lang_instruction = "" if not lang_instruction else lang_instruction lang_contact = mowas_messages_to_send[mowas_message_id]["lang_contact"] else: lang_headline = lang_instruction = lang_contact = lang_description = None @@ -616,6 +619,10 @@ def generate_email_messages( mail_subject_message = mail_subject_message.replace( "REPLACE_DATETIME_CREATED", msg_string ) + logger.info( + msg=f"Sending Email Message to {smtp_server_address}:{smtp_server_port}" + ) + # Ultimately, send this particular message via Email and then loop to the next one success = send_email_message( plaintext_message=plaintext_message, diff --git a/src/utils.py b/src/utils.py index d2899a8..4169558 100644 --- a/src/utils.py +++ b/src/utils.py @@ -26,6 +26,7 @@ import sys import argparse import string +import os.path # Set up the global logger variable logging.basicConfig( @@ -477,11 +478,19 @@ def get_command_line_params(): help="ISO639-1 target language for MOWAS messages (will not be invoked for DAPNET messages)", ) + parser.add_argument( + "--localfile", + default=None, + type=str, + help="local MOWAS JSON file (for testing purposes only)", + ) + parser.set_defaults(add_example_data=False) args = parser.parse_args() mowas_configfile = args.configfile.name + mowas_localfile = args.localfile mowas_standard_run_interval = args.standard_run_interval mowas_emergency_run_interval = args.emergency_run_interval mowas_dapnet_destination_callsign = args.dapnet_destination_callsign @@ -498,6 +507,14 @@ def get_command_line_params(): mowas_enable_covid_content = args.enable_covid_content mowas_target_language = args.target_language + # Did the user specify an optional JSON file for testing? + # if yes, check if that file exists + if mowas_localfile: + if not os.path.isfile(mowas_localfile): + raise ValueError( + f"Local MOWAS test file '{mowas_localfile}' does not exist" + ) + # Convert requested call sign to upper case whereas present if mowas_follow_the_ham: mowas_follow_the_ham = mowas_follow_the_ham.upper() @@ -526,6 +543,7 @@ def get_command_line_params(): mowas_email_recipient, mowas_enable_covid_content, mowas_target_language, + mowas_localfile, )