Narrow allowed payload-system time window, add warnings for various i…

…ssues.

horusdemodlib/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.11"
+__version__ = "0.3.12"

horusdemodlib/decoder.py

@@ -178,6 +178,11 @@ def decode_packet(data:bytes, packet_format:dict = None, ignore_crc:bool = False
             _ukhas_fields.append(_decoded_str)
 
 
+    # Check the payload ID if > 256 for a Horus v2 packet.
+    if _output['modulation'] == 'Horus Binary v2':
+        if _raw_fields[0] < 256:
+            logging.warning("Found Payload ID < 256 in a Horus Binary v2 packet! This may lead to undefined behaviour. Please use a payload ID > 256!")
+
     # Convert to a UKHAS-compliant string.
     _ukhas_str = ",".join(_ukhas_fields)
     _ukhas_crc = ukhas_crc(_ukhas_str.encode('ascii'))

horusdemodlib/sondehubamateur.py

@@ -155,6 +155,18 @@ def reformat_data(self, telemetry):
         # Datetime
         try:
             _datetime = fix_datetime(telemetry['time'])
+
+            # Compare system time and payload time, to look for issues where system time is way out.
+            _timedelta = abs((_datetime - datetime.datetime.utcnow()).total_seconds())
+
+            if _timedelta > 3*60:
+                # Greater than 3 minutes time difference. Discard packet in this case.
+                self.log_error("Payload and Receiver times are offset by more than 3 minutes. Either payload does not have GNSS lock, or your system time is not set correctly. Not uploading.")
+                return None
+
+            if _timedelta > 60:
+                self.log_warning("Payload and Receiver times are offset by more than 1 minute. Either payload does not have GNSS lock, or your system time is not set correctly.")
+
             _output["datetime"] = _datetime.strftime(
                 "%Y-%m-%dT%H:%M:%S.%fZ"
             )
@@ -165,6 +177,8 @@ def reformat_data(self, telemetry):
             self.log_debug("Offending datetime_dt: %s" % str(telemetry["time"]))
             return None
 
+
+
         # Callsign - Break if this is an unknown payload ID.
         if telemetry["callsign"] == "UNKNOWN_PAYLOAD_ID":
             self.log_error("Not uploading telemetry from unknown payload ID. Is your payload ID list old?")
@@ -183,10 +197,6 @@ def reformat_data(self, telemetry):
         _output["lon"] = telemetry["longitude"]
         _output["alt"] = telemetry["altitude"]
 
-        if (_output["lat"] == 0.0) and (_output["lon"] == 0.0):
-            self.log_error("Lat/Lon both 0.0 - not uploading telemetry.")
-            return None
-
         # # Optional Fields
         if "temperature" in telemetry:
             if telemetry["temperature"] > -273.15:
@@ -195,10 +205,6 @@ def reformat_data(self, telemetry):
         if "satellites" in telemetry:
             _output["sats"] = telemetry["satellites"]
 
-            if _output["sats"] == 0:
-                self.log_error("Satellites field provided, and is 0. Not uploading due to potentially inaccurate position.")
-                return None
-
         if "battery_voltage" in telemetry:
             if telemetry["battery_voltage"] >= 0.0:
                 _output["batt"] = telemetry["battery_voltage"]
@@ -338,7 +344,7 @@ def upload_telemetry(self, telem_list):
                 _upload_success = True
                 break
 
-            elif _req.status_code == 500:
+            elif _req.status_code in [500,501,502,503,504]:
                 # Server Error, Retry.
                 _retries += 1
                 continue

horusdemodlib/uploader.py

@@ -98,6 +98,9 @@ def main():
     else:
         _logfile = None
 
+    # Some variables to handle re-downloading of payload ID lists.
+    min_download_time = 30*60 # Only try and download new payload ID / custom field lists every 30 min.
+    next_download_time = time.time()
 
     if args.rtty == False:
 
@@ -106,7 +109,7 @@ def main():
             horusdemodlib.payloads.HORUS_PAYLOAD_LIST = read_payload_list(filename=args.payload_list)
             horusdemodlib.payloads.HORUS_CUSTOM_FIELDS = read_custom_field_list(filename=args.custom_fields)
         else:
-            # Downlaod
+            # Download
             horusdemodlib.payloads.HORUS_PAYLOAD_LIST = init_payload_id_list(filename=args.payload_list)
             horusdemodlib.payloads.HORUS_CUSTOM_FIELDS = init_custom_field_list(filename=args.custom_fields)
 
@@ -121,16 +124,6 @@ def main():
     else:
         _listener_freq_str = ""
 
-    # Habitat uploader disabled as of 2022-12-18
-    # habitat_uploader = HabitatUploader(
-    #     user_callsign = user_config['user_call'],
-    #     listener_lat = user_config['station_lat'],
-    #     listener_lon = user_config['station_lon'],
-    #     listener_radio = user_config['radio_comment'] + _listener_freq_str,
-    #     listener_antenna = user_config['antenna_comment'],
-    #     inhibit=args.noupload
-    # )
-
     if user_config['station_lat'] == 0.0 and user_config['station_lon'] == 0.0:
         _sondehub_user_pos = None
     else:
@@ -140,7 +133,7 @@ def main():
         upload_rate = 2,
         user_callsign = user_config['user_call'],
         user_position = _sondehub_user_pos,
-        user_radio = user_config['radio_comment'],
+        user_radio = user_config['radio_comment'] + _listener_freq_str,
         user_antenna = user_config['antenna_comment'],
         software_name = "horusdemodlib",
         software_version = horusdemodlib.__version__,
@@ -225,6 +218,23 @@ def main():
                 try:
                     _decoded = decode_packet(_binary_string)
                     # If we get here, we have a valid packet!
+
+                    if (_decoded['callsign'] == "UNKNOWN_PAYLOAD_ID") and not args.nodownload:
+                        # We haven't seen this payload ID. Our payload ID list might be out of date.
+                        if time.time() > next_download_time:
+                            logging.info("Observed unknown Payload ID, attempting to re-download lists.")
+
+                            # Download lists.
+                            horusdemodlib.payloads.HORUS_PAYLOAD_LIST = init_payload_id_list(filename=args.payload_list)
+                            horusdemodlib.payloads.HORUS_CUSTOM_FIELDS = init_custom_field_list(filename=args.custom_fields)
+
+                            # Update next_download_time so we don't re-attempt to download with every new packet.
+                            next_download_time = time.time() + min_download_time
+
+                            # Re-attempt to decode the packet.
+                            _decoded = decode_packet(_binary_string)
+                            if _decoded['callsign'] != "UNKNOWN_PAYLOAD_ID":
+                                logging.info(f"Payload found in new payload ID list - {_decoded['callsign']}")
 
                     # Add in SNR data.
                     _snr = demod_stats.snr

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "horusdemodlib"
-version = "0.3.11"
+version = "0.3.12"
 description = "Project Horus HAB Telemetry Demodulators"
 authors = ["Mark Jessop"]
 license = "LGPL-2.1-or-later"

scripts/docker_dual_4fsk.sh

@@ -47,4 +47,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($RXFREQ) and 'target' signal frequency ($MFSK1_CENTRE)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK1_LOWER --fsk_upper=$MFSK1_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK1_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK2_LOWER --fsk_upper=$MFSK2_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ ) > /dev/null
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK1_LOWER --fsk_upper=$MFSK1_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK1_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK2_LOWER --fsk_upper=$MFSK2_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ ) > /dev/null

scripts/docker_dual_rtty_4fsk.sh

@@ -49,4 +49,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($RXFREQ) and 'target' signal frequency ($RTTY_CENTRE / $MFSK_CENTRE)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python3 -m horusdemodlib.uploader --rtty --freq_hz $RXFREQ --freq_target_hz $RTTY_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK_CENTRE ) > /dev/null
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python3 -m horusdemodlib.uploader --rtty --freq_hz $RXFREQ --freq_target_hz $RTTY_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK_CENTRE ) > /dev/null

scripts/docker_single.sh

@@ -34,4 +34,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($SDR_RX_FREQ) and 'target' signal frequency ($RXFREQ)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $SDR_RX_FREQ --freq_target_hz $RXFREQ $@
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python3 -m horusdemodlib.uploader --freq_hz $SDR_RX_FREQ --freq_target_hz $RXFREQ $@

start_dual_4fsk.sh

@@ -29,6 +29,11 @@ MFSK2_SIGNAL=15000
 # but the higher the chance that the modem will lock on to a strong spurious signal.
 RXBANDWIDTH=5000
 
+# RTLSDR Device Selection
+# If you want to use a specific RTLSDR, you can change this setting to match the 
+# device identifier of your SDR (use rtl_test to get a list)
+SDR_DEVICE=0
+
 # Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a
 # preamplifier, you may want to experiment with different gain settings to optimize
 # your receiver setup.
@@ -113,4 +118,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($RXFREQ) and 'target' signal frequency ($MFSK1_CENTRE)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK1_LOWER --fsk_upper=$MFSK1_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK1_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK2_LOWER --fsk_upper=$MFSK2_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ ) > /dev/null
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK1_LOWER --fsk_upper=$MFSK1_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK1_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK2_LOWER --fsk_upper=$MFSK2_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ ) > /dev/null

start_dual_rtty_4fsk.sh

@@ -31,6 +31,11 @@ MFSK_SIGNAL=15000
 # but the higher the chance that the modem will lock on to a strong spurious signal.
 RXBANDWIDTH=8000
 
+# RTLSDR Device Selection
+# If you want to use a specific RTLSDR, you can change this setting to match the 
+# device identifier of your SDR (use rtl_test to get a list)
+SDR_DEVICE=0
+
 # Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a
 # preamplifier, you may want to experiment with different gain settings to optimize
 # your receiver setup.
@@ -115,4 +120,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($RXFREQ) and 'target' signal frequency ($RTTY_CENTRE / $MFSK_CENTRE)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python -m horusdemodlib.uploader --rtty --freq_hz $RXFREQ --freq_target_hz $RTTY_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK_CENTRE ) > /dev/null
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python -m horusdemodlib.uploader --rtty --freq_hz $RXFREQ --freq_target_hz $RTTY_CENTRE ) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python -m horusdemodlib.uploader --freq_hz $RXFREQ --freq_target_hz $MFSK_CENTRE ) > /dev/null

start_rtlsdr.sh

@@ -15,6 +15,12 @@ cd /home/pi/horusdemodlib/
 # Note: The SDR will be tuned to RXBANDWIDTH/2 below this frequency.
 RXFREQ=434200000
 
+
+# RTLSDR Device Selection
+# If you want to use a specific RTLSDR, you can change this setting to match the 
+# device identifier of your SDR (use rtl_test to get a list)
+SDR_DEVICE=0
+
 # Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a
 # preamplifier, you may want to experiment with different gain settings to optimize
 # your receiver setup.
@@ -91,4 +97,4 @@ fi
 # Start the receive chain.
 # Note that we now pass in the SDR centre frequency ($SDR_RX_FREQ) and 'target' signal frequency ($RXFREQ)
 # to enable providing additional metadata to Habitat / Sondehub.
-rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python -m horusdemodlib.uploader --freq_hz $SDR_RX_FREQ --freq_target_hz $RXFREQ $@
+rtl_fm -M raw -F9 -d $SDR_DEVICE -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python -m horusdemodlib.uploader --freq_hz $SDR_RX_FREQ --freq_target_hz $RXFREQ $@

user.env.example

@@ -1,5 +1,10 @@
 ### General SDR settings ###
 
+# RTLSDR Device Selection
+# If you want to use a specific RTLSDR, you can change this setting to match the 
+# device identifier of your SDR (use rtl_test to get a list)
+SDR_DEVICE=0
+
 # Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a
 # preamplifier, you may want to experiment with different gain settings to optimize
 # your receiver setup.