Merge pull request #126 from SNEWS2/internal_testing_logic

Internal testing logic

.github/workflows/tests.yml

@@ -22,8 +22,8 @@ jobs:
       matrix:
         # Test all supported Python versions under Ubuntu
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.11']
-#        python-version: ['3.9', '3.10', '3.11', '3.12']
+#        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.11', '3.12']
 #        # Additionally, test one Python version under MacOS and Windows, to detect OS-specific issues
 #        include:
 #          - os: macos-latest
@@ -52,8 +52,35 @@ jobs:
       - name: Install SNEWS_Coincidence_System
         run: |
           python -m poetry install
-      
+
+      - name: Install hop-client
+        run: |
+          python -m pip install setuptools wheel
+          python -m pip install hop-client==0.9.0
+          cd /home/runner/work/SNEWS_Coincidence_System/SNEWS_Coincidence_System
+
+      - shell: bash
+        env:
+          USERNAME: ${{ secrets.hop_user_name }}
+          PASSWORD: ${{ secrets.hop_user_password }}
+        run: |
+          sudo apt-get install -y expect
+          which expect
+          /usr/bin/expect << HOP
+          spawn hop auth add
+          expect "Username:"
+          send "$USERNAME\n"
+          expect "Password:"
+          send "$PASSWORD\n"
+          expect "Hostname (may be empty):"
+          send "kafka.scimma.org\n"
+          expect "Token endpoint (empty if not applicable):"
+          send "\n"
+          expect eof
+          HOP
+          hop auth locate
+
       # Run the unit tests
       - name: Test with pytest
         run: |
-          python -m poetry run pytest
+          python -m poetry run pytest snews_cs

pyproject.toml

@@ -29,20 +29,20 @@ classifiers = [
 snews_cs = "snews_cs.__main__:main"
 
 [tool.poetry.dependencies]
-python = ">=3.9, <3.13"
+python = ">=3.11, <3.13"
 click = "^8.1.7"
-hop-client = "^0.8.0"
+hop-client = "^0.9.0"
 ipython = "<7.33.0"  # Constrained by snews_pt
 numpy = "^1.26.0"
 pandas = "^2.1.1"
 python-dotenv = "0.19.2"  # Constrained by snews_pt
 pymongo = "^4.5.0"
-slack-sdk = "^3.22.0"
-snews-pt = "^1.3.3"
+slack-sdk = "^3.26.2"
 matplotlib = "^3.8.0"
 scipy = "^1.11.3"
 sqlalchemy = "^2.0.32"
 tabulate = "^0.9.0"
+snews-pt = { git = "https://github.com/SNEWS2/SNEWS_Publishing_Tools.git", branch = "main" }
 
 [tool.poetry.group.dev.dependencies]
 autopep8 = "^2.0.4"

requirements.txt

@@ -1,114 +1,65 @@
-adc-streaming==2.3.2 ; python_version >= "3.9" and python_version < "3.13"
-alabaster==0.7.16 ; python_version >= "3.9" and python_version < "3.13"
-ansicon==1.89.0 ; python_version >= "3.9" and python_version < "3.13" and platform_system == "Windows"
-appnope==0.1.3 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "darwin"
-astroid==3.0.2 ; python_version >= "3.9" and python_version < "3.13"
-atomicwrites==1.4.1 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32"
-attrs==21.4.0 ; python_version >= "3.9" and python_version < "3.13"
-authlib==1.3.0 ; python_version >= "3.9" and python_version < "3.13"
-babel==2.14.0 ; python_version >= "3.9" and python_version < "3.13"
-backcall==0.2.0 ; python_version >= "3.9" and python_version < "3.13"
-blessed==1.20.0 ; python_version >= "3.9" and python_version < "3.13"
-certifi==2024.2.2 ; python_version >= "3.9" and python_version < "3.13"
-cffi==1.16.0 ; python_version >= "3.9" and python_version < "3.13" and platform_python_implementation != "PyPy"
-charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "3.13"
-click==8.1.7 ; python_version >= "3.9" and python_version < "3.13"
-colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (platform_system == "Windows" or sys_platform == "win32")
-confluent-kafka==2.3.0 ; python_version >= "3.9" and python_version < "3.13"
-contourpy==1.2.0 ; python_version >= "3.9" and python_version < "3.13"
-cryptography==42.0.2 ; python_version >= "3.9" and python_version < "3.13"
-cycler==0.12.1 ; python_version >= "3.9" and python_version < "3.13"
-decorator==5.1.1 ; python_version >= "3.9" and python_version < "3.13"
-distlib==0.3.8 ; python_version >= "3.9" and python_version < "3.13"
-dnspython==2.5.0 ; python_version >= "3.9" and python_version < "3.13"
-docutils==0.17.1 ; python_version >= "3.9" and python_version < "3.13"
-editor==1.6.6 ; python_version >= "3.9" and python_version < "3.13"
-extras==1.0.0 ; python_version >= "3.9" and python_version < "3.13"
-fastavro==1.9.3 ; python_version >= "3.9" and python_version < "3.13"
-filelock==3.13.1 ; python_version >= "3.9" and python_version < "3.13"
-fixtures==3.0.0 ; python_version >= "3.9" and python_version < "3.13"
-fonttools==4.47.2 ; python_version >= "3.9" and python_version < "3.13"
-greenlet==3.0.3 ; python_version < "3.13" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.9"
-hop-client==0.8.0 ; python_version >= "3.9" and python_version < "3.13"
-idna==3.6 ; python_version >= "3.9" and python_version < "3.13"
-imagesize==1.4.1 ; python_version >= "3.9" and python_version < "3.13"
-importlib-metadata==7.0.1 ; python_version >= "3.9" and python_version < "3.10"
-importlib-resources==6.1.1 ; python_version >= "3.9" and python_version < "3.10"
-iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "3.13"
-inquirer==3.2.3 ; python_version >= "3.9" and python_version < "3.13"
-ipython==7.32.0 ; python_version >= "3.9" and python_version < "3.13"
-iso8601==2.1.0 ; python_version >= "3.9" and python_version < "3.13"
-jedi==0.19.1 ; python_version >= "3.9" and python_version < "3.13"
-jinja2==3.1.3 ; python_version >= "3.9" and python_version < "3.13"
-jinxed==1.2.1 ; python_version >= "3.9" and python_version < "3.13" and platform_system == "Windows"
-kiwisolver==1.4.5 ; python_version >= "3.9" and python_version < "3.13"
-markdown-it-py==2.2.0 ; python_version >= "3.9" and python_version < "3.13"
-markupsafe==2.1.4 ; python_version >= "3.9" and python_version < "3.13"
-matplotlib-inline==0.1.6 ; python_version >= "3.9" and python_version < "3.13"
-matplotlib==3.8.2 ; python_version >= "3.9" and python_version < "3.13"
-mdit-py-plugins==0.3.5 ; python_version >= "3.9" and python_version < "3.13"
-mdurl==0.1.2 ; python_version >= "3.9" and python_version < "3.13"
-mock==4.0.3 ; python_version >= "3.9" and python_version < "3.13"
-myst-parser==0.16.1 ; python_version >= "3.9" and python_version < "3.13"
-nose==1.3.7 ; python_version >= "3.9" and python_version < "3.13"
-numpy==1.26.3 ; python_version >= "3.9" and python_version < "3.13"
-packaging==23.2 ; python_version >= "3.9" and python_version < "3.13"
-pandas==2.2.0 ; python_version >= "3.9" and python_version < "3.13"
-parso==0.8.3 ; python_version >= "3.9" and python_version < "3.13"
-pbr==6.0.0 ; python_version >= "3.9" and python_version < "3.13"
-pexpect==4.9.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform != "win32"
-pickleshare==0.7.5 ; python_version >= "3.9" and python_version < "3.13"
-pillow==10.2.0 ; python_version >= "3.9" and python_version < "3.13"
-platformdirs==2.6.2 ; python_version >= "3.9" and python_version < "3.13"
-pluggy==1.4.0 ; python_version >= "3.9" and python_version < "3.13"
-prompt-toolkit==3.0.43 ; python_version >= "3.9" and python_version < "3.13"
-ptyprocess==0.7.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform != "win32"
-py==1.11.0 ; python_version >= "3.9" and python_version < "3.13"
-pycparser==2.21 ; python_version >= "3.9" and python_version < "3.13" and platform_python_implementation != "PyPy"
-pygments==2.17.2 ; python_version >= "3.9" and python_version < "3.13"
-pymongo==4.6.1 ; python_version >= "3.9" and python_version < "3.13"
-pyparsing==3.1.1 ; python_version >= "3.9" and python_version < "3.13"
-pytest==6.2.5 ; python_version >= "3.9" and python_version < "3.13"
-python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "3.13"
-python-dotenv==0.19.2 ; python_version >= "3.9" and python_version < "3.13"
-python-subunit==1.4.4 ; python_version >= "3.9" and python_version < "3.13"
-pytz==2024.1 ; python_version >= "3.9" and python_version < "3.13"
-pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13"
-readchar==4.0.5 ; python_version >= "3.9" and python_version < "3.13"
-requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13"
-runs==1.2.2 ; python_version >= "3.9" and python_version < "3.13"
-scipy==1.12.0 ; python_version >= "3.9" and python_version < "3.13"
-setuptools==62.1.0 ; python_version >= "3.9" and python_version < "3.13"
-six==1.14.0 ; python_version >= "3.9" and python_version < "3.13"
-slack-sdk==3.26.2 ; python_version >= "3.9" and python_version < "3.13"
-snews-pt==1.3.3 ; python_version >= "3.9" and python_version < "3.13"
-snowballstemmer==2.2.0 ; python_version >= "3.9" and python_version < "3.13"
-sphinx-autoapi==1.8.4 ; python_version >= "3.9" and python_version < "3.13"
-sphinx-rtd-theme==1.0.0 ; python_version >= "3.9" and python_version < "3.13"
-sphinx==4.5.0 ; python_version >= "3.9" and python_version < "3.13"
-sphinxcontrib-applehelp==1.0.8 ; python_version >= "3.9" and python_version < "3.13"
-sphinxcontrib-devhelp==1.0.6 ; python_version >= "3.9" and python_version < "3.13"
-sphinxcontrib-htmlhelp==2.0.5 ; python_version >= "3.9" and python_version < "3.13"
-sphinxcontrib-jsmath==1.0.1 ; python_version >= "3.9" and python_version < "3.13"
-sphinxcontrib-programoutput==0.17 ; python_version >= "3.9" and python_version < "3.13"
-sphinxcontrib-qthelp==1.0.7 ; python_version >= "3.9" and python_version < "3.13"
-sphinxcontrib-serializinghtml==1.1.10 ; python_version >= "3.9" and python_version < "3.13"
-sqlalchemy==2.0.32 ; python_version >= "3.9" and python_version < "3.13"
-tabulate==0.9.0 ; python_version >= "3.9" and python_version < "3.13"
-testrepository==0.0.20 ; python_version >= "3.9" and python_version < "3.13"
-testresources==2.0.1 ; python_version >= "3.9" and python_version < "3.13"
-testscenarios==0.5.0 ; python_version >= "3.9" and python_version < "3.13"
-testtools==2.5.0 ; python_version >= "3.9" and python_version < "3.13"
-toml==0.10.2 ; python_version >= "3.9" and python_version < "3.13"
-tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13"
-traitlets==5.14.1 ; python_version >= "3.9" and python_version < "3.13"
-typing-extensions==4.9.0 ; python_version >= "3.9" and python_version < "3.13"
-tzdata==2023.4 ; python_version >= "3.9" and python_version < "3.13"
-unidecode==1.3.8 ; python_version >= "3.9" and python_version < "3.13"
-urllib3==2.2.0 ; python_version >= "3.9" and python_version < "3.13"
-virtualenv==20.13.0 ; python_version >= "3.9" and python_version < "3.13"
-wcwidth==0.2.13 ; python_version >= "3.9" and python_version < "3.13"
-wheel==0.34.2 ; python_version >= "3.9" and python_version < "3.13"
-xmltodict==0.13.0 ; python_version >= "3.9" and python_version < "3.13"
-xmod==1.8.1 ; python_version >= "3.9" and python_version < "3.13"
-zipp==3.17.0 ; python_version >= "3.9" and python_version < "3.10"
+adc-streaming==2.4.0 ; python_version >= "3.11" and python_version < "3.13"
+ansicon==1.89.0 ; python_version >= "3.11" and python_version < "3.13" and platform_system == "Windows"
+appnope==0.1.4 ; python_version >= "3.11" and python_version < "3.13" and sys_platform == "darwin"
+authlib==1.3.1 ; python_version >= "3.11" and python_version < "3.13"
+backcall==0.2.0 ; python_version >= "3.11" and python_version < "3.13"
+blessed==1.20.0 ; python_version >= "3.11" and python_version < "3.13"
+certifi==2024.7.4 ; python_version >= "3.11" and python_version < "3.13"
+cffi==1.17.0 ; python_version >= "3.11" and python_version < "3.13" and platform_python_implementation != "PyPy"
+charset-normalizer==3.3.2 ; python_version >= "3.11" and python_version < "3.13"
+click==8.1.7 ; python_version >= "3.11" and python_version < "3.13"
+colorama==0.4.6 ; python_version >= "3.11" and python_version < "3.13" and (platform_system == "Windows" or sys_platform == "win32")
+confluent-kafka==2.5.0 ; python_version >= "3.11" and python_version < "3.13"
+contourpy==1.2.1 ; python_version >= "3.11" and python_version < "3.13"
+cryptography==43.0.0 ; python_version >= "3.11" and python_version < "3.13"
+cycler==0.12.1 ; python_version >= "3.11" and python_version < "3.13"
+decorator==5.1.1 ; python_version >= "3.11" and python_version < "3.13"
+dnspython==2.6.1 ; python_version >= "3.11" and python_version < "3.13"
+fastavro==1.9.5 ; python_version >= "3.11" and python_version < "3.13"
+fonttools==4.53.1 ; python_version >= "3.11" and python_version < "3.13"
+greenlet==3.0.3 ; python_version < "3.13" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.11"
+hop-client==0.9.0 ; python_version >= "3.11" and python_version < "3.13"
+idna==3.7 ; python_version >= "3.11" and python_version < "3.13"
+inquirer==2.10.1 ; python_version >= "3.11" and python_version < "3.13"
+ipython==7.32.0 ; python_version >= "3.11" and python_version < "3.13"
+jedi==0.19.1 ; python_version >= "3.11" and python_version < "3.13"
+jinxed==1.3.0 ; python_version >= "3.11" and python_version < "3.13" and platform_system == "Windows"
+kiwisolver==1.4.5 ; python_version >= "3.11" and python_version < "3.13"
+matplotlib-inline==0.1.7 ; python_version >= "3.11" and python_version < "3.13"
+matplotlib==3.9.2 ; python_version >= "3.11" and python_version < "3.13"
+numpy==1.26.4 ; python_version >= "3.11" and python_version < "3.13"
+packaging==24.1 ; python_version >= "3.11" and python_version < "3.13"
+pandas==2.2.2 ; python_version >= "3.11" and python_version < "3.13"
+parso==0.8.4 ; python_version >= "3.11" and python_version < "3.13"
+pexpect==4.9.0 ; python_version >= "3.11" and python_version < "3.13" and sys_platform != "win32"
+pickleshare==0.7.5 ; python_version >= "3.11" and python_version < "3.13"
+pillow==10.4.0 ; python_version >= "3.11" and python_version < "3.13"
+pluggy==1.5.0 ; python_version >= "3.11" and python_version < "3.13"
+prompt-toolkit==3.0.47 ; python_version >= "3.11" and python_version < "3.13"
+ptyprocess==0.7.0 ; python_version >= "3.11" and python_version < "3.13" and sys_platform != "win32"
+pycparser==2.22 ; python_version >= "3.11" and python_version < "3.13" and platform_python_implementation != "PyPy"
+pygments==2.18.0 ; python_version >= "3.11" and python_version < "3.13"
+pymongo==4.8.0 ; python_version >= "3.11" and python_version < "3.13"
+pyparsing==3.1.2 ; python_version >= "3.11" and python_version < "3.13"
+python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "3.13"
+python-dotenv==0.19.2 ; python_version >= "3.11" and python_version < "3.13"
+python-editor==1.0.4 ; python_version >= "3.11" and python_version < "3.13"
+pytz==2024.1 ; python_version >= "3.11" and python_version < "3.13"
+readchar==4.2.0 ; python_version >= "3.11" and python_version < "3.13"
+requests==2.32.3 ; python_version >= "3.11" and python_version < "3.13"
+scipy==1.14.0 ; python_version >= "3.11" and python_version < "3.13"
+setuptools==62.1.0 ; python_version >= "3.11" and python_version < "3.13"
+six==1.16.0 ; python_version >= "3.11" and python_version < "3.13"
+slack-sdk==3.31.0 ; python_version >= "3.11" and python_version < "3.13"
+snews-pt @ git+https://github.com/SNEWS2/SNEWS_Publishing_Tools.git@e513fb7561ad1a037c06489a8e4ecb20ef5efb88 ; python_version >= "3.11" and python_version < "3.13"
+sqlalchemy==2.0.32 ; python_version >= "3.11" and python_version < "3.13"
+tabulate==0.9.0 ; python_version >= "3.11" and python_version < "3.13"
+toml==0.10.2 ; python_version >= "3.11" and python_version < "3.13"
+tqdm==4.66.5 ; python_version >= "3.11" and python_version < "3.13"
+traitlets==5.14.3 ; python_version >= "3.11" and python_version < "3.13"
+typing-extensions==4.12.2 ; python_version >= "3.11" and python_version < "3.13"
+tzdata==2024.1 ; python_version >= "3.11" and python_version < "3.13"
+urllib3==2.2.2 ; python_version >= "3.11" and python_version < "3.13"
+wcwidth==0.2.13 ; python_version >= "3.11" and python_version < "3.13"
+wheel==0.34.2 ; python_version >= "3.11" and python_version < "3.13"
+xmltodict==0.13.0 ; python_version >= "3.11" and python_version < "3.13"

snews_cs/alert_pub.py

@@ -15,7 +15,7 @@ class AlertPublisher:
     """ Class to publish SNEWS SuperNova Alerts based on coincidence
 
     """
-    def __init__(self, env_path=None, verbose=True, auth=True, firedrill_mode=True):
+    def __init__(self, env_path=None, verbose=True, auth=True, firedrill_mode=True, is_test=False):
         """
         Alert publisher constructor 
         Parameters
@@ -36,6 +36,11 @@ def __init__(self, env_path=None, verbose=True, auth=True, firedrill_mode=True):
             self.alert_topic = os.getenv("ALERT_TOPIC")
         self.verbose = verbose
 
+        if is_test:
+            # use a test topic
+            self.alert_topic = os.getenv("CONNECTION_TEST_TOPIC")
+
+
     def __enter__(self):
         self.stream = Stream(until_eos=True, auth=self.auth).open(self.alert_topic, 'w')
         return self

snews_cs/core/logging.py

@@ -25,6 +25,15 @@
 log_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../logs"))
 
 log_file = f"{log_dir}/snews_cs.log"
+
+# Check if the directory exists, if not, create it
+if not os.path.exists(log_dir):
+    os.makedirs(log_dir)
+
+# Check if the log file exists, if not, create it
+if not os.path.isfile(log_file):
+    open(log_file, 'w').close()
+
 fh = FileHandler(log_file)
 
 formatter = Formatter(

snews_cs/cs_alert_schema.py

@@ -34,7 +34,7 @@ def id_format(self, num_detectors):
         else:
             return f'SNEWS_Coincidence_ALERT-UPDATE {date_time}'
 
-    def get_cs_alert_schema(self, data):
+    def get_cs_alert_schema(self, data, is_test=False):
         """ Create a message schema for alert.
             Internally called in hop_pub
         
@@ -50,10 +50,15 @@ def get_cs_alert_schema(self, data):
 
         """
         id = self.id_format(len(data['detector_names']))
+        alert_type = "TEST "+ data['alert_type'] if is_test else data['alert_type']
+        try:
+            far = f"Would happen every {data['false_alarm_prob']:.2e} year"
+        except:
+            far = data['false_alarm_prob']
         return {"_id": id,
-                "alert_type":data['alert_type'],
+                "alert_type":alert_type,
                 "server_tag": data['server_tag'],
-                "False Alarm Prob": f"Would happen every {data['false_alarm_prob']:.2e} year",
+                "False Alarm Prob": far,
                 "detector_names": data['detector_names'],
                 "sent_time": id.split(' ')[1],
                 "p_values": data['p_vals'],