Filter out log messages about non-existing pars- reffiles

.github/workflows/ci.yml

@@ -21,12 +21,18 @@ jobs:
   test:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
     with:
+      setenv: |
+        CRDS_PATH: /tmp/data/crds_cache
+        CRDS_CLIENT_RETRY_COUNT: 3
+        CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
       envs: |
         - linux: py39-oldestdeps-cov-xdist
         - linux: py310-xdist
         - macos: py311-xdist
         - linux: py311-cov-xdist
           coverage: 'codecov'
+        - linux: py311-downstreamdeps-cov-xdist
+          coverage: 'codecov'
         - linux: py312-xdist
   test_downstream:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1

.github/workflows/ci_cron.yml

@@ -23,6 +23,10 @@ jobs:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
     if: (github.repository == 'spacetelescope/stpipe' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Weekly CI')))
     with:
+      setenv: |
+        CRDS_PATH: /tmp/data/crds_cache
+        CRDS_CLIENT_RETRY_COUNT: 3
+        CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
       envs: |
         - macos: py39-xdist
         - macos: py310-xdist

CHANGES.rst

@@ -6,6 +6,7 @@
 - Remove bundled ``configobj`` package in favor of the ``configobj`` package
   bundled into ``astropy``. [#122]
 - Fix ``datetime.utcnow()`` DeprecationWarning [#134]
+- Filter out log messages about non-existing pars- reffiles [#135]
 
 0.5.1 (2023-10-02)
 ==================

src/stpipe/crds_client.py

@@ -7,6 +7,7 @@
 directly in modules other than this crds_client so that dependency order and
 general integration can be managed here.
 """
+import logging
 import re
 
 import crds
@@ -49,6 +50,14 @@ def get_multiple_reference_paths(parameters, reference_file_types, observatory):
         raise TypeError("First argument must be a dict of parameters")
 
     log.set_log_time(True)
+
+    def parsfilter(record):
+        if "Error determining best reference for 'pars-" in record.getMessage():
+            return False
+        return True
+
+    logging.getLogger("CRDS").addFilter(parsfilter)
+
     return _get_refpaths(parameters, tuple(reference_file_types), observatory)
 
 

tests/test_crds_client.py

@@ -0,0 +1,30 @@
+import pytest
+from crds.core.exceptions import CrdsLookupError
+
+from stpipe import crds_client
+
+
+def test_missing_pars_log_filtering(capfd):
+    # CRDS needs stdatamodels to handle JWST lookups
+    pytest.importorskip("stdatamodels.jwst")
+
+    # A made-up pars- reffile will raise an exception in CRDS
+    with pytest.raises(CrdsLookupError):
+        crds_client.get_multiple_reference_paths(
+            parameters={
+                "meta.instrument.name": "NIRCAM",
+                "meta.telescope": "JWST",
+            },
+            reference_file_types=["pars-crunchyfrogstep"],
+            observatory="jwst",
+        )
+
+    # The following will always be true because of a bug in how pytest handles
+    # oddball logging setups, as used by crds.  See issue
+    # https://github.com/pytest-dev/pytest/issues/5997
+    # So don't rely on this test passing (currently) to be actually testing what
+    # you think it is.
+    capture = capfd.readouterr()
+    assert (
+        "Error determining best reference for 'pars-crunchyfrogstep'" not in capture.err
+    )

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 envlist =
     check-{style,security,build}
-    test{,-warnings,-cov,-xdist,-oldestdeps,-devdeps}
+    test{,-warnings,-cov,-xdist,-oldestdeps,-devdeps,-downstreamdeps}
     test-{jwst,romancal}-xdist
     build-{docs,dist}
 
@@ -37,6 +37,7 @@ description =
     jwst: of JWST pipeline
     romancal: of Romancal pipeline
     oldestdeps: with the oldest supported version of key dependencies
+    downstreamdeps: with the downstream packages that depend on stpipe
     warnings: treating warnings as errors
     cov: with coverage
     xdist: using parallel processing
@@ -49,12 +50,13 @@ deps =
     romancal: romancal[test] @ git+https://github.com/spacetelescope/romancal.git
     oldestdeps: minimum_dependencies
     devdeps: astropy>=0.0.dev0
+    downstreamdeps: stdatamodels
 pass_env =
     CRDS_*
     CI
 set_env =
     devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
-    jwst: CRDS_SERVER_URL=https://jwst-crds.stsci.edu
+    !romancal: CRDS_SERVER_URL=https://jwst-crds.stsci.edu
     romancal: CRDS_SERVER_URL=https://roman-crds.stsci.edu
 package =
     !cov: wheel