feat: refresh F10.7 and ap data handling with partial data support

pymsis/__init__.py

@@ -7,4 +7,4 @@
 
 __version__ = importlib.metadata.version("pymsis")
 
-__all__ = ["__version__", "Variable", "calculate"]
+__all__ = ["Variable", "__version__", "calculate"]

pymsis/utils.py

@@ -43,8 +43,7 @@ def download_f107_ap() -> None:
        Space Weather, https://doi.org/10.1029/2020SW002641
     """
     warnings.warn(f"Downloading ap and F10.7 data from {_F107_AP_URL}")
-    req = urllib.request.urlopen(_F107_AP_URL)
-    with _F107_AP_PATH.open("wb") as f:
+    with _F107_AP_PATH.open("wb") as f, urllib.request.urlopen(_F107_AP_URL) as req:
         f.write(req.read())
 
 
@@ -90,27 +89,21 @@ def _load_f107_ap_data() -> dict[str, npt.NDArray]:
     # Use a buffer to read in and load so we can quickly get rid of
     # the extra "PRD" lines at the end of the file (unknown length
     # so we can't just go back in line lengths)
-    with _F107_AP_PATH.open() as fin:
-        with BytesIO() as fout:
-            for line in fin:
-                if "PRM" in line:
-                    # We don't want the monthly predicted values
-                    continue
-                if ",,,,,,,," in line:
-                    # We don't want lines with missing values
-                    continue
-                fout.write(line.encode("utf-8"))
-            fout.seek(0)
-            arr = np.loadtxt(
-                fout, delimiter=",", dtype=dtype, usecols=usecols, skiprows=1
-            )  # type: ignore
+    with _F107_AP_PATH.open() as fin, BytesIO() as fout:
+        for line in fin:
+            if "PRM" in line or ",,,,,,,," in line:
+                # We don't want the monthly predicted values or missing values
+                continue
+            fout.write(line.encode("utf-8"))
+        fout.seek(0)
+        arr = np.loadtxt(fout, delimiter=",", dtype=dtype, usecols=usecols, skiprows=1)  # type: ignore
 
     # transform each day's 8 3-hourly ap values into a single column
     ap = np.empty(len(arr) * 8, dtype=float)
     daily_ap = arr["Ap"].astype(float)
     dates = np.repeat(arr["date"], 8).astype("datetime64[m]")
     for i in range(8):
-        ap[i::8] = arr[f"ap{i+1}"]
+        ap[i::8] = arr[f"ap{i + 1}"]
         dates[i::8] += i * np.timedelta64(3, "h")
 
     # data file has missing values as negatives
@@ -208,6 +201,12 @@ def get_f107_ap(dates: npt.ArrayLike) -> tuple[npt.NDArray, npt.NDArray, npt.NDA
     """
     dates = np.asarray(dates, dtype=np.datetime64)
     data = _DATA or _load_f107_ap_data()
+    # If our requested data time is after the cached values we have,
+    # go and download a new file to refresh the local file cache
+    last_time_in_file = data["dates"][7::8][~data["warn_data"]].max()
+    if np.any((dates > last_time_in_file) & (dates < np.datetime64("now"))):
+        download_f107_ap()
+        data = _load_f107_ap_data()
 
     data_start = data["dates"][0]
     data_end = data["dates"][-1]

pyproject.toml

@@ -71,6 +71,10 @@ testpaths = [
 addopts = [
     "--import-mode=importlib",
 ]
+filterwarnings = [
+# Ignore warnings loading from file specifically
+    'ignore:Downloading ap and F10.7 data from file:UserWarning',
+]
 
 [tool.cibuildwheel]
 # skip Python <3.10

tests/conftest.py

@@ -6,21 +6,15 @@
 
 
 @pytest.fixture(autouse=True)
-def local_path(monkeypatch):
-    # Update the data location to our test data
-    test_file = Path(__file__).parent / "f107_ap_test_data.txt"
+def _path_setup(monkeypatch, tmp_path):
     # Monkeypatch the url and expected download location, so we aren't
     # dependent on an internet connection.
-    monkeypatch.setattr(utils, "_F107_AP_PATH", test_file)
-    return test_file
-
+    monkeypatch.setattr(utils, "_F107_AP_PATH", tmp_path / "f107_ap_test_data.txt")
 
-@pytest.fixture(autouse=True)
-def remote_path(monkeypatch, local_path):
     # Update the remote URL to point to a local file system test path
     # by prepending file:// so that it can be opened by urlopen()
-    test_url = local_path.absolute().as_uri()
+    test_file = Path(__file__).parent / "f107_ap_test_data.txt"
+    test_url = test_file.absolute().as_uri()
     # Monkeypatch the url and expected download location, so we aren't
     # dependent on an internet connection.
     monkeypatch.setattr(utils, "_F107_AP_URL", test_url)
-    return test_url

tests/test_utils.py

@@ -1,3 +1,6 @@
+from pathlib import Path
+from unittest.mock import patch
+
 import numpy as np
 import pytest
 from numpy.testing import assert_allclose, assert_array_equal
@@ -130,3 +133,41 @@ def test_get_f107_ap_interpolated_warns(dates):
         UserWarning, match="There is data that was either interpolated or"
     ):
         utils.get_f107_ap(dates)
+
+
+@patch("pymsis.utils.download_f107_ap")
+def test_auto_refresh(download_data_mock, monkeypatch):
+    test_file = Path(__file__).parent / "f107_ap_test_data.txt"
+    # Monkeypatch the url and expected download location, so we aren't
+    # dependent on an internet connection.
+    monkeypatch.setattr(utils, "_F107_AP_PATH", test_file)
+
+    def call_with_time(time):
+        try:
+            utils.get_f107_ap(time)
+        except ValueError:
+            # There is no data in our test file for this, so we will error later
+            # But this is enough to trigger an attempt at a refresh
+            pass
+
+    # Should not trigger a refresh, data before the time in the file
+    call_with_time(np.datetime64("1990-12-31T23:00"))
+    assert download_data_mock.call_count == 0
+
+    # Final observed time in the file
+    call_with_time(np.datetime64("2000-12-29T21:00"))
+    assert download_data_mock.call_count == 0
+
+    # One hour beyond our current time shouldn't trigger a refresh
+    # there would be no data to get for that time period
+    call_with_time(np.datetime64("now") + np.timedelta64(1, "h"))
+    assert download_data_mock.call_count == 0
+
+    # Within the predicted data in the file should try to get a refresh
+    with pytest.warns(UserWarning, match="There is data that was either"):
+        call_with_time(np.datetime64("2000-12-30T00:00"))
+    assert download_data_mock.call_count == 1
+
+    # Should trigger a refresh, after the data in the file but before current time
+    call_with_time(np.datetime64("2005-01-01T00:00"))
+    assert download_data_mock.call_count == 2  # noqa: PLR2004

tools/download_source.py

@@ -24,7 +24,7 @@ def get_source():
     if not Path("src/msis2.0/msis_init.F90").exists():
         # No source code yet, so go download and extract it
         try:
-            warnings.warn("Downloading the MSIS2.0 source code from " f"{MSIS20_FILE}")
+            warnings.warn(f"Downloading the MSIS2.0 source code from {MSIS20_FILE}")
             with urllib.request.urlopen(MSIS20_FILE) as stream:
                 tf = tarfile.open(fileobj=stream, mode="r|gz")
                 tf.extractall(path=Path("src/msis2.0"))
@@ -49,7 +49,7 @@ def get_source():
     if not Path("src/msis2.1/msis_init.F90").exists():
         # No source code yet, so go download and extract it
         try:
-            warnings.warn("Downloading the MSIS2.1 source code from " f"{MSIS21_FILE}")
+            warnings.warn(f"Downloading the MSIS2.1 source code from {MSIS21_FILE}")
             with urllib.request.urlopen(MSIS21_FILE) as stream:
                 tf = tarfile.open(fileobj=stream, mode="r|gz")
                 tf.extractall(path=Path("src/msis2.1"))
@@ -76,7 +76,7 @@ def get_source():
         local_msis00_path.parent.mkdir(parents=True, exist_ok=True)
         # No source code yet, so go download and extract it
         try:
-            warnings.warn("Downloading the MSIS-00 source code from " f"{MSIS00_FILE}")
+            warnings.warn(f"Downloading the MSIS-00 source code from {MSIS00_FILE}")
 
             with urllib.request.urlopen(MSIS00_FILE) as response:
                 with open(local_msis00_path, "wb") as f: