diff --git a/.github/workflows/flink.yaml b/.github/workflows/flink.yaml index aaa9f884..6d015d6c 100644 --- a/.github/workflows/flink.yaml +++ b/.github/workflows/flink.yaml @@ -104,6 +104,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -r dev-requirements.txt python -m pip install -e .[flink] + python -m pip install -U ${{ matrix.recipes-version }} - name: Set up min.io as a k3s service run: | diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 9d4c51ae..1036f633 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -45,7 +45,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r dev-requirements.txt - python -m pip install -e . + python -m pip install -e ".[flink]" python -m pip install -U ${{ matrix.recipes-version }} - name: Test with pytest diff --git a/pangeo_forge_runner/commands/bake.py b/pangeo_forge_runner/commands/bake.py index 4fffca99..f9638f9b 100644 --- a/pangeo_forge_runner/commands/bake.py +++ b/pangeo_forge_runner/commands/bake.py @@ -5,6 +5,7 @@ import re import string import time +from importlib.metadata import distributions from pathlib import Path import escapism @@ -173,6 +174,10 @@ def start(self): """ Start the baking process """ + if not "pangeo-forge-recipes" in [d.metadata["Name"] for d in distributions()]: + raise ValueError( + "To use the `bake` command, `pangeo-forge-recipes` must be installed." + ) # Create our storage configurations. Traitlets will do its magic, populate these # with appropriate config from config file / commandline / defaults. target_storage = TargetStorage(parent=self) diff --git a/setup.py b/setup.py index 46c72688..b498f94e 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ install_requires=[ "jupyter-repo2docker", "ruamel.yaml", - "pangeo-forge-recipes>=0.9.2", "escapism", "jsonschema", "traitlets", diff --git a/tests/integration/flink/test_flink_integration.py b/tests/integration/flink/test_flink_integration.py index 963d78f3..84e13799 100644 --- a/tests/integration/flink/test_flink_integration.py +++ b/tests/integration/flink/test_flink_integration.py @@ -4,6 +4,7 @@ import time from importlib.metadata import version +import pytest import xarray as xr from packaging.version import parse as parse_version @@ -20,6 +21,10 @@ def test_flink_bake(minio_service, flinkversion, pythonversion, beamversion): recipe_version_ref = str(pfr_version) else: recipe_version_ref = "0.9.x" + pytest.xfail( + f"{pfr_version = }, which is < 0.10. " + "Flink tests timeout with this recipes version, so we xfail this test." + ) bucket = "s3://gpcp-out" config = { diff --git a/tests/unit/test_bake.py b/tests/unit/test_bake.py index 40da0a4d..9ea12079 100644 --- a/tests/unit/test_bake.py +++ b/tests/unit/test_bake.py @@ -1,8 +1,9 @@ import json import re import subprocess +import sys import tempfile -from importlib.metadata import version +from importlib.metadata import distributions, version import pytest import xarray as xr @@ -11,6 +12,45 @@ from pangeo_forge_runner.commands.bake import Bake +@pytest.fixture +def recipes_uninstalled(): + """Uninstall `pangeo-forge-recipes` for `test_bake_requires_recipes_installed`.""" + # first confirm that it's installed to begin with + assert "pangeo-forge-recipes" in [d.metadata["Name"] for d in distributions()] + # and capture the version, which we'll reinstall after the test + recipes_version = parse_version(version("pangeo-forge-recipes")) + # now uninstall it + uninstall = subprocess.run( + f"{sys.executable} -m pip uninstall pangeo-forge-recipes -y".split() + ) + assert uninstall.returncode == 0 + assert "pangeo-forge-recipes" not in [d.metadata["Name"] for d in distributions()] + # and yield to the test + yield True + # test is complete, now reinstall pangeo-forge-recipes in the test env + reinstall = subprocess.run( + f"{sys.executable} -m pip install pangeo-forge-recipes=={recipes_version}".split() + ) + assert reinstall.returncode == 0 + # make sure it's there, and in the expected version + assert "pangeo-forge-recipes" in [d.metadata["Name"] for d in distributions()] + assert parse_version(version("pangeo-forge-recipes")) == recipes_version + + +def test_bake_requires_recipes_installed(recipes_uninstalled): + """`pangeo-forge-runner` does not require `pangeo-forge-recipes` to be installed, + but `pangeo-forge-recipes` *is* required to use the `bake` command, so test that + we get a descriptive error if we try to invoke this command without it installed. + """ + assert recipes_uninstalled + bake = Bake() + with pytest.raises( + ValueError, + match="To use the `bake` command, `pangeo-forge-recipes` must be installed.", + ): + bake.start() + + @pytest.mark.parametrize( "job_name, raises", (