Merge pull request #638 from norlandrhagen/minio

Add `minio` to Integration Tests

.github/workflows/main.yaml

@@ -36,7 +36,7 @@ jobs:
       - name: 🌈 Install pangeo-forge-recipes package
         shell: bash -l {0}
         run: |
-          python -m pip install -e  ".[dev]"
+          python -m pip install -e  ".[test]"
       - name: 🧑‍💻 On the nightly run, test upstream dev versions
         if: |
           github.event_name == 'schedule'

.github/workflows/test-integration.yaml

@@ -43,12 +43,21 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: pip
           cache-dependency-path: pyproject.toml
+
+      - name: 'Setup minio'
+        run: |
+          wget --quiet https://dl.min.io/server/minio/release/linux-amd64/minio
+          chmod +x minio
+          mv minio /usr/local/bin/minio
+
+          minio --version
+
       - name: 🎯 Check cache hit
         run: echo '${{ steps.setup-python.outputs.cache-hit }}'
       - name: 🌈 Install pangeo-forge-recipes & pangeo-forge-runner
         shell: bash -l {0}
         run: |
-          python -m pip install -e ".[dev]"
+          python -m pip install -e ".[test,minio]"
           python -m pip install ${{ matrix.runner-version }}
       - name: 🏄‍♂️ Run Tests
         shell: bash -l {0}

examples/feedstock/noaa_oisst.py

@@ -40,5 +40,5 @@ def test_ds(store: zarr.storage.FSStore) -> zarr.storage.FSStore:
         store_name="noaa-oisst.zarr",
         combine_dims=pattern.combine_dim_keys,
     )
-    | "Test dataset" >> beam.Map(test_ds)
+    | beam.Map(test_ds)
 )

examples/runner-config/s3.json

@@ -0,0 +1,15 @@
+{
+  "Bake": {
+    "bakery_class": "pangeo_forge_runner.bakery.local.LocalDirectBakery"
+  },
+  "TargetStorage": {
+    "fsspec_class": "s3fs.S3FileSystem",
+    "fsspec_args": {},
+    "root_path": "./target"
+  },
+  "InputCacheStorage": {
+    "fsspec_class": "s3fs.S3FileSystem",
+    "fsspec_args": {},
+    "root_path": "./cache"
+  }
+}

pangeo_forge_recipes/openers.py

@@ -220,7 +220,6 @@ def open_with_xarray(
 
     kw = xarray_open_kwargs or {}
     kw = _set_engine(file_type, kw)
-
     if copy_to_local:
         if file_type in [FileType.zarr or FileType.opendap]:
             raise ValueError(f"File type {file_type} can't be copied to a local file.")

pangeo_forge_recipes/transforms.py

@@ -539,4 +539,5 @@ def expand(
         )
         # TODO: optionally use `singleton_target_store` to
         # consolidate metadata and/or coordinate dims here
+
         return singleton_target_store

pyproject.toml

@@ -40,17 +40,19 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-dev = [
+test = [
     "click",
     "pytest",
     "pytest-cov",
     "pytest-lazy-fixture",
     "pytest-sugar",
     "pytest-timeout",
     "s3fs",
-    "scipy"
+    "scipy",
+]
+minio = [
+    "docker",
 ]
-
 
 [project.urls]
 Homepage = "https://github.com/pangeo-forge/pangeo-forge-recipes"

tests/test_integration.py

@@ -1,5 +1,6 @@
 import json
 import os
+import secrets
 import subprocess
 import time
 from pathlib import Path
@@ -17,6 +18,32 @@
 EXAMPLES = Path(__file__).parent.parent / "examples"
 
 
+@pytest.fixture(scope="session")
+def minio():
+    import docker
+
+    client = docker.from_env()
+    port = 9000
+    username = secrets.token_hex(16)
+    password = secrets.token_hex(16)
+    minio_container = client.containers.run(
+        "minio/minio",
+        "server /data",
+        detach=True,
+        ports={f"{port}/tcp": port},
+        environment={
+            "MINIO_ACCESS_KEY": username,
+            "MINIO_SECRET_KEY": password,
+        },
+    )
+    time.sleep(10)  # give it time to boot
+    # enter
+    yield {"endpoint": f"http://localhost:{port}", "username": username, "password": password}
+    # exit
+    minio_container.stop()
+    minio_container.remove()
+
+
 def test_python_json_configs_identical():
     """We provide examples of both Python and JSON config. By ensuring they are
     identical, we can confidently use just one of them for the integration tests.
@@ -32,7 +59,7 @@ def test_python_json_configs_identical():
 
 
 @pytest.fixture
-def confpath(tmp_path_factory: pytest.TempPathFactory):
+def local_confpath(tmp_path_factory: pytest.TempPathFactory):
     """The JSON config is easier to modify with tempdirs, so we use that here for
     convenience. But we know it's the same as the Python config, because we test that.
     """
@@ -49,6 +76,29 @@ def confpath(tmp_path_factory: pytest.TempPathFactory):
     return dstpath.absolute().as_posix()
 
 
+@pytest.fixture
+def minio_confpath(minio, tmp_path_factory: pytest.TempPathFactory):
+    tmp = tmp_path_factory.mktemp("tmp")
+    fsspec_args = {
+        "key": minio["username"],
+        "secret": minio["password"],
+        "client_kwargs": {"endpoint_url": minio["endpoint"]},
+    }
+
+    fname = "s3.json"
+    dstpath = tmp / fname
+    with open(EXAMPLES / "runner-config" / fname) as src:
+        with dstpath.open(mode="w") as dst:
+            c = json.load(src)
+            c["TargetStorage"]["root_path"] = (tmp / "target").absolute().as_posix()
+            c["TargetStorage"]["fsspec_args"] = fsspec_args
+            c["InputCacheStorage"]["root_path"] = (tmp / "cache").absolute().as_posix()
+            c["InputCacheStorage"]["fsspec_args"] = fsspec_args
+
+            json.dump(c, dst)
+    return dstpath.absolute().as_posix()
+
+
 @pytest.mark.parametrize(
     "recipe_id",
     [
@@ -57,9 +107,10 @@ def confpath(tmp_path_factory: pytest.TempPathFactory):
         if p.suffix == ".py" and not p.stem.startswith("_")
     ],
 )
-def test_integration(recipe_id: str, confpath: str):
+@pytest.mark.parametrize("confpath_option", ["local_confpath", "minio_confpath"])
+def test_integration(confpath_option: str, recipe_id: str, request):
     """Run the example recipes in the ``examples/feedstock`` directory."""
-
+    # pytest tests/test_integration.py -k 'test_integration' --run-integration
     xfails = {
         "hrrr-kerchunk-concat-step": "WriteCombineReference doesn't return zarr.storage.FSStore",
         "hrrr-kerchunk-concat-valid-time": "Can't serialize drop_unknown callback function.",
@@ -69,6 +120,8 @@ def test_integration(recipe_id: str, confpath: str):
     if recipe_id in xfails:
         pytest.xfail(xfails[recipe_id])
 
+    confpath = request.getfixturevalue(confpath_option)
+
     bake_script = (EXAMPLES / "runner-commands" / "bake.sh").absolute().as_posix()
     cmd = ["sh", bake_script]
     env = os.environ.copy() | {
@@ -77,5 +130,8 @@ def test_integration(recipe_id: str, confpath: str):
         "RECIPE_ID": recipe_id,
         "JOB_NAME": f"{recipe_id}-{str(int(time.time()))}",
     }
-    proc = subprocess.run(cmd, capture_output=True, env=env)
-    assert proc.returncode == 0
+    proc = subprocess.run(cmd, capture_output=True, env=env, text=True)
+    if proc.returncode != 0:
+        print("Command failed!")
+        print("Stdout:", proc.stdout)
+        print("Stderr:", proc.stderr)