Merge pull request #6 from ricardogsilva/1-setup-ci-pipeline

Setup CI pipeline

.dockerignore

@@ -1 +1,3 @@
-tileserver
+.env
+
+tileserver

.github/workflows/ci.yaml

@@ -0,0 +1,33 @@
+name: CI
+
+on:
+  - push
+  - pull_request
+
+
+jobs:
+  run-dagger-ci:
+    runs-on: ubuntu-22.04
+    steps:
+
+      - name: grab code
+        uses: actions/checkout@v4
+
+      - name: setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: pip
+          cache-dependency-path: docker/backend/project_requirements.txt
+
+      - name: install dagger for python
+        uses: insightsengineering/pip-action@v2
+        with:
+          packages: dagger-io==0.9.8
+
+      - name: run dagger
+        uses: dagger/dagger-for-github@v5
+        with:
+          verb: run
+          args: python tests/ci/main.py
+          version: 0.9.9

.gitignore

@@ -138,4 +138,6 @@ backend/.env
 /docker/basemap/gebco/
 /proxy/node_modules/
 
-/Arpav-PPCV
+/Arpav-PPCV
+
+.venv

backend/padoa/forecastattributes/views.py

@@ -9,7 +9,7 @@
 from djcore.djcore.core.mixins import OptionListModelMixin
 
 class CachedParametersList(OptionListModelMixin, ListAPIView, GenericViewSet):
-    @method_decorator(cache_page(3600*7*24))
+    # @method_decorator(cache_page(3600*7*24))
     def list(self, request, *args, **kwargs):
         return super().list(request, *args, **kwargs)
 

tests/ci/main.py

@@ -0,0 +1,101 @@
+import asyncio
+import os
+import shlex
+import sys
+from pathlib import Path
+
+import dagger
+
+POSTGIS_IMAGE_VERSION = "postgis/postgis:16-3.4"
+
+
+def get_env_variables() -> dict[str, str | None]:
+    return {
+        "API_COMMAND": os.getenv("API_COMMAND", "daphne"),
+        "DEBUG": os.getenv("DEBUG", "0"),
+        "PGPASSWORD": os.getenv("PGPASSWORD", "postgres"),
+        "POSTGRES_DB_NAME": os.getenv("POSTGRES_DB_NAME", "postgres"),
+        "POSTGRES_PORT_5432_TCP_ADDR": os.getenv("POSTGRES_PORT_5432_TCP_ADDR", "postgis"),
+        "POSTGRES_USER": os.getenv("POSTGRES_USER", "postgres"),
+        "REDIS_HOST": os.getenv("REDIS_HOST", "redis"),
+        "SECRET_KEY": os.getenv("SECRET_KEY", "generate it e.g. from https://djecrety.ir/"),
+        "SSL_CERTIFICATE": os.getenv("SSL_CERTIFICATE", "/etc/letsencrypt/live/yourdomain/fullchain.pem"),
+        "SSL_KEY": os.getenv("SSL_KEY", "/etc/letsencrypt/live/yourdomain/privkey.pem"),
+        "THREDDS_AUTH_URL": os.getenv("THREDDS_AUTH_URL", "https://thredds.arpa.veneto.it/thredds/restrictedAccess/dati_accordo"),
+        "THREDDS_HOST": os.getenv("THREDDS_HOST", "https://thredds.arpa.veneto.it/thredds/"),
+        "THREDDS_PASSWORD": os.getenv("THREDDS_PASSWORD", ""),
+        "THREDDS_USER": os.getenv("THREDDS_USER", ""),
+    }
+
+
+async def build_and_test():
+    env_variables = get_env_variables()
+    conf = dagger.Config(
+        log_output=sys.stderr,
+    )
+    repo_root = Path(__file__).parents[2]
+    async with dagger.Connection(conf) as client:
+        postgis_service = (
+            client.container()
+            .from_(POSTGIS_IMAGE_VERSION)
+            .with_env_variable("PGDATA", "/var/lib/postgresql/data/pgdata")
+            .with_env_variable("POSTGRES_DB", env_variables["POSTGRES_DB_NAME"])
+            .with_env_variable("POSTGRES_PASSWORD", env_variables["PGPASSWORD"])
+            .with_env_variable("POSTGRES_USER", env_variables["POSTGRES_USER"])
+            .with_exposed_port(5432)
+            .as_service()
+        )
+
+        src = client.host().directory(str(repo_root))
+        built_container = (
+            client.container()
+            .build(
+                context=src,
+                dockerfile="Dockerfile"
+            )
+        )
+
+        test_results = await (
+            built_container.with_service_binding("db", postgis_service)
+            .with_mounted_directory("/opt/api/tests", client.host().directory("./tests"))
+            .with_env_variable("DEBUG", env_variables["DEBUG"])
+            .with_env_variable("POSTGRES_DB_NAME", env_variables["POSTGRES_DB_NAME"])
+            .with_env_variable("POSTGRES_USER", env_variables["POSTGRES_USER"])
+            .with_env_variable("PGPASSWORD", env_variables["PGPASSWORD"])
+            .with_env_variable("POSTGRES_PORT_5432_TCP_ADDR", "db")
+            .with_env_variable("REDIS_HOST", env_variables["REDIS_HOST"])
+            .with_env_variable("SECRET_KEY", env_variables["SECRET_KEY"])
+            .with_env_variable("THREDDS_HOST", env_variables["THREDDS_HOST"])
+            .with_env_variable("THREDDS_PASSWORD", env_variables["THREDDS_PASSWORD"])
+            .with_env_variable("THREDDS_USER", env_variables["THREDDS_USER"])
+            .with_exec(
+                shlex.split(
+                    "pip install "
+                    "pytest==8.0.0 "
+                    "pytest-django==4.8.0"
+                ),
+                skip_entrypoint=True
+            )
+            .with_exec(
+                shlex.split(
+                    "python manage.py makemigrations "
+                    "users "
+                    "groups "
+                    "forecastattributes "
+                    "places "
+                    "thredds"
+                ),
+                skip_entrypoint=True
+            )
+            .with_exec(shlex.split("python manage.py migrate"), skip_entrypoint=True)
+            .with_exec(
+                shlex.split("pytest --verbose -x --reuse-db ../tests/test_padoa_forecastattributes_views.py"),
+                skip_entrypoint=True
+            )
+        ).stdout()
+        print("Done")
+        print(f"{test_results=}")
+
+
+if __name__ == "__main__":
+    asyncio.run(build_and_test())

tests/test_padoa_forecastattributes_views.py

@@ -0,0 +1,115 @@
+import pytest
+
+from padoa.forecastattributes import models
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("model_class, list_url_path", [
+    pytest.param(models.Variable, "/forcastattributes/variables/"),
+    pytest.param(models.ForecastModel, "/forcastattributes/forecast_models/"),
+    pytest.param(models.Scenario, "/forcastattributes/scenarios/"),
+    pytest.param(models.DataSeries, "/forcastattributes/data_series/"),
+    pytest.param(models.YearPeriod, "/forcastattributes/year_periods/"),
+    pytest.param(models.TimeWindow, "/forcastattributes/time_windows/"),
+    pytest.param(models.ValueType, "/forcastattributes/value_types/"),
+])
+def test_list_instances(client, model_class, list_url_path):
+    num_items = 3
+    for i in range(1, num_items + 1):
+        model_class.objects.create(
+            id=f"fake{model_class.__name__.lower()}{i}",
+            name=f"Fake {model_class.__name__} {i}",
+            description=f"This is a fake {model_class.__name__}, useful for testing",
+            order_item=i
+        )
+    response = client.get(list_url_path)
+    assert response.status_code == 200
+    contents = response.json()
+    assert contents.get("count") == num_items
+    assert len(contents.get("results")) == num_items
+    for i in range(1, num_items + 1):
+        print(f"looking for scenario #{i}...")
+        for model_ in contents["results"]:
+            if model_.get("id") == f"fake{model_class.__name__.lower()}{i}":
+                assert model_["name"] == f"Fake {model_class.__name__} {i}"
+                assert model_["description"] == (
+                    f"This is a fake {model_class.__name__}, useful for testing")
+                assert model_["order_item"] == i
+                break
+        else:
+            print(f"Did not find instance #{i}")
+            assert False
+
+
+# these are disabled because changing django-rest-framework page size with
+# pytest_django's `settings` fixture does not seem to work.
+# @pytest.mark.django_db
+# @pytest.mark.parametrize("model_class, list_url_path", [
+#     pytest.param(models.Variable, "/forcastattributes/variables/"),
+#     pytest.param(models.ForecastModel, "/forcastattributes/forecast_models/"),
+#     pytest.param(models.Scenario, "/forcastattributes/scenarios/"),
+#     pytest.param(models.DataSeries, "/forcastattributes/data_series/"),
+#     pytest.param(models.YearPeriod, "/forcastattributes/year_periods/"),
+#     pytest.param(models.TimeWindow, "/forcastattributes/time_windows/"),
+#     pytest.param(models.ValueType, "/forcastattributes/value_types/"),
+# ])
+# def test_list_instances_with_pagination_has_next(
+#         client,
+#         settings,
+#         model_class,
+#         list_url_path
+# ):
+#     page_size = 5
+#     num_items = page_size + 3
+#     settings.REST_FRAMEWORK["PAGE_SIZE"] = page_size
+#     for i in range(1, num_items + 1):
+#         model_class.objects.create(
+#             id=f"fake{model_class.__name__.lower()}{i}",
+#             name=f"Fake {model_class.__name__} {i}",
+#             description=f"This is a fake {model_class.__name__}, useful for testing",
+#             order_item=i
+#         )
+#     response = client.get(list_url_path)
+#     assert response.status_code == 200
+#     contents = response.json()
+#     print(f"{contents=}")
+#     assert contents.get("count") == num_items
+#     assert len(contents.get("results")) == page_size
+#     assert contents.get("next") is not None
+#     assert contents.get("previous") is None
+#
+#
+# @pytest.mark.django_db
+# @pytest.mark.parametrize("model_class, list_url_path", [
+#     pytest.param(models.Variable, "/forcastattributes/variables/"),
+#     pytest.param(models.ForecastModel, "/forcastattributes/forecast_models/"),
+#     pytest.param(models.Scenario, "/forcastattributes/scenarios/"),
+#     pytest.param(models.DataSeries, "/forcastattributes/data_series/"),
+#     pytest.param(models.YearPeriod, "/forcastattributes/year_periods/"),
+#     pytest.param(models.TimeWindow, "/forcastattributes/time_windows/"),
+#     pytest.param(models.ValueType, "/forcastattributes/value_types/"),
+# ])
+# def test_list_instances_with_pagination_has_previous(
+#         client,
+#         settings,
+#         model_class,
+#         list_url_path
+# ):
+#     page_size = 5
+#     num_items = page_size + 3
+#     settings.REST_FRAMEWORK["PAGE_SIZE"] = page_size
+#     for i in range(1, num_items + 1):
+#         model_class.objects.create(
+#             id=f"fake{model_class.__name__.lower()}{i}",
+#             name=f"Fake {model_class.__name__} {i}",
+#             description=f"This is a fake {model_class.__name__}, useful for testing",
+#             order_item=i
+#         )
+#     response = client.get(list_url_path, data={"page": 2})
+#     assert response.status_code == 200
+#     contents = response.json()
+#     print(f"contents: {contents}")
+#     assert contents.get("count") == num_items
+#     assert len(contents.get("results")) == num_items - page_size
+#     assert contents.get("next") is None
+#     assert contents.get("previous") is not None

tests/test_padoa_thredds_views.py

@@ -0,0 +1,46 @@
+import pytest
+from django.contrib.gis.geos import GEOSGeometry
+
+from padoa.thredds import models as tm
+from padoa.forecastattributes import models as fam
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("url_list_path", [
+    pytest.param("/maps/ncss/timeserie/"),
+])
+def test_ncsstimeserie_list(client, url_list_path):
+    num_items = 3
+    variable = fam.Variable.objects.create(id="fakevar1", name="Fake var 1")
+    forecast_model = fam.ForecastModel.objects.create(id="fakeforecastmodel1", name="Fake forecast model 1")
+    scenario = fam.Scenario.objects.create(id="fakescenario1", name="Fake scenario 1")
+    data_series = fam.DataSeries.objects.create(id="fakedataseries1", name="Fake data series 1")
+    year_period = fam.YearPeriod.objects.create(id="fakeyearperiod1", name="Fake year period 1")
+    value_type = fam.ValueType.objects.create(id="fakevaluetype1", name="Fake value type 1")
+
+    created_ids = []
+    for i in range(1, num_items + 1):
+        # regardless of the model definition:
+        # - spatialbounds must be provided, otherwise serialization fails
+        # - palette must be provided, otherwise serialization fails
+        instance = tm.Map.objects.create(
+            variable=variable,
+            forecast_model=forecast_model,
+            scenario=scenario,
+            data_series=data_series,
+            year_period=year_period,
+            time_window=None,
+            value_type=value_type,
+            layer_id=f"fakelayerid{i}",
+            path="/some/fake/path",
+            spatialbounds=GEOSGeometry("POLYGON(( 10 10, 10 20, 20 20, 20 15, 10 10))"),
+            palette="fakepalette",
+        )
+        created_ids.append(instance.id)
+    response = client.get(
+        url_list_path,
+        data={"ids": ",".join(str(i) for i in created_ids)}
+    )
+    print(f"{response.json()=}")
+    assert response.status_code == 200
+    assert False