diff --git a/.dockerignore b/.dockerignore index b7925ae2..c3ad29ef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,3 @@ -tileserver \ No newline at end of file +.env + +tileserver diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..2b0f4871 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,33 @@ +name: CI + +on: + - push + - pull_request + + +jobs: + run-dagger-ci: + runs-on: ubuntu-22.04 + steps: + + - name: grab code + uses: actions/checkout@v4 + + - name: setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + cache-dependency-path: docker/backend/project_requirements.txt + + - name: install dagger for python + uses: insightsengineering/pip-action@v2 + with: + packages: dagger-io==0.9.8 + + - name: run dagger + uses: dagger/dagger-for-github@v5 + with: + verb: run + args: python tests/ci/main.py + version: 0.9.9 diff --git a/.gitignore b/.gitignore index 918eef64..4bf1b09d 100644 --- a/.gitignore +++ b/.gitignore @@ -138,4 +138,6 @@ backend/.env /docker/basemap/gebco/ /proxy/node_modules/ -/Arpav-PPCV \ No newline at end of file +/Arpav-PPCV + +.venv \ No newline at end of file diff --git a/backend/padoa/forecastattributes/views.py b/backend/padoa/forecastattributes/views.py index f293fb95..1174de97 100644 --- a/backend/padoa/forecastattributes/views.py +++ b/backend/padoa/forecastattributes/views.py @@ -9,7 +9,7 @@ from djcore.djcore.core.mixins import OptionListModelMixin class CachedParametersList(OptionListModelMixin, ListAPIView, GenericViewSet): - @method_decorator(cache_page(3600*7*24)) + # @method_decorator(cache_page(3600*7*24)) def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) diff --git a/backend/padoa/thredds/tests.py b/backend/padoa/thredds/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/backend/padoa/thredds/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/tests/ci/main.py b/tests/ci/main.py new file mode 100644 index 00000000..01a9c52f --- /dev/null +++ b/tests/ci/main.py @@ -0,0 +1,101 @@ +import asyncio +import os +import shlex +import sys +from pathlib import Path + +import dagger + +POSTGIS_IMAGE_VERSION = "postgis/postgis:16-3.4" + + +def get_env_variables() -> dict[str, str | None]: + return { + "API_COMMAND": os.getenv("API_COMMAND", "daphne"), + "DEBUG": os.getenv("DEBUG", "0"), + "PGPASSWORD": os.getenv("PGPASSWORD", "postgres"), + "POSTGRES_DB_NAME": os.getenv("POSTGRES_DB_NAME", "postgres"), + "POSTGRES_PORT_5432_TCP_ADDR": os.getenv("POSTGRES_PORT_5432_TCP_ADDR", "postgis"), + "POSTGRES_USER": os.getenv("POSTGRES_USER", "postgres"), + "REDIS_HOST": os.getenv("REDIS_HOST", "redis"), + "SECRET_KEY": os.getenv("SECRET_KEY", "generate it e.g. from https://djecrety.ir/"), + "SSL_CERTIFICATE": os.getenv("SSL_CERTIFICATE", "/etc/letsencrypt/live/yourdomain/fullchain.pem"), + "SSL_KEY": os.getenv("SSL_KEY", "/etc/letsencrypt/live/yourdomain/privkey.pem"), + "THREDDS_AUTH_URL": os.getenv("THREDDS_AUTH_URL", "https://thredds.arpa.veneto.it/thredds/restrictedAccess/dati_accordo"), + "THREDDS_HOST": os.getenv("THREDDS_HOST", "https://thredds.arpa.veneto.it/thredds/"), + "THREDDS_PASSWORD": os.getenv("THREDDS_PASSWORD", ""), + "THREDDS_USER": os.getenv("THREDDS_USER", ""), + } + + +async def build_and_test(): + env_variables = get_env_variables() + conf = dagger.Config( + log_output=sys.stderr, + ) + repo_root = Path(__file__).parents[2] + async with dagger.Connection(conf) as client: + postgis_service = ( + client.container() + .from_(POSTGIS_IMAGE_VERSION) + .with_env_variable("PGDATA", "/var/lib/postgresql/data/pgdata") + .with_env_variable("POSTGRES_DB", env_variables["POSTGRES_DB_NAME"]) + .with_env_variable("POSTGRES_PASSWORD", env_variables["PGPASSWORD"]) + .with_env_variable("POSTGRES_USER", env_variables["POSTGRES_USER"]) + .with_exposed_port(5432) + .as_service() + ) + + src = client.host().directory(str(repo_root)) + built_container = ( + client.container() + .build( + context=src, + dockerfile="Dockerfile" + ) + ) + + test_results = await ( + built_container.with_service_binding("db", postgis_service) + .with_mounted_directory("/opt/api/tests", client.host().directory("./tests")) + .with_env_variable("DEBUG", env_variables["DEBUG"]) + .with_env_variable("POSTGRES_DB_NAME", env_variables["POSTGRES_DB_NAME"]) + .with_env_variable("POSTGRES_USER", env_variables["POSTGRES_USER"]) + .with_env_variable("PGPASSWORD", env_variables["PGPASSWORD"]) + .with_env_variable("POSTGRES_PORT_5432_TCP_ADDR", "db") + .with_env_variable("REDIS_HOST", env_variables["REDIS_HOST"]) + .with_env_variable("SECRET_KEY", env_variables["SECRET_KEY"]) + .with_env_variable("THREDDS_HOST", env_variables["THREDDS_HOST"]) + .with_env_variable("THREDDS_PASSWORD", env_variables["THREDDS_PASSWORD"]) + .with_env_variable("THREDDS_USER", env_variables["THREDDS_USER"]) + .with_exec( + shlex.split( + "pip install " + "pytest==8.0.0 " + "pytest-django==4.8.0" + ), + skip_entrypoint=True + ) + .with_exec( + shlex.split( + "python manage.py makemigrations " + "users " + "groups " + "forecastattributes " + "places " + "thredds" + ), + skip_entrypoint=True + ) + .with_exec(shlex.split("python manage.py migrate"), skip_entrypoint=True) + .with_exec( + shlex.split("pytest --verbose -x --reuse-db ../tests/test_padoa_forecastattributes_views.py"), + skip_entrypoint=True + ) + ).stdout() + print("Done") + print(f"{test_results=}") + + +if __name__ == "__main__": + asyncio.run(build_and_test()) diff --git a/tests/test_padoa_forecastattributes_views.py b/tests/test_padoa_forecastattributes_views.py new file mode 100644 index 00000000..acc7c6e3 --- /dev/null +++ b/tests/test_padoa_forecastattributes_views.py @@ -0,0 +1,115 @@ +import pytest + +from padoa.forecastattributes import models + + +@pytest.mark.django_db +@pytest.mark.parametrize("model_class, list_url_path", [ + pytest.param(models.Variable, "/forcastattributes/variables/"), + pytest.param(models.ForecastModel, "/forcastattributes/forecast_models/"), + pytest.param(models.Scenario, "/forcastattributes/scenarios/"), + pytest.param(models.DataSeries, "/forcastattributes/data_series/"), + pytest.param(models.YearPeriod, "/forcastattributes/year_periods/"), + pytest.param(models.TimeWindow, "/forcastattributes/time_windows/"), + pytest.param(models.ValueType, "/forcastattributes/value_types/"), +]) +def test_list_instances(client, model_class, list_url_path): + num_items = 3 + for i in range(1, num_items + 1): + model_class.objects.create( + id=f"fake{model_class.__name__.lower()}{i}", + name=f"Fake {model_class.__name__} {i}", + description=f"This is a fake {model_class.__name__}, useful for testing", + order_item=i + ) + response = client.get(list_url_path) + assert response.status_code == 200 + contents = response.json() + assert contents.get("count") == num_items + assert len(contents.get("results")) == num_items + for i in range(1, num_items + 1): + print(f"looking for scenario #{i}...") + for model_ in contents["results"]: + if model_.get("id") == f"fake{model_class.__name__.lower()}{i}": + assert model_["name"] == f"Fake {model_class.__name__} {i}" + assert model_["description"] == ( + f"This is a fake {model_class.__name__}, useful for testing") + assert model_["order_item"] == i + break + else: + print(f"Did not find instance #{i}") + assert False + + +# these are disabled because changing django-rest-framework page size with +# pytest_django's `settings` fixture does not seem to work. +# @pytest.mark.django_db +# @pytest.mark.parametrize("model_class, list_url_path", [ +# pytest.param(models.Variable, "/forcastattributes/variables/"), +# pytest.param(models.ForecastModel, "/forcastattributes/forecast_models/"), +# pytest.param(models.Scenario, "/forcastattributes/scenarios/"), +# pytest.param(models.DataSeries, "/forcastattributes/data_series/"), +# pytest.param(models.YearPeriod, "/forcastattributes/year_periods/"), +# pytest.param(models.TimeWindow, "/forcastattributes/time_windows/"), +# pytest.param(models.ValueType, "/forcastattributes/value_types/"), +# ]) +# def test_list_instances_with_pagination_has_next( +# client, +# settings, +# model_class, +# list_url_path +# ): +# page_size = 5 +# num_items = page_size + 3 +# settings.REST_FRAMEWORK["PAGE_SIZE"] = page_size +# for i in range(1, num_items + 1): +# model_class.objects.create( +# id=f"fake{model_class.__name__.lower()}{i}", +# name=f"Fake {model_class.__name__} {i}", +# description=f"This is a fake {model_class.__name__}, useful for testing", +# order_item=i +# ) +# response = client.get(list_url_path) +# assert response.status_code == 200 +# contents = response.json() +# print(f"{contents=}") +# assert contents.get("count") == num_items +# assert len(contents.get("results")) == page_size +# assert contents.get("next") is not None +# assert contents.get("previous") is None +# +# +# @pytest.mark.django_db +# @pytest.mark.parametrize("model_class, list_url_path", [ +# pytest.param(models.Variable, "/forcastattributes/variables/"), +# pytest.param(models.ForecastModel, "/forcastattributes/forecast_models/"), +# pytest.param(models.Scenario, "/forcastattributes/scenarios/"), +# pytest.param(models.DataSeries, "/forcastattributes/data_series/"), +# pytest.param(models.YearPeriod, "/forcastattributes/year_periods/"), +# pytest.param(models.TimeWindow, "/forcastattributes/time_windows/"), +# pytest.param(models.ValueType, "/forcastattributes/value_types/"), +# ]) +# def test_list_instances_with_pagination_has_previous( +# client, +# settings, +# model_class, +# list_url_path +# ): +# page_size = 5 +# num_items = page_size + 3 +# settings.REST_FRAMEWORK["PAGE_SIZE"] = page_size +# for i in range(1, num_items + 1): +# model_class.objects.create( +# id=f"fake{model_class.__name__.lower()}{i}", +# name=f"Fake {model_class.__name__} {i}", +# description=f"This is a fake {model_class.__name__}, useful for testing", +# order_item=i +# ) +# response = client.get(list_url_path, data={"page": 2}) +# assert response.status_code == 200 +# contents = response.json() +# print(f"contents: {contents}") +# assert contents.get("count") == num_items +# assert len(contents.get("results")) == num_items - page_size +# assert contents.get("next") is None +# assert contents.get("previous") is not None diff --git a/tests/test_padoa_thredds_views.py b/tests/test_padoa_thredds_views.py new file mode 100644 index 00000000..34b26bca --- /dev/null +++ b/tests/test_padoa_thredds_views.py @@ -0,0 +1,46 @@ +import pytest +from django.contrib.gis.geos import GEOSGeometry + +from padoa.thredds import models as tm +from padoa.forecastattributes import models as fam + + +@pytest.mark.django_db +@pytest.mark.parametrize("url_list_path", [ + pytest.param("/maps/ncss/timeserie/"), +]) +def test_ncsstimeserie_list(client, url_list_path): + num_items = 3 + variable = fam.Variable.objects.create(id="fakevar1", name="Fake var 1") + forecast_model = fam.ForecastModel.objects.create(id="fakeforecastmodel1", name="Fake forecast model 1") + scenario = fam.Scenario.objects.create(id="fakescenario1", name="Fake scenario 1") + data_series = fam.DataSeries.objects.create(id="fakedataseries1", name="Fake data series 1") + year_period = fam.YearPeriod.objects.create(id="fakeyearperiod1", name="Fake year period 1") + value_type = fam.ValueType.objects.create(id="fakevaluetype1", name="Fake value type 1") + + created_ids = [] + for i in range(1, num_items + 1): + # regardless of the model definition: + # - spatialbounds must be provided, otherwise serialization fails + # - palette must be provided, otherwise serialization fails + instance = tm.Map.objects.create( + variable=variable, + forecast_model=forecast_model, + scenario=scenario, + data_series=data_series, + year_period=year_period, + time_window=None, + value_type=value_type, + layer_id=f"fakelayerid{i}", + path="/some/fake/path", + spatialbounds=GEOSGeometry("POLYGON(( 10 10, 10 20, 20 20, 20 15, 10 10))"), + palette="fakepalette", + ) + created_ids.append(instance.id) + response = client.get( + url_list_path, + data={"ids": ",".join(str(i) for i in created_ids)} + ) + print(f"{response.json()=}") + assert response.status_code == 200 + assert False