From 07350f5c437dce3d795e01474049203773e09f9c Mon Sep 17 00:00:00 2001 From: Vincent Sarago Date: Mon, 22 Apr 2024 22:14:32 +0200 Subject: [PATCH] refactor tests suite in order to run it locally outside docker (#102) * refactor tests suite in order to run it locally outside docker * lint * remove service * try pg install * use python 3.9 for pre-commit to avoid deprecation issue form pydocstyle * revert --------- Co-authored-by: Jonathan Healy --- .github/workflows/cicd.yaml | 61 +++++++++-------------- .pre-commit-config.yaml | 13 +++-- Dockerfile | 8 +-- Dockerfile.tests | 19 ++++++++ Makefile | 4 +- docker-compose.yml | 14 +++++- setup.py | 1 + tests/api/test_api.py | 13 ++++- tests/conftest.py | 97 ++++++++++++++++--------------------- 9 files changed, 124 insertions(+), 106 deletions(-) create mode 100644 Dockerfile.tests diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 864d96e..b363efa 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -5,38 +5,15 @@ on: pull_request: branches: [main] + jobs: test: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] timeout-minutes: 20 - services: - db_service: - image: ghcr.io/stac-utils/pgstac:v0.7.10 - env: - POSTGRES_USER: username - POSTGRES_PASSWORD: password - POSTGRES_DB: postgis - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 - PGUSER: username - PGPASSWORD: password - PGDATABASE: postgis - ALLOW_IP_RANGE: 0.0.0.0/0 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 10s - --health-retries 10 - --log-driver none - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 - steps: - name: Check out repository code uses: actions/checkout@v4 @@ -55,20 +32,16 @@ jobs: python -m pip install pre-commit pre-commit run --all-files - - name: Install + - name: install lib postgres + uses: nyurik/action-setup-postgis@v1 + + - name: Install dependencies run: | - pip install .[dev,server] + python -m pip install --upgrade pip + python -m pip install .[dev,server] - name: Run test suite - run: make test - env: - ENVIRONMENT: testing - POSTGRES_USER: username - POSTGRES_PASS: password - POSTGRES_DBNAME: postgis - POSTGRES_HOST_READER: localhost - POSTGRES_HOST_WRITER: localhost - POSTGRES_PORT: 5432 + run: python -m pytest --cov stac_fastapi.pgstac --cov-report xml --cov-report term-missing validate: runs-on: ubuntu-latest @@ -90,17 +63,23 @@ jobs: --log-driver none ports: - 5432:5432 + steps: - name: Check out repository code uses: actions/checkout@v4 + - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.10" cache: pip cache-dependency-path: setup.py + - name: Install stac-fastapi and stac-api-validator - run: pip install .[server] stac-api-validator==0.4.1 + run: | + python -m pip install --upgrade pip + python -m pip install .[server] stac-api-validator==0.4.1 + - name: Load data and validate run: python -m stac_fastapi.pgstac.app & ./scripts/wait-for-it.sh localhost:8080 && python ./scripts/ingest_joplin.py http://localhost:8080 && ./scripts/validate http://localhost:8080 env: @@ -120,15 +99,21 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.10" cache: pip cache-dependency-path: setup.py + - name: Install with documentation dependencies - run: pip install .[docs,dev,server] + run: | + python -m pip install --upgrade pip + python -m pip install .[docs,dev,server] + - name: Generate API docs run: make docs + - name: Build documentation run: mkdocs build --strict diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e92bd6d..49f90ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,18 +3,20 @@ repos: rev: 5.12.0 hooks: - id: isort - language_version: python3.8 + language_version: python + - repo: https://github.com/psf/black rev: 22.12.0 hooks: - id: black args: ["--safe"] - language_version: python3.8 + language_version: python + - repo: https://github.com/pycqa/flake8 rev: 6.0.0 hooks: - id: flake8 - language_version: python3.8 + language_version: python args: [ # E501 let black handle all line length decisions # W503 black conflicts with "line break before operator" rule @@ -26,7 +28,7 @@ repos: rev: v2.1.1 hooks: - id: pydocstyle - language_version: python3.8 + language_version: python exclude: ".*(test|scripts).*" args: [ @@ -42,11 +44,12 @@ repos: # - id: mypy # language_version: python3.8 # args: [--no-strict-optional, --ignore-missing-imports] + - repo: https://github.com/PyCQA/pydocstyle rev: 6.3.0 hooks: - id: pydocstyle - language_version: python3.8 + language_version: python exclude: ".*(test|scripts).*" #args: [ # Don't require docstrings for tests diff --git a/Dockerfile b/Dockerfile index aeb12c7..5015019 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.8-slim as base +ARG PYTHON_VERSION=3.11 + +FROM python:${PYTHON_VERSION}-slim as base # Any python libraries that require system libraries to be installed will likely # need the following packages in order to build @@ -16,6 +18,6 @@ WORKDIR /app COPY . /app -RUN pip install -e .[dev,server] +RUN python -m pip install -e .[server] -CMD ["uvicorn", "stac_fastapi.pgstac.app:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file +CMD ["uvicorn", "stac_fastapi.pgstac.app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Dockerfile.tests b/Dockerfile.tests new file mode 100644 index 0000000..4af0dd7 --- /dev/null +++ b/Dockerfile.tests @@ -0,0 +1,19 @@ +ARG PYTHON_VERSION=3.11 + +FROM python:${PYTHON_VERSION}-slim as base + +# Any python libraries that require system libraries to be installed will likely +# need the following packages in order to build +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get install -y build-essential git libpq-dev postgresql-15-postgis-3 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN useradd -ms /bin/bash newuser +USER newuser + +WORKDIR /app +COPY . /app + +RUN python -m pip install -e .[dev,server] --user diff --git a/Makefile b/Makefile index f4a62bd..f816f13 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,8 @@ run = docker-compose run --rm \ -e APP_PORT=${APP_PORT} \ app +runtests = docker-compose run --rm tests + .PHONY: image image: docker-compose build @@ -28,7 +30,7 @@ docker-shell: .PHONY: test test: - $(run) /bin/bash -c 'export && ./scripts/wait-for-it.sh database:5432 && cd /app/tests/ && pytest -vvv --log-cli-level $(LOG_LEVEL)' + $(runtests) /bin/bash -c 'export && python -m pytest /app/tests/api/test_api.py --log-cli-level $(LOG_LEVEL)' .PHONY: run-database run-database: diff --git a/docker-compose.yml b/docker-compose.yml index a2c0709..bf79b65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,6 @@ services: app: container_name: stac-fastapi-pgstac image: stac-utils/stac-fastapi-pgstac - platform: linux/amd64 build: . environment: - APP_HOST=0.0.0.0 @@ -31,6 +30,19 @@ services: - database command: bash -c "./scripts/wait-for-it.sh database:5432 && python -m stac_fastapi.pgstac.app" + tests: + container_name: stac-fastapi-pgstac-test + image: stac-utils/stac-fastapi-pgstac-test + build: + context: . + dockerfile: Dockerfile.tests + environment: + - ENVIRONMENT=local + - DB_MIN_CONN_SIZE=1 + - DB_MAX_CONN_SIZE=1 + - USE_API_HYDRATE=${USE_API_HYDRATE:-false} + command: bash -c "python -m pytest -s -vv" + database: container_name: stac-db image: ghcr.io/stac-utils/pgstac:v0.7.10 diff --git a/setup.py b/setup.py index 568ee5b..f83fa4f 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ "dev": [ "pystac[validation]", "pypgstac[psycopg]==0.7.*", + "pytest-postgresql", "pytest", "pytest-cov", "pytest-asyncio>=0.17,<0.23.0", diff --git a/tests/api/test_api.py b/tests/api/test_api.py index 02f9505..5dcc95d 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -636,7 +636,7 @@ async def search(query: Dict[str, Any]) -> List[Item]: @pytest.mark.asyncio -async def test_wrapped_function(load_test_data) -> None: +async def test_wrapped_function(load_test_data, database) -> None: # Ensure wrappers, e.g. Planetary Computer's rate limiting, work. # https://github.com/gadomski/planetary-computer-apis/blob/2719ccf6ead3e06de0784c39a2918d4d1811368b/pccommon/pccommon/redis.py#L205-L238 @@ -672,7 +672,16 @@ async def get_collection( collection_id, request=request, **kwargs ) - settings = Settings(testing=True) + settings = Settings( + postgres_user=database.user, + postgres_pass=database.password, + postgres_host_reader=database.host, + postgres_host_writer=database.host, + postgres_port=database.port, + postgres_dbname=database.dbname, + testing=True, + ) + extensions = [ TransactionExtension(client=TransactionsClient(), settings=settings), FieldsExtension(), diff --git a/tests/conftest.py b/tests/conftest.py index 86d9909..b8786c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ from httpx import AsyncClient from pypgstac.db import PgstacDB from pypgstac.migrate import Migrate +from pytest_postgresql.janitor import DatabaseJanitor from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model from stac_fastapi.extensions.core import ( @@ -36,8 +37,6 @@ DATA_DIR = os.path.join(os.path.dirname(__file__), "data") -settings = Settings(testing=True) -pgstac_api_hydrate_settings = Settings(testing=True, use_api_hydrate=True) logger = logging.getLogger(__name__) @@ -48,82 +47,68 @@ def event_loop(): @pytest.fixture(scope="session") -async def pg(): - logger.info(f"Connecting to write database {settings.writer_connection_string}") - os.environ["orig_postgres_dbname"] = settings.postgres_dbname - conn = await asyncpg.connect(dsn=settings.writer_connection_string) - try: - await conn.execute("CREATE DATABASE pgstactestdb;") - await conn.execute( - """ - ALTER DATABASE pgstactestdb SET search_path to pgstac, public; - ALTER DATABASE pgstactestdb SET log_statement to 'all'; - """ +def database(postgresql_proc): + with DatabaseJanitor( + user=postgresql_proc.user, + host=postgresql_proc.host, + port=postgresql_proc.port, + dbname="pgstactestdb", + version=postgresql_proc.version, + password="secret", + ) as jan: + connection = ( + f"postgresql://{jan.user}:{jan.password}@{jan.host}:{jan.port}/{jan.dbname}" ) - except asyncpg.exceptions.DuplicateDatabaseError: - await conn.execute("DROP DATABASE pgstactestdb;") - await conn.execute("CREATE DATABASE pgstactestdb;") - await conn.execute( - "ALTER DATABASE pgstactestdb SET search_path to pgstac, public;" - ) - await conn.close() - logger.info("migrating...") - os.environ["postgres_dbname"] = "pgstactestdb" - conn = await asyncpg.connect(dsn=settings.testing_connection_string) - await conn.execute("SELECT true") - await conn.close() - db = PgstacDB(dsn=settings.testing_connection_string) - migrator = Migrate(db) - version = migrator.run_migration() - db.close() - logger.info(f"PGStac Migrated to {version}") + with PgstacDB(dsn=connection) as db: + migrator = Migrate(db) + version = migrator.run_migration() + assert version - yield settings.testing_connection_string - - print("Getting rid of test database") - os.environ["postgres_dbname"] = os.environ["orig_postgres_dbname"] - conn = await asyncpg.connect(dsn=settings.writer_connection_string) - try: - await conn.execute("DROP DATABASE pgstactestdb;") - await conn.close() - except Exception: - try: - await conn.execute("DROP DATABASE pgstactestdb WITH (force);") - await conn.close() - except Exception: - pass + yield jan @pytest.fixture(autouse=True) -async def pgstac(pg): - logger.info(f"{os.environ['postgres_dbname']}") +async def pgstac(database): + connection = f"postgresql://{database.user}:{database.password}@{database.host}:{database.port}/{database.dbname}" yield - logger.info("Truncating Data") - conn = await asyncpg.connect(dsn=settings.testing_connection_string) + conn = await asyncpg.connect(dsn=connection) await conn.execute( """ DROP SCHEMA IF EXISTS pgstac CASCADE; """ ) await conn.close() - with PgstacDB(dsn=settings.testing_connection_string) as db: + with PgstacDB(dsn=connection) as db: migrator = Migrate(db) version = migrator.run_migration() + logger.info(f"PGStac Migrated to {version}") # Run all the tests that use the api_client in both db hydrate and api hydrate mode @pytest.fixture( params=[ - (settings, ""), - (settings, "/router_prefix"), - (pgstac_api_hydrate_settings, ""), - (pgstac_api_hydrate_settings, "/router_prefix"), + # hydratation, prefix + (False, ""), + (False, "/router_prefix"), + (True, ""), + (True, "/router_prefix"), ], scope="session", ) -def api_client(request, pg): - api_settings, prefix = request.param +def api_client(request, database): + hydrate, prefix = request.param + + api_settings = Settings( + postgres_user=database.user, + postgres_pass=database.password, + postgres_host_reader=database.host, + postgres_host_writer=database.host, + postgres_port=database.port, + postgres_dbname=database.dbname, + use_api_hydrate=hydrate, + testing=True, + ) api_settings.openapi_url = prefix + api_settings.openapi_url api_settings.docs_url = prefix + api_settings.docs_url @@ -135,7 +120,7 @@ def api_client(request, pg): ) extensions = [ - TransactionExtension(client=TransactionsClient(), settings=settings), + TransactionExtension(client=TransactionsClient(), settings=api_settings), QueryExtension(), SortExtension(), FieldsExtension(),