From c55b6c764b209bdcde0812eeba0fea021359abfa Mon Sep 17 00:00:00 2001 From: David Marteau Date: Wed, 23 Mar 2022 16:25:56 +0100 Subject: [PATCH] Sanitize tests run --- .gitignore | 6 +++++ Makefile | 47 ++++++++++++++++++++++++++++++++++++++-- setup.cfg | 15 +++++++++---- tests/Makefile | 40 +++++++++++++++++++++++++--------- tests/conftest.py | 40 +++++++++++++++++----------------- tests/docker-compose.yml | 23 ++++++++++++++++++++ tests/pytest.ini | 2 ++ tests/run-tests.sh | 11 +++------- 8 files changed, 140 insertions(+), 44 deletions(-) create mode 100644 tests/docker-compose.yml diff --git a/.gitignore b/.gitignore index 7d5a261..861e02e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ wfsOutputExtension.zip .venv/ venv/ +.env + # Packages *.egg *.egg-info @@ -55,3 +57,7 @@ tests/__output__ tests/qgis_server.log .idea/ wfsoutputextension.*.zip + +/tests/data/local + +__output__ diff --git a/Makefile b/Makefile index b26711c..86c1b19 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,47 @@ -.PHONY: tests +SHELL = bash +.ONESHELL: +.PHONY: env -test: +# +# plugin makefile +# + +COMMITID=$(shell git rev-parse --short HEAD) + +PYTHON:=python3 + +BUILDDIR:=build +DIST:=${BUILDDIR}/dist + +manifest: + +dirs: + mkdir -p $(DIST) + +# Build dependencies +wheel-deps: dirs + pip wheel -w $(DIST) -r requirements.txt + +wheel: + mkdir -p $(DIST) + $(PYTHON) setup.py bdist_wheel --dist-dir=$(DIST) + +deliver: + twine upload -r storage $(DIST)/* + +dist: dirs + rm -rf *.egg-info + $(PYTHON) setup.py sdist --dist-dir=$(DIST) + +clean: + rm -rf *.egg-info + rm -rf $(BUILDDIR) + +# Checke setup.cfg for flake8 configuration +lint: + @flake8 + +test: lint $(MAKE) -C tests + + diff --git a/setup.cfg b/setup.cfg index 7cafe4d..d31053e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,13 +4,20 @@ github_organization_slug = 3liz project_slug = qgis-wfsOutputExtension [flake8] -max-line-length = 110 +max-line-length = 120 + +ignore = E123,E2,E3,E5,W2,W3 exclude = - .venv/, - venv/, - tests/.local/, + .git, + .github, + .venv, + venv, + __pycache__, tests/conftest.py, + .local, + .cache, + [isort] multi_line_output = 3 diff --git a/tests/Makefile b/tests/Makefile index 99b7f28..b8ae4ed 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,30 +1,50 @@ SHELL:=bash +.ONESHELL: +.PHONY: env + COMMITID=$(shell git rev-parse --short HEAD) -ifdef REGISTRY_URL - REGISTRY_PREFIX=$(REGISTRY_URL)/ -endif +REGISTRY_URL ?= 3liz +REGISTRY_PREFIX=$(REGISTRY_URL)/ # Qgis version flavor FLAVOR:=3.16 BECOME_USER:=$(shell id -u) +BECOME_GROUP:=$(shell id -g) QGIS_IMAGE=$(REGISTRY_PREFIX)qgis-platform:$(FLAVOR) -LOCAL_HOME ?= $(shell pwd) - SRCDIR=$(shell realpath ..) test: - mkdir -p $$(pwd)/.local $(LOCAL_HOME)/.cache + mkdir -p $$(pwd)/.local $$(pwd)/.cache docker run --rm --name qgis-wfsoutputextension-test-$(FLAVOR)-$(COMMITID) -w /src/tests \ - -u $(BECOME_USER) \ + -u $(BECOME_USER):$(BECOME_GROUP) \ -v $(SRCDIR):/src \ -v $$(pwd)/.local:/.local \ - -v $(LOCAL_HOME)/.cache:/.cache \ + -v $$(pwd)/.cache:/.cache \ -e PIP_CACHE_DIR=/.cache \ - -e PYTEST_ADDOPTS="$(TEST_OPTS)" \ - $(QGIS_IMAGE) ./run-tests.sh + $(QGIS_IMAGE) ./run-tests.sh $(PYTEST_ADDOPTS) + +clean: + rm -rf $$(pwd)/.local $$(pwd)/.cache + +run: env + docker compose up -V --force-recreate + +stop: + docker compose down -v --remove-orphans + +env: + @echo "Creating environment file for docker-compose" + @cat <<-EOF > .env + WORKDIR=$(shell pwd) + QGIS_VERSION=$(FLAVOR) + QGIS_USER_ID=$(BECOME_USER) + QGIS_USER_GID=$(BECOME_GROUP) + SERVER_HTTP_PORT=127.0.0.1:8888 + SERVER_MANAGEMENT_PORT=127.0.0.1:19876 + EOF diff --git a/tests/conftest.py b/tests/conftest.py index 38f323e..7743a54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,22 +5,15 @@ import sys import tempfile import warnings +import lxml.etree +import pytest with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) import gdal -import lxml.etree -import pytest - from qgis.PyQt import Qt -logging.basicConfig(stream=sys.stderr) -logging.disable(logging.NOTSET) - -LOGGER = logging.getLogger('server') -LOGGER.setLevel(logging.DEBUG) - from typing import Any, Dict, Generator from qgis.core import Qgis, QgsApplication, QgsFontUtils, QgsProject @@ -31,6 +24,9 @@ QgsServerRequest, ) +LOGGER = logging.getLogger('server') +LOGGER.setLevel(logging.DEBUG) + qgis_application = None @@ -68,7 +64,7 @@ def pytest_sessionstart(session): qgis_application.initQgis() # Install logger hook - install_logger_hook(verbose=True) + install_logger_hook() def pytest_sessionfinish(session, exitstatus): @@ -89,7 +85,6 @@ def pytest_sessionfinish(session, exitstatus): 'xsi': "http://www.w3.org/2001/XMLSchema-instance" } - class OWSResponse: def __init__(self, resp: QgsBufferServerResponse) -> None: @@ -159,25 +154,32 @@ def getplugin(self, name) -> Any: def getprojectpath(self, name: str) -> str: return self.datapath.join(name) - def get(self, query: str, project: str=None) -> OWSResponse: + def request(self, method, query:str, project: str=None, data: bytes=None, + headers: dict={}) -> OWSResponse: """ Return server response from query """ - request = QgsBufferServerRequest(query, QgsServerRequest.GetMethod, {}, None) + request = QgsBufferServerRequest(query, method, headers, data) response = QgsBufferServerResponse() if project is not None and not os.path.isabs(project): projectpath = self.datapath.join(project) - qgsproject = QgsProject() + qgsproject = QgsProject() if not qgsproject.read(projectpath.strpath): - raise ValueError("Error reading project '%s':" % projectpath.strpath) + raise ValueError(f"Error reading project '{projectpath.strpath}'") else: qgsproject = None self.server.handleRequest(request, response, project=qgsproject) return OWSResponse(response) + def get(self, *args, **kwargs) -> OWSResponse: + return self.request(QgsServerRequest.GetMethod, *args, **kwargs) + + def delete(self, *args, **kwargs) -> OWSResponse: + return self.request(QgsServerRequest.DeleteMethod, *args, **kwargs) + return _Client() -## +## ## Plugins ## @@ -277,14 +279,12 @@ def install_logger_hook( verbose: bool=False ) -> None: # Add a hook to qgis message log def writelogmessage(message, tag, level): - arg = '{}: {}'.format(tag, message) + arg = '{}: {}'.format(tag,message) if level == Qgis.Warning: LOGGER.warning(arg) elif level == Qgis.Critical: LOGGER.error(arg) - elif verbose: - # Qgis is somehow very noisy - # log only if verbose is set + else: LOGGER.info(arg) messageLog = QgsApplication.messageLog() diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..8c216ee --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3.8' +services: + qgis-server: + image: 3liz/qgis-map-server:${QGIS_VERSION} + environment: + QGSRV_CACHE_ROOTDIR: /srv/projects + QGSRV_LOGGING_LEVEL: DEBUG + QGSRV_USER: ${QGIS_USER_ID}:${QGIS_USER_GID} + QGSRV_SERVER_WORKERS: '1' + QGSRV_SERVER_PLUGINPATH: /srv/plugins + QGSRV_MANAGEMENT_ENABLED: 'yes' + QGSRV_MANAGEMENT_INTERFACES: '0.0.0.0' + QGSRV_API_ENABLED_LANDING_PAGE: 'yes' + QGSRV_SERVER_TIMEOUT: '300' + QGIS_SERVER_TRUST_LAYER_METADATA: 'yes' + QGSRV_CACHE_STRICT_CHECK: 'no' + volumes: + - ${WORKDIR}:/srv/plugins + - ${WORKDIR}/tests/data:/srv/projects + ports: + - ${SERVER_HTTP_PORT}:8080 + - ${SERVER_MANAGEMENT_PORT}:19876 + diff --git a/tests/pytest.ini b/tests/pytest.ini index d7e467b..8360173 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,6 +1,8 @@ [pytest] addopts= --junit-xml=__output__/junit.xml junit_family=xunit2 +log_cli=1 +log_cli_level=critical norecursedirs= tests/data .local diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 6060960..5e3bdda 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -2,9 +2,6 @@ set -e -# Add /.local to path -export PATH=$PATH:/.local/bin - VENV_PATH=/.local/venv PIP_INSTALL="$VENV_PATH/bin/pip install -U" @@ -14,7 +11,7 @@ python3 -m venv --system-site-packages $VENV_PATH echo "Installing required packages..." $PIP_INSTALL -q pip setuptools wheel -$PIP_INSTALL -q --prefer-binary --user -r requirements.txt +$PIP_INSTALL -q --prefer-binary -r requirements.txt # Disable qDebug stuff that bloats test outputs export QT_LOGGING_RULES="*.debug=false;*.warning=false" @@ -22,9 +19,7 @@ export QT_LOGGING_RULES="*.debug=false;*.warning=false" # Disable python hooks/overrides export QGIS_DISABLE_MESSAGE_HOOKS=1 export QGIS_NO_OVERRIDE_IMPORT=1 - -echo -n "" > /src/tests/qgis_server.log -export QGIS_SERVER_LOG_FILE="/src/tests/qgis_server.log" export QGIS_SERVER_LOG_LEVEL=0 -pytest -v --qgis-plugins=/src $@ +exec $VENV_PATH/bin/pytest -v --qgis-plugins=/src $@ +