Merge pull request #1522 from camptocamp/add-image-check-function

Add functions to test images

.dockerignore

@@ -11,5 +11,4 @@
 !Pipfile.lock
 !setup.py
 !setup.cfg
-!tests
 **/__pycache__

.github/workflows/main.yaml

@@ -36,6 +36,9 @@ jobs:
       - name: Build
         run: make build
 
+      - name: Checks
+        run: make checks
+
       - name: Acceptance
         run: make acceptance
 

.gitignore

@@ -7,3 +7,4 @@ __pycache__
 /docs/*/build/
 /.mypy_cache
 /acceptance_tests/tests/docker-compose.override.yaml
+/results/

Dockerfile

@@ -80,8 +80,8 @@ FROM base-lint as tests
 COPY . /opt/c2cwsgiutils/
 RUN python3 -m pip install --disable-pip-version-check --no-cache-dir --no-deps \
     --editable=/opt/c2cwsgiutils
-RUN (cd /opt/c2cwsgiutils/ && prospector -X --output=pylint)
-RUN (cd /opt/c2cwsgiutils/ && pytest -vv --cov=c2cwsgiutils --color=yes tests && rm -r tests)
+
+WORKDIR /opt/c2cwsgiutils
 
 
 FROM base as standard

Makefile

@@ -21,13 +21,30 @@ all: build acceptance
 .PHONY: build
 build: build_docker build_acceptance build_test_app
 
+.PHONY: checks
+checks: prospector pytest
+
+.PHONY: prospector
+prospector: build_docker_test
+	docker run \
+	--volume=$(shell pwd)/c2cwsgiutils:/opt/c2cwsgiutils/c2cwsgiutils \
+	--volume=$(shell pwd)/tests:/opt/c2cwsgiutils/tests \
+	$(DOCKER_BASE):tests prospector --die-on-tool-error --output=pylint .
+
+.PHONY: pytest
+pytest: build_docker_test
+	mkdir -p reports/coverage/api/
+	docker run \
+	--volume=$(shell pwd)/results:/results \
+	--volume=$(shell pwd)/c2cwsgiutils:/opt/c2cwsgiutils/c2cwsgiutils \
+	--volume=$(shell pwd)/tests:/opt/c2cwsgiutils/tests \
+	$(DOCKER_BASE):tests pytest -vv --cov=c2cwsgiutils --color=yes tests > reports/coverage/api/coverage.ut.1
+
 .PHONY: acceptance
-acceptance: build_acceptance build_test_app
+acceptance: build_acceptance build_test_app pytest
 	docker build --tag=camptocamp/c2cwsgiutils-redis-sentinel:6 acceptance_tests/tests/redis/
 	rm -rf reports/coverage/api reports/acceptance.xml
 	mkdir -p reports/coverage/api
-	# Get the UT reports
-	docker run --rm $(DOCKER_BASE):tests cat /opt/c2cwsgiutils/.coverage > reports/coverage/api/coverage.ut.1
 	# Run the tests
 	docker run $(DOCKER_TTY) --volume=/var/run/docker.sock:/var/run/docker.sock \
 		--name=c2cwsgiutils_acceptance_$$PPID --env=WAITRESS $(DOCKER_BASE)_acceptance \
@@ -95,5 +112,5 @@ c2cciutils: .venv/timestamp
 .PHONY: acceptance_local
 acceptance_local: .venv/timestamp
 	DOCKER_RUN=0 ./.venv/bin/pytest \
-	-vv --color=yes --junitxml reports/acceptance.xml --html reports/acceptance.html \
+	-vv --color=yes --junitxml=reports/acceptance.xml --html=reports/acceptance.html \
 	--self-contained-html $(PYTEST_OPTS) acceptance_tests/tests

Pipfile

@@ -16,6 +16,7 @@ types-redis = "==4.2.8"
 types-ujson = "==5.2.0"
 types-python-dateutil = "==2.8.18"
 typing-extensions = "==4.2.0"
+scikit-image = "==0.19.3"
 
 [packages]
 alembic = "==1.8.0"

c2cwsgiutils/acceptance/image.py

@@ -0,0 +1,89 @@
+import os
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import skimage.color
+import skimage.io
+import skimage.metrics
+import skimage.transform
+
+if TYPE_CHECKING:
+    NpNdarrayInt = np.ndarray[np.uint8, Any]
+else:
+    NpNdarrayInt = np.ndarray
+
+
+def check_image_file(
+    result_folder: str,
+    image_filename_to_check: str,
+    expected_filename: str,
+    level: float = 1.0,
+    generate_expected_image: bool = False,
+) -> None:
+    """
+    Test that the `<image_filename_to_check>` corresponds to the image `tests/<image_filename>.expected.png`.
+
+    If the images differ too much, `<result_folder>`/<image_filename>.result.png and
+    `<result_folder>`/<image_filename>.diff.png are created with the corresponding content.
+
+    Where `<image_filename>` is the name of the `expected_filename`.
+
+    `generate_expected_image` can be set to True to generate the expected image, but it should be
+    set to False in the committed code, because it also disable the test.
+    """
+    result = skimage.io.imread(image_filename_to_check)
+    assert result is not None, "Wrong image: " + image_filename_to_check
+    check_image(result_folder, result, expected_filename, level, generate_expected_image)
+
+
+def check_image(
+    result_folder: str,
+    image_to_check: NpNdarrayInt,
+    expected_filename: str,
+    level: float = 1.0,
+    generate_expected_image: bool = False,
+) -> None:
+    """
+    Test that the `<image_to_check>` corresponds to the image `tests/<image_filename>.expected.png`.
+
+    If they don't corresponds the images `<result_folder>`/<image_filename>.result.png and
+    `<result_folder>`/<image_filename>.diff.png are created with the corresponding content.
+
+    Where is the name if the `expected_filename`.
+
+    `generate_expected_image` can be set to True to generate the expected image, but it should be
+    set to False in the committed code, because it also disable the test.
+    """
+    assert image_to_check is not None, "Image required"
+    image_file_basename = os.path.splitext(os.path.basename(expected_filename))[0]
+    if image_file_basename.endswith(".expected"):
+        image_file_basename = os.path.splitext(image_file_basename)[0]
+    result_filename = os.path.join(result_folder, f"{image_file_basename}.result.png")
+    diff_filename = os.path.join(result_folder, f"{image_file_basename}.diff.png")
+
+    if not os.path.exists(result_folder):
+        os.makedirs(result_folder)
+    if generate_expected_image:
+        skimage.io.imsave(expected_filename, image_to_check)
+        return
+    if not os.path.isfile(expected_filename):
+        skimage.io.imsave(result_filename, image_to_check)
+        skimage.io.imsave(expected_filename, image_to_check)
+        assert False, "Expected image not found: " + expected_filename
+    expected = skimage.io.imread(expected_filename)
+    assert expected is not None, "Wrong image: " + expected_filename
+
+    score, diff = skimage.metrics.structural_similarity(
+        expected, image_to_check, multichannel=True, full=True
+    )
+    diff = (255 - diff * 255).astype("uint8")
+
+    if diff is None:
+        skimage.io.imsave(result_filename, image_to_check)
+        assert diff is not None, "No diff generated"
+    if score < level:
+        skimage.io.imsave(result_filename, image_to_check)
+        skimage.io.imsave(diff_filename, diff)
+        assert (
+            score >= level
+        ), f"{result_filename} != {expected_filename} => {diff_filename} ({score} > {level})"

c2cwsgiutils/setup_process.py

@@ -9,22 +9,14 @@
 import warnings
 from typing import Any, Callable, Dict, Optional, TypedDict, cast
 
+import pyramid.config
 import pyramid.registry
 import pyramid.request
 import pyramid.router
-import pyramid.config
-
 from pyramid.paster import bootstrap
 from pyramid.scripts.common import get_config_loader, parse_vars
 
-from c2cwsgiutils import (
-    broadcast,
-    coverage_setup,
-    redis_stats,
-    sentry,
-    stats,
-    sql_profiler,
-)
+from c2cwsgiutils import broadcast, coverage_setup, redis_stats, sentry, sql_profiler, stats
 
 
 def fill_arguments(

setup.py

@@ -52,6 +52,7 @@ def long_description() -> str:
     extras_require={
         "standard": [e for e in INSTALL_REQUIRES if e != "redis"],
         "broadcast": ["redis"],
+        "test_images": ["scikit-image"],
     },
     entry_points={
         "console_scripts": [

tests/test_acceptance_image.py

@@ -0,0 +1,21 @@
+import os
+
+import pytest
+
+from c2cwsgiutils.acceptance.image import check_image_file
+
+
+def test_good():
+    image = os.path.join(os.path.dirname(__file__), "test.expected.png")
+    check_image_file("/results", image, os.path.join(os.path.dirname(__file__), "test.expected.png"))
+
+
+def test_wrong():
+    image = os.path.join(os.path.dirname(__file__), os.path.join(os.path.dirname(__file__), "test.wrong.png"))
+    with pytest.raises(AssertionError):
+        check_image_file("/results", image, os.path.join(os.path.dirname(__file__), "test.expected.png"))
+    check_image_file(
+        "/results",
+        "/results/test.diff.png",
+        os.path.join(os.path.dirname(__file__), "test.diff.expected.png"),
+    )