Audit Snyk check/fix prod-2-9-advance

.dockerignore

@@ -17,3 +17,7 @@ geoportal/geomapfish_geoportal/locale/*.pot
 !package-lock.json
 !tsconfig.json
 !vite.config.ts
+!ui
+ui/dist/
+ui/node_modules/
+ui/static/fa-*

.pre-commit-config.yaml

@@ -28,13 +28,17 @@ repos:
             |geoportal/webpack.apps.js
           )$
   - repo: https://github.com/sbrunner/integrity-updater
-    rev: 0.1.0
+    rev: 0.2.0
     hooks:
       - id: integrity-updater
         include: |-
           (?x)^(
             geoportal/geomapfish_geoportal/static/.*
+            |ui/.*
           )$
+        args:
+          - --pre-commit
+          - --blacklist=https://maps\.googleapis\.com/
   - repo: https://github.com/PyCQA/autoflake
     rev: v2.3.1
     hooks:
@@ -71,6 +75,10 @@ repos:
       - id: check-toml
       - id: check-yaml
       - id: check-json
+        exclude: |-
+          (?x)^(
+            ui/tsconfig\.json
+          )$
       - id: end-of-file-fixer
       - id: trailing-whitespace
       - id: mixed-line-ending
@@ -92,19 +100,22 @@ repos:
           (?x)^(
             pyproject\.toml
           )$
+      - id: npm-lock
   - repo: https://github.com/codespell-project/codespell
     rev: v2.3.0
     hooks:
       - id: codespell
         exclude: |-
           (?x)^(
             (.*/)?poetry\.lock
+            |(.*/)?package-lock\.json
             |ci/cleanup
             |geoportal/geomapfish_geoportal/locale/.*
             |qgisserver/.*\.qg[sz]
             |geoportal/geomapfish_geoportal/static/story-map\.html
             |tilegeneration/config\.yaml\.tmpl
             |webcomponents/feedback\.ts
+            |ui/src/webcomponents/feedback\.ts
           )$
         args:
           - --ignore-words=.github/spell-ignore-words.txt

.python-version

@@ -1 +1 @@
-3.10
+3.11

.secretsignore

@@ -1 +1,2 @@
-/geoportal/geomapfish_geoportal/static-ngeo/js/apps/desktop_alt.html.ejs
+[secrets]
+AIzaSyA3NVIy-HOYT0a0CkChA6nFwqEFqHYWBVk

Dockerfile

@@ -14,7 +14,7 @@ ENV CONFIG_VARS sqlalchemy.url sqlalchemy.pool_recycle sqlalchemy.pool_size sqla
     dbsessions urllogin host_forward_host headers_whitelist headers_blacklist \
     smtp c2c.base_path welcome_email \
     lingva_extractor interfaces_config interfaces devserver_url api authentication intranet metrics pdfreport \
-    vector_tiles i18next main_ogc_server
+    vector_tiles i18next main_ogc_server static_files
 
 COPY . /tmp/config/
 
@@ -77,7 +77,7 @@ VOLUME /etc/geomapfish \
 
 ###############################################################################
 
-FROM node:21.2-slim AS custom-build
+FROM node:20.18.0-slim AS webcomponent-build
 
 WORKDIR /app
 COPY package.json package-lock.json ./
@@ -86,9 +86,26 @@ RUN npm install --ignore-scripts
 
 COPY tsconfig.json vite.config.ts ./
 COPY webcomponents/ ./webcomponents/
-RUN npm run build
+RUN NODE_ENV=production npm run build
+
+###############################################################################
+
+FROM node:20.18.0-slim AS ui-build
+
+WORKDIR /app
+COPY ui/package.json ui/package-lock.json ./
+
+RUN npm install --ignore-scripts
+
+COPY ui/ ./
+RUN NODE_ENV=production npm run build
 
 ###############################################################################
 
 FROM gmf_config AS config
-COPY --from=custom-build /app/dist/ /etc/geomapfish/static/custom/
+
+COPY --from=webcomponent-build /app/dist/ /etc/geomapfish/static/custom/
+
+COPY --from=ui-build /app/dist/* /etc/static-frontend/
+COPY --from=ui-build /app/node_modules/ngeo/dist/fa-* /etc/static-frontend/
+VOLUME /etc/static-frontend/

Makefile

@@ -1,4 +1,4 @@
-PROJECT_PUBLIC_URL=https://geomapfish-demo-2-8.camptocamp.com/
+PROJECT_PUBLIC_URL=https://geomapfish-demo-2-9.camptocamp.com/
 DUMP_FILE=data/prod-2-7.dump
 PACKAGE=geomapfish
 LANGUAGES=en fr de it
@@ -29,6 +29,8 @@ checks: prospector eslint ## Runs the checks
 prospector: ## Runs the Prospector checks
 	docker compose run --entrypoint= --rm --volume=$(CURDIR)/geoportal:/app geoportal \
 		prospector --output-format=pylint --die-on-tool-error
+	docker build --tag=custom-checks --target=checks custom
+	docker run --rm custom-checks prospector --output-format=pylint --die-on-tool-error
 
 .PHONY: eslint
 eslint: ## Runs the eslint checks

ci/cleanup

@@ -42,4 +42,3 @@ rm "$1/package-lock.json"
 rm "$1/custom/requirements.txt"
 rm "$1/custom/pyproject.toml"
 rm "$1/custom/poetry.lock"
-rm "$1/.secretsignore"

custom/.prospector.yaml

@@ -1,8 +1,13 @@
+inherits:
+  - utils:base
+  - utils:fix
+  - utils:no-design-checks
+  - duplicated
+
 pylint:
   disable:
     - missing-timeout # Default timeout set by c2cwsgiutils
 
 bandit:
-  run: true
   options:
     config: .bandit.yaml

custom/Dockerfile

@@ -1,21 +1,23 @@
-FROM ghcr.io/osgeo/gdal:ubuntu-small-3.6.4 AS base-all
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.9.2 AS base-all
 
 # Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006.
 # Treat unset variables as an error when substituting.
 # Print commands and their arguments as they are executed.
 SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
 
-ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
-    SETUPTOOLS_USE_DISTUTILS=stdlib
+ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
 
 RUN --mount=type=cache,target=/var/lib/apt/lists \
     apt-get update
 
 RUN --mount=type=cache,target=/var/lib/apt/lists \
     --mount=type=cache,target=/var/cache,sharing=locked \
-    apt-get install --assume-yes --no-install-recommends python3-pip python3-dev
+    apt-get install --assume-yes --no-install-recommends python3-pip python3-dev python3-venv \
+    && python3 -m venv /venv
 
-FROM base-all as poetry
+ENV PATH=/venv/bin:$PATH
+
+FROM base-all AS poetry
 
 WORKDIR /tmp
 COPY requirements.txt ./
@@ -40,13 +42,13 @@ RUN --mount=type=cache,target=/var/cache,sharing=locked \
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=bind,from=poetry,source=/tmp,target=/tmp \
     python3 -m pip install --disable-pip-version-check --no-deps --requirement=/tmp/requirements.txt \
-    && strip /usr/local/lib/python3.*/dist-packages/*/*.so \
+    && strip /usr/lib/python3/dist-packages/*/*.so \
     && python3 -m compileall -q /usr/local/lib/python3.* -x '/(ptvsd|.*pydev.*|networkx)/'
 
 # hadolint ignore=DL3059
 RUN apt-get remove --autoremove --assume-yes gcc
 
-FROM base AS lint
+FROM base AS checks
 
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=bind,from=poetry,source=/tmp,target=/tmp \
@@ -56,22 +58,17 @@ WORKDIR /app
 COPY . ./
 RUN --mount=type=cache,target=/root/.cache \
     python3 -m pip install --disable-pip-version-check --no-deps --editable=. \
-    && python3 -m compileall -q /app/custom \
-    && prospector --output=pylint -X . \
-    && touch /tmp/lint.ok
+    && python3 -m compileall -q /app/custom
 
 FROM base AS runtime
 
-# Force to urn the lint with BUILD KIT
-COPY --from=lint /tmp/lint.ok /tmp/
-
 WORKDIR /app
 COPY . ./
 RUN --mount=type=cache,target=/root/.cache \
     python3 -m pip install --disable-pip-version-check --no-deps --editable=. \
     && python3 -m compileall -q /app/custom
 
-CMD [ "/usr/local/bin/gunicorn", "--paste=production.ini" ]
+CMD [ "/venv/bin/gunicorn", "--paste=production.ini" ]
 
 ARG GIT_HASH
 ENV GIT_HASH=${GIT_HASH}

custom/custom/__init__.py

@@ -1,6 +1,7 @@
 import c2cwsgiutils.db
 import c2cwsgiutils.health_check
-from pyramid.config import Configurator
+from papyrus.renderers import GeoJSON  # type: ignore[import-untyped]
+from pyramid.config import Configurator  # type: ignore[import-untyped]
 
 
 def main(global_config, **settings):
@@ -12,6 +13,7 @@ def main(global_config, **settings):
         config.include(".routes")
         config.include("c2cwsgiutils.pyramid")
         dbsession = c2cwsgiutils.db.init(config, "sqlalchemy", "sqlalchemy_slave")
+        config.add_renderer("geojson", GeoJSON())
         config.scan()
         # Initialize the health checks
         health_check = c2cwsgiutils.health_check.HealthCheck(config)

custom/custom/alembic/env.py

@@ -1,7 +1,7 @@
 """Pyramid bootstrap environment. """
 
 from alembic import context
-from pyramid.paster import get_appsettings, setup_logging
+from pyramid.paster import get_appsettings, setup_logging  # type: ignore[import-untyped]
 from sqlalchemy import engine_from_config
 
 from custom.models.meta import Base

custom/custom/scripts/initialize_db.py

@@ -1,7 +1,7 @@
 import argparse
 import sys
 
-from pyramid.paster import bootstrap, setup_logging
+from pyramid.paster import bootstrap, setup_logging  # type: ignore[import-untyped]
 from sqlalchemy.exc import OperationalError
 
 from custom.models.meta import Base

custom/custom/views/__init__.py

@@ -1,8 +1,8 @@
 import os
 
-import pyramid.request
-import pyramid.response
-from cornice import Service
+import pyramid.request  # type: ignore[import-untyped]
+import pyramid.response  # type: ignore[import-untyped]
+from cornice import Service  # type: ignore[import-untyped]
 
 index = Service(
     name="index",

custom/custom/views/cog.py

@@ -1,12 +1,12 @@
 import logging
 import os
 
-import pyramid.request
-import pyramid.response
+import pyramid.request  # type: ignore[import-untyped]
+import pyramid.response  # type: ignore[import-untyped]
 from azure.identity import DefaultAzureCredential
 from azure.storage.blob import BlobServiceClient, ContainerClient
-from cornice import Service
-from pyramid.httpexceptions import HTTPBadRequest
+from cornice import Service  # type: ignore[import-untyped]
+from pyramid.httpexceptions import HTTPBadRequest  # type: ignore[import-untyped]
 
 _LOGGING = logging.getLogger(__name__)
 _CLIENT = None
@@ -38,7 +38,7 @@ def _get_azure_container_client(container: str) -> ContainerClient:
 @feedback.get()
 def swissalti3d(request: pyramid.request.Request) -> pyramid.response.Response:
     # Just to demonstrate that we can fet the user information
-    global _CLIENT
+    global _CLIENT  # pylint: disable=global-statement
     if _CLIENT is None:
         _CLIENT = _get_azure_container_client(os.environ["AZURE_CONTAINER_NAME"])
     blob = _CLIENT.get_blob_client(blob="swissalti3d_2m_archeo.tif")

custom/custom/views/feedback.py

@@ -2,10 +2,10 @@
 import os
 from typing import Any
 
-import pyramid.request
+import pyramid.request  # type: ignore[import-untyped]
 import requests
-from cornice import Service
-from pyramid.httpexceptions import HTTPBadRequest
+from cornice import Service  # type: ignore[import-untyped]
+from pyramid.httpexceptions import HTTPBadRequest  # type: ignore[import-untyped]
 
 from custom.models.feedback import Feedback
 from custom.util.send_mail import send_mail
@@ -18,6 +18,11 @@
     path="/feedback",
     cors_origins=(
         (f'https://{os.environ["VISIBLE_WEB_HOST"]}' if "VISIBLE_WEB_HOST" in os.environ else "*"),
+        *(
+            ["https://localhost:3002"]
+            if os.environ.get("DEV", "false").lower() in ("1", "true", "yes")
+            else []
+        ),
     ),
 )
 
@@ -32,7 +37,7 @@ def feedback_post(request: pyramid.request.Request) -> Any:
                 headers={"Cookie": request.headers.get("Cookie"), "Referer": request.referrer},
             ).json()
         )
-    except Exception:
+    except Exception:  # pylint: disable=broad-exception-caught
         LOG.exception("Error on get user information")
 
     if (
@@ -64,12 +69,12 @@ def feedback_post(request: pyramid.request.Request) -> Any:
     text = "\n\n".join(
         [
             "Ceci est un email automatique. Un nouveau feedback a été inséré dans la BD.",
-            "Cela concerne l'instance : " + instance,
-            "Son identifiant est le : " + str(new_feedback.id_feedback),
-            "User agent : " + new_feedback.ua,
-            "Permalink : " + new_feedback.permalink,
-            "User email : " + new_feedback.email,
-            "User text : " + new_feedback.text,
+            "Cela concerne l'instance: " + instance,
+            "Son identifiant est le: " + str(new_feedback.id_feedback),
+            "User agent: " + new_feedback.ua,  # type: ignore[list-item]
+            "Permalink: " + new_feedback.permalink,  # type: ignore[list-item]
+            "User email: " + new_feedback.email,  # type: ignore[list-item]
+            "User text: " + new_feedback.text,  # type: ignore[list-item]
         ]
     )
     subject = "Feedback - Guichet cartographique"

custom/custom/views/notfound.py

@@ -1,4 +1,4 @@
-from pyramid.view import notfound_view_config
+from pyramid.view import notfound_view_config  # type: ignore[import-untyped]
 
 
 @notfound_view_config(renderer="custom:templates/404.mako")