From 553601c570a6ce5d777d3846e8924c54966b495d Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 27 Sep 2023 16:52:39 +0200 Subject: [PATCH 01/13] Update actions - Add PR build & test - Add PR closed build & push ckan-docker image --- .github/{workflows => .old}/build-master.yml | 0 .github/{workflows => .old}/build-tags.yml | 0 .github/{workflows => .old}/docker-master.yml | 10 +- .github/{workflows => .old}/docker-tags.yml | 0 .github/workflows/docker-build.yml | 78 ++++++++++++ .github/workflows/docker-pr.yml | 113 ++++++++++++++++++ 6 files changed, 196 insertions(+), 5 deletions(-) rename .github/{workflows => .old}/build-master.yml (100%) rename .github/{workflows => .old}/build-tags.yml (100%) rename .github/{workflows => .old}/docker-master.yml (85%) rename .github/{workflows => .old}/docker-tags.yml (100%) create mode 100644 .github/workflows/docker-build.yml create mode 100644 .github/workflows/docker-pr.yml diff --git a/.github/workflows/build-master.yml b/.github/.old/build-master.yml similarity index 100% rename from .github/workflows/build-master.yml rename to .github/.old/build-master.yml diff --git a/.github/workflows/build-tags.yml b/.github/.old/build-tags.yml similarity index 100% rename from .github/workflows/build-tags.yml rename to .github/.old/build-tags.yml diff --git a/.github/workflows/docker-master.yml b/.github/.old/docker-master.yml similarity index 85% rename from .github/workflows/docker-master.yml rename to .github/.old/docker-master.yml index 6fa27c89..826cb673 100644 --- a/.github/workflows/docker-master.yml +++ b/.github/.old/docker-master.yml @@ -20,17 +20,17 @@ jobs: steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Login to registry if: github.event_name != 'pull_request' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -43,7 +43,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/docker-tags.yml b/.github/.old/docker-tags.yml similarity index 100% rename from .github/workflows/docker-tags.yml rename to .github/.old/docker-tags.yml diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 00000000..555e7e5d --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,78 @@ +name: Build and push ckan-docker image from PR Merge + +on: + pull_request: + types: + - closed + branches: + - main + - 'ckan-*.*.*' + - '!dev/ckan-*.*.*' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + TAG: ghcr.io/${{ github.repository }}:${{ github.head_ref }} + CONTEXT: . + BRANCH: ${{ github.head_ref }} + DOCKERFILE_PATH: /ckan + DOCKERFILE: Dockerfile + +jobs: + docker: + name: runner/build-docker-push:${{ github.head_ref }} + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + labels: | + org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md + org.opencontainers.image.version=${{ env.BRANCH }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + push: true + tags: ${{ env.TAG }} + labels: ${{ steps.meta.outputs.labels }} + context: ${{ env.CONTEXT }} + file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + + - name: Linting Dockerfile with hadolint in GH Actions + uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + + - name: Run Trivy container image vulnerability scanner + uses: aquasecurity/trivy-action@0.12.0 + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + format: sarif + output: trivy-results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: trivy-results.sarif \ No newline at end of file diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml new file mode 100644 index 00000000..ca60c187 --- /dev/null +++ b/.github/workflows/docker-pr.yml @@ -0,0 +1,113 @@ +name: Test ckan-docker images (PR) + +on: + pull_request: + branches: + - main + - 'ckan-*.*.*' + - '!dev/ckan-*.*.*' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + CONTEXT: . + BRANCH: ${{ github.head_ref }} + DOCKERFILE_PATH: /ckan + DOCKERFILE: Dockerfile + HADOLINT_VERSION: 2.12.0 + +jobs: + docker: + name: runner/test-docker-pr:${{ github.head_ref }} + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Checkout + uses: actions/checkout@v4 + + - name: NGINX build + uses: docker/build-push-action@v5 + with: + context: ./nginx + file: ./nginx/Dockerfile + push: false + tags: mjanez/ckan-docker-nginx:test-build-only + + - name: Apache HTTP Server build + uses: docker/build-push-action@v5 + with: + context: ./apache + file: ./apache/Dockerfile + push: false + tags: mjanez/ckan-docker-apache:test-build-only + + - name: PostgreSQL build + uses: docker/build-push-action@v5 + with: + context: ./postgresql + file: ./postgresql/Dockerfile + push: false + tags: mjanez/ckan-docker-postgresql:test-build-only + + - name: Solr build + uses: docker/build-push-action@v5 + with: + context: ./solr + file: ./solr/Dockerfile + push: false + tags: mjanez/ckan-docker-solr:test-build-only + + - name: ckan-pycsw build + uses: docker/build-push-action@v4 + with: + context: ./ckan-pycsw + file: ./ckan-pycsw/Dockerfile + push: false + tags: mjanez/ckan-docker-pycsw:test-build-only + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + labels: | + org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md + org.opencontainers.image.version=${{ env.BRANCH }} + + - name: Build to test + uses: docker/build-push-action@v5 + id: docker-push + with: + push: false + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + labels: ${{ steps.meta.outputs.labels }} + context: ${{ env.CONTEXT }} + file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + + - name: Linting Dockerfile and annotate code inline in the github PR viewer + id: hadolint + uses: jbergstroem/hadolint-gh-action@v1.11.0 + with: + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + version: ${{ env.HADOLINT_VERSION }} + annotate: true + error_level: -1 + + - name: Run Trivy container image vulnerability scanner + uses: aquasecurity/trivy-action@0.12.0 + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + format: sarif + output: trivy-results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: trivy-results.sarif \ No newline at end of file From e95f2a1a210f9472d5765abfab464f17bf49d147 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 27 Sep 2023 16:56:16 +0200 Subject: [PATCH 02/13] Avoid */* branches test --- .github/workflows/docker-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index ca60c187..f79d72ac 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -6,6 +6,7 @@ on: - main - 'ckan-*.*.*' - '!dev/ckan-*.*.*' + - '!*/*' env: REGISTRY: ghcr.io From c8eb770920f4dd527ac278498abcf86b1b5dd8e0 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:01:30 +0200 Subject: [PATCH 03/13] Add :test-build-only tag --- .github/workflows/docker-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index f79d72ac..502e08fb 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -86,7 +86,7 @@ jobs: id: docker-push with: push: false - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test-build-only labels: ${{ steps.meta.outputs.labels }} context: ${{ env.CONTEXT }} file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} @@ -103,7 +103,7 @@ jobs: - name: Run Trivy container image vulnerability scanner uses: aquasecurity/trivy-action@0.12.0 with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test-build-only format: sarif output: trivy-results.sarif From 21b09029593c699b4230482a9e57ef6fb755bf21 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:17:03 +0200 Subject: [PATCH 04/13] Fix docker-pr --- .github/workflows/docker-pr.yml | 11 ++++------- ckan/Dockerfile | 1 + 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index 502e08fb..f62163ba 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -10,7 +10,7 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME: ckan-docker-spatial CONTEXT: . BRANCH: ${{ github.head_ref }} DOCKERFILE_PATH: /ckan @@ -77,9 +77,6 @@ jobs: uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - labels: | - org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md - org.opencontainers.image.version=${{ env.BRANCH }} - name: Build to test uses: docker/build-push-action@v5 @@ -88,14 +85,14 @@ jobs: push: false tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test-build-only labels: ${{ steps.meta.outputs.labels }} - context: ${{ env.CONTEXT }} - file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + context: ./ckan + file: ./ckan/Dockerfile - name: Linting Dockerfile and annotate code inline in the github PR viewer id: hadolint uses: jbergstroem/hadolint-gh-action@v1.11.0 with: - dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + dockerfile: ./ckan/Dockerfile version: ${{ env.HADOLINT_VERSION }} annotate: true error_level: -1 diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 66e9b08b..da268584 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,5 @@ FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.9 +LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables ENV APP_DIR=/srv/app \ From 83ecd027d2fd748c47d680718c9f5beaec94d143 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:37:35 +0200 Subject: [PATCH 05/13] Update actions --- .github/workflows/docker-manual.yml | 6 +++--- .github/workflows/docker-pr.yml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-manual.yml b/.github/workflows/docker-manual.yml index 24ea7240..909cad5f 100644 --- a/.github/workflows/docker-manual.yml +++ b/.github/workflows/docker-manual.yml @@ -48,8 +48,8 @@ jobs: push: true tags: ${{ env.TAG }} labels: ${{ steps.meta.outputs.labels }} - context: ${{ env.CONTEXT }} - file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} + context: ./ckan + file: ./ckan/Dockerfile - name: Linting Dockerfile with hadolint in GH Actions uses: hadolint/hadolint-action@v3.1.0 @@ -59,7 +59,7 @@ jobs: - name: Run Trivy container image vulnerability scanner uses: aquasecurity/trivy-action@0.12.0 with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} + image-ref: ${{ steps.meta.outputs.tags }} format: sarif output: trivy-results.sarif diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index f62163ba..a66409ed 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -14,7 +14,7 @@ env: CONTEXT: . BRANCH: ${{ github.head_ref }} DOCKERFILE_PATH: /ckan - DOCKERFILE: Dockerfile + DOCKERFILE: Dockerfile.ghcr HADOLINT_VERSION: 2.12.0 jobs: @@ -83,16 +83,16 @@ jobs: id: docker-push with: push: false - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test-build-only + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} context: ./ckan - file: ./ckan/Dockerfile + file: ./ckan/Dockerfile.ghcr - name: Linting Dockerfile and annotate code inline in the github PR viewer id: hadolint uses: jbergstroem/hadolint-gh-action@v1.11.0 with: - dockerfile: ./ckan/Dockerfile + dockerfile: ./ckan/Dockerfile.ghcr version: ${{ env.HADOLINT_VERSION }} annotate: true error_level: -1 @@ -100,7 +100,7 @@ jobs: - name: Run Trivy container image vulnerability scanner uses: aquasecurity/trivy-action@0.12.0 with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test-build-only + image-ref: ${{ steps.meta.outputs.tags }} format: sarif output: trivy-results.sarif From 42d2ddb91c7144540f95401b3f3353d53e834d80 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 27 Sep 2023 18:17:59 +0200 Subject: [PATCH 06/13] Fix actions --- .github/workflows/docker-manual.yml | 6 +++--- .github/workflows/docker-pr.yml | 29 +++-------------------------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/.github/workflows/docker-manual.yml b/.github/workflows/docker-manual.yml index 909cad5f..24ea7240 100644 --- a/.github/workflows/docker-manual.yml +++ b/.github/workflows/docker-manual.yml @@ -48,8 +48,8 @@ jobs: push: true tags: ${{ env.TAG }} labels: ${{ steps.meta.outputs.labels }} - context: ./ckan - file: ./ckan/Dockerfile + context: ${{ env.CONTEXT }} + file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} - name: Linting Dockerfile with hadolint in GH Actions uses: hadolint/hadolint-action@v3.1.0 @@ -59,7 +59,7 @@ jobs: - name: Run Trivy container image vulnerability scanner uses: aquasecurity/trivy-action@0.12.0 with: - image-ref: ${{ steps.meta.outputs.tags }} + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }} format: sarif output: trivy-results.sarif diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index a66409ed..f9a6a52e 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -14,7 +14,7 @@ env: CONTEXT: . BRANCH: ${{ github.head_ref }} DOCKERFILE_PATH: /ckan - DOCKERFILE: Dockerfile.ghcr + DOCKERFILE: Dockerfile HADOLINT_VERSION: 2.12.0 jobs: @@ -78,34 +78,11 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Build to test - uses: docker/build-push-action@v5 - id: docker-push - with: - push: false - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - context: ./ckan - file: ./ckan/Dockerfile.ghcr - - name: Linting Dockerfile and annotate code inline in the github PR viewer id: hadolint uses: jbergstroem/hadolint-gh-action@v1.11.0 with: - dockerfile: ./ckan/Dockerfile.ghcr + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} version: ${{ env.HADOLINT_VERSION }} annotate: true - error_level: -1 - - - name: Run Trivy container image vulnerability scanner - uses: aquasecurity/trivy-action@0.12.0 - with: - image-ref: ${{ steps.meta.outputs.tags }} - format: sarif - output: trivy-results.sarif - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - if: always() - with: - sarif_file: trivy-results.sarif \ No newline at end of file + error_level: -1 \ No newline at end of file From 6d0ef87ab9c1232cc79ce4e884989bb75361f4e6 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 28 Sep 2023 09:37:49 +0200 Subject: [PATCH 07/13] Allow master branch Also avoid all branches except master and ckan-*.*.* --- .github/workflows/docker-build.yml | 4 +++- .github/workflows/docker-pr.yml | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 555e7e5d..1e1719fe 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -5,9 +5,11 @@ on: types: - closed branches: - - main + - master - 'ckan-*.*.*' - '!dev/ckan-*.*.*' + - '!feature/*' + - '!fix/*' env: REGISTRY: ghcr.io diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index f9a6a52e..6747fb2a 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -3,10 +3,11 @@ name: Test ckan-docker images (PR) on: pull_request: branches: - - main + - master - 'ckan-*.*.*' - '!dev/ckan-*.*.*' - - '!*/*' + - '!feature/*' + - '!fix/*' env: REGISTRY: ghcr.io From 3c9d9166f1542ce2f1f592ef7ab2893f51d19189 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 28 Sep 2023 09:42:22 +0200 Subject: [PATCH 08/13] Add hadolint to Dockerfile.ghcr image --- .github/workflows/docker-pr.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index 6747fb2a..d2c3c3c3 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -86,4 +86,13 @@ jobs: dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} version: ${{ env.HADOLINT_VERSION }} annotate: true + error_level: -1 + + - name: Linting Dockerfile.ghcr and annotate code inline in the github PR viewer + id: hadolint_ghcr + uses: jbergstroem/hadolint-gh-action@v1.11.0 + with: + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}.ghcr + version: ${{ env.HADOLINT_VERSION }} + annotate: true error_level: -1 \ No newline at end of file From 2c99632c1d285b00a7b3fc0e32d7f8a911cf096a Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 28 Sep 2023 09:50:50 +0200 Subject: [PATCH 09/13] Add multiple Dockerfiles to hadolint --- .github/workflows/docker-pr.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index d2c3c3c3..dac2339a 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -79,20 +79,11 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Linting Dockerfile and annotate code inline in the github PR viewer + - name: Linting Dockerfiles and annotate code inline in the github PR viewer id: hadolint uses: jbergstroem/hadolint-gh-action@v1.11.0 with: - dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} - version: ${{ env.HADOLINT_VERSION }} - annotate: true - error_level: -1 - - - name: Linting Dockerfile.ghcr and annotate code inline in the github PR viewer - id: hadolint_ghcr - uses: jbergstroem/hadolint-gh-action@v1.11.0 - with: - dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}.ghcr + dockerfile: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }} ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}.ghcr version: ${{ env.HADOLINT_VERSION }} annotate: true error_level: -1 \ No newline at end of file From 5d9ad919c2f22724d074ae330d1a8e8618751d89 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 28 Sep 2023 09:54:45 +0000 Subject: [PATCH 10/13] Add ckanext-pages patch for root_path Pages/Blogs needs a ckan.root_path in the URL if set in the instance configuration. --- ckan/patches/ckanext-pages/00_root_path.patch | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 ckan/patches/ckanext-pages/00_root_path.patch diff --git a/ckan/patches/ckanext-pages/00_root_path.patch b/ckan/patches/ckanext-pages/00_root_path.patch new file mode 100644 index 00000000..71e06fae --- /dev/null +++ b/ckan/patches/ckanext-pages/00_root_path.patch @@ -0,0 +1,31 @@ +diff --git a/ckanext/pages/plugin.py b/ckanext/pages/plugin.py +index fbd1be0..bdb9a35 100644 +--- a/ckanext/pages/plugin.py ++++ b/ckanext/pages/plugin.py +@@ -1,5 +1,6 @@ + + import logging ++import re + from html import escape as html_escape + + from six.moves.urllib.parse import quote +@@ -25,6 +26,10 @@ def build_pages_nav_main(*args): + about_menu = tk.asbool(tk.config.get('ckanext.pages.about_menu', True)) + group_menu = tk.asbool(tk.config.get('ckanext.pages.group_menu', True)) + org_menu = tk.asbool(tk.config.get('ckanext.pages.organization_menu', True)) ++ ++ #FIXME: link needs root_path if exists ++ root_path = tk.config.get('ckan.root_path', '') ++ root_path = re.sub('/{{LANG}}', '', root_path) + + new_args = [] + for arg in args: +@@ -51,7 +56,7 @@ def build_pages_nav_main(*args): + type_ = 'blog' if page['page_type'] == 'blog' else 'pages' + name = quote(page['name']) + title = html_escape(page['title']) +- link = tk.h.literal(u'{}'.format(type_, name, title)) ++ link = tk.h.literal(u'{}'.format(root_path, type_, name, title)) + if page['name'] == page_name: + li = tk.literal('
  • ') + link + tk.literal('
  • ') + else: From ce7f950bcd608dc80be2e3c3970f8437f6fdadcc Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 28 Sep 2023 11:13:04 +0000 Subject: [PATCH 11/13] Update ckanext-pages patch to include lang if not default --- ckan/patches/ckanext-pages/00_root_path.patch | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ckan/patches/ckanext-pages/00_root_path.patch b/ckan/patches/ckanext-pages/00_root_path.patch index 71e06fae..9e388115 100644 --- a/ckan/patches/ckanext-pages/00_root_path.patch +++ b/ckan/patches/ckanext-pages/00_root_path.patch @@ -1,22 +1,23 @@ diff --git a/ckanext/pages/plugin.py b/ckanext/pages/plugin.py -index fbd1be0..bdb9a35 100644 +index fbd1be0..2882f97 100644 --- a/ckanext/pages/plugin.py +++ b/ckanext/pages/plugin.py -@@ -1,5 +1,6 @@ +@@ -8,6 +8,7 @@ from ckan.plugins import toolkit as tk - import logging -+import re - from html import escape as html_escape + import ckan.plugins as p + from ckan.lib.helpers import build_nav_main as core_build_nav_main ++from ckan.lib.helpers import lang - from six.moves.urllib.parse import quote + from ckanext.pages import actions, db + from ckanext.pages import auth @@ -25,6 +26,10 @@ def build_pages_nav_main(*args): about_menu = tk.asbool(tk.config.get('ckanext.pages.about_menu', True)) group_menu = tk.asbool(tk.config.get('ckanext.pages.group_menu', True)) org_menu = tk.asbool(tk.config.get('ckanext.pages.organization_menu', True)) + -+ #FIXME: link needs root_path if exists ++ # get the root path from the config and use lang if needed + root_path = tk.config.get('ckan.root_path', '') -+ root_path = re.sub('/{{LANG}}', '', root_path) ++ root_path = root_path.replace('/{{LANG}}', '') if lang() == tk.config.get('ckan.locale_default') else root_path.replace('/{{LANG}}', '/{}'.format(lang())) new_args = [] for arg in args: From b3ea1d2facd80407713789ac17c80efb489a1d94 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 28 Sep 2023 11:36:09 +0000 Subject: [PATCH 12/13] Improve Dockerfiles with hadolint --- ckan/Dockerfile | 47 +++++++++++++++++++++++--------------------- ckan/Dockerfile.dev | 38 ++++++++++++++++++++++++----------- ckan/Dockerfile.ghcr | 18 ++++++++++------- 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index da268584..66836b06 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -2,11 +2,14 @@ FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.9 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables -ENV APP_DIR=/srv/app \ - TZ=UTC +ENV APP_DIR=/srv/app +ENV TZ=UTC + +# Set working directory +WORKDIR ${APP_DIR} # requirements.txt files fixed until next releases -COPY req_fixes ${APP_DIR}/req_fixes +COPY req_fixes req_fixes # Extensions ### XLoader - 1.0.1 ### @@ -24,40 +27,40 @@ RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime ; fi && \ # Install CKAN extensions echo "ckan/ckanext-xloader" && \ - pip3 install -e git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader && \ - pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ - pip3 install -U requests[security] && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader && \ + pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ + pip3 install --no-cache-dir -U requests[security] && \ echo "ckan/ckanext-harvest" && \ - pip3 install -e git+https://github.com/ckan/ckanext-harvest.git@v1.5.1#egg=ckanext-harvest && \ - pip3 install -r ${APP_DIR}/src/ckanext-harvest/pip-requirements.txt && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-harvest.git@v1.5.1#egg=ckanext-harvest && \ + pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-harvest/pip-requirements.txt && \ echo "ckan/ckanext-geoview" && \ - pip3 install -e git+https://github.com/ckan/ckanext-geoview.git@v0.0.20#egg=ckanext-geoview && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-geoview.git@v0.0.20#egg=ckanext-geoview && \ echo "ckan/ckanext-spatial" && \ - pip3 install -e git+https://github.com/ckan/ckanext-spatial.git@v2.0.0#egg=ckanext-spatial && \ - pip3 install -r ${APP_DIR}/req_fixes/ckanext-spatial_requirements.txt && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-spatial.git@v2.0.0#egg=ckanext-spatial && \ + pip3 install --no-cache-dir -r ${APP_DIR}/req_fixes/ckanext-spatial_requirements.txt && \ echo "mjanez/ckanext-dcat (GeoDCAT-AP extended version)" && \ - pip3 install -e git+https://github.com/mjanez/ckanext-dcat.git@v1.2.0-geodcatap#egg=ckanext-dcat && \ - pip3 install -r ${APP_DIR}/src/ckanext-dcat/requirements.txt && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-dcat.git@v1.2.0-geodcatap#egg=ckanext-dcat && \ + pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-dcat/requirements.txt && \ echo "ckan/ckanext-scheming" && \ - pip3 install -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming && \ echo "mjanez/ckanext-resourcedictionary" && \ - pip3 install -e git+https://github.com/mjanez/ckanext-resourcedictionary.git@v1.0.1#egg=ckanext-resourcedictionary && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-resourcedictionary.git@v1.0.1#egg=ckanext-resourcedictionary && \ echo "ckan/ckanext-pages" && \ - pip3 install -e git+https://github.com/ckan/ckanext-pages.git@v0.5.2#egg=ckanext-pages && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-pages.git@v0.5.2#egg=ckanext-pages && \ echo "ckan/ckanext-pdfview" && \ - pip3 install -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.8#egg=ckanext-pdfview && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.8#egg=ckanext-pdfview && \ echo "mjanez/ckanext-scheming_dcat" && \ - pip3 install -e git+https://github.com/mjanez/ckanext-scheming_dcat.git@v2.0.0#egg=ckanext_scheming_dcat && \ - pip3 install -r https://raw.githubusercontent.com/mjanez/ckanext-scheming_dcat/v2.0.0/requirements.txt + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-scheming_dcat.git@v2.0.0#egg=ckanext_scheming_dcat && \ + pip3 install --no-cache-dir -r https://raw.githubusercontent.com/mjanez/ckanext-scheming_dcat/v2.0.0/requirements.txt # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc COPY docker-entrypoint.d/* /docker-entrypoint.d/ # Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ${APP_DIR}/ +COPY setup/who.ini ./ # Apply any patches needed to CKAN core -COPY patches ${APP_DIR}/patches +COPY patches patches RUN for d in $APP_DIR/patches/*; do \ if [ -d $d ]; then \ @@ -67,4 +70,4 @@ RUN for d in $APP_DIR/patches/*; do \ fi ; \ done -CMD $APP_DIR/start_ckan.sh +CMD ["/bin/sh", "-c", "$APP_DIR/start_ckan.sh"] \ No newline at end of file diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index fb723658..490ad8e4 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,9 +1,13 @@ FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.9-dev +LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables -ENV APP_DIR=/srv/app \ - TZ=UTC \ - SRC_EXTENSIONS_DIR=/srv/app/src_extensions +ENV APP_DIR=/srv/app +ENV TZ=UTC +ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions + +# Set working directory +WORKDIR ${APP_DIR} RUN echo ${TZ} > /etc/timezone && \ set -ex && apk --no-cache add sudo && \ @@ -47,23 +51,33 @@ RUN echo ${TZ} > /etc/timezone && \ # to get them mounted in this image at runtime # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc -COPY docker-entrypoint.d/* /docker-entrypoint.d/ +COPY docker-entrypoint.d/* docker-entrypoint.d/ # Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ${APP_DIR}/ +COPY setup/who.ini ./ # Override start_ckan.sh with DEV sh -COPY setup/start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh -RUN chmod +x ${APP_DIR}/start_ckan_development.sh +COPY setup/start_ckan_development.sh.override start_ckan_development.sh +RUN chmod +x start_ckan_development.sh # Apply any patches needed to CKAN core or any of the built extensions (not the # runtime mounted ones) -COPY patches ${APP_DIR}/patches +COPY patches patches + +# Apply any patches needed to CKAN core or any of the built extensions (not the +# runtime mounted ones!) +COPY patches patches RUN for d in $APP_DIR/patches/*; do \ if [ -d $d ]; then \ - for f in `ls $d/*.patch | sort -g`; do \ - cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \ - done ; \ + for f in `ls $d/*.patch | sort -g`; do \ + if [ -d $SRC_EXTENSIONS_DIR/`basename "$d"` ]; then \ + cd $SRC_EXTENSIONS_DIR/`basename "$d"` && \ + echo "$0: Applying patch $f to $SRC_EXTENSIONS_DIR/`basename $d`" && \ + patch -p1 < "$f" ; \ + else \ + echo "$0: Skipping patch $f because directory $SRC_EXTENSIONS_DIR/`basename $d` does not exist. Built the extension: `basename $d`" ; \ + fi \ + done ; \ fi ; \ - done \ No newline at end of file +done \ No newline at end of file diff --git a/ckan/Dockerfile.ghcr b/ckan/Dockerfile.ghcr index f0cad95c..899a2f11 100644 --- a/ckan/Dockerfile.ghcr +++ b/ckan/Dockerfile.ghcr @@ -1,23 +1,27 @@ FROM ghcr.io/mjanez/ckan-spatial:ckan-2.9.9 +LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables ENV APP_DIR=/srv/app ENV TZ=UTC -RUN echo ${TZ} > /etc/timezone + +# Set working directory +WORKDIR ${APP_DIR} # Make sure both files are not exactly the same -RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \ - cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\ +RUN echo ${TZ} > /etc/timezone && \ + if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \ + cp /usr/share/zoneinfo/${TZ} /etc/localtime ; \ fi ; # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc -COPY docker-entrypoint.d/* /docker-entrypoint.d/ +COPY docker-entrypoint.d/* docker-entrypoint.d/ # Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ${APP_DIR}/ +COPY setup/who.ini ./ # Apply any patches needed to CKAN core -COPY patches ${APP_DIR}/patches +COPY patches patches # Updated version of the Dockerfile RUN command that skips applying a patch if a reversed or previously applied patch is detected RUN for d in $APP_DIR/patches/*; do \ @@ -34,4 +38,4 @@ RUN for d in $APP_DIR/patches/*; do \ fi ; \ done -CMD $APP_DIR/start_ckan.sh +CMD ["/bin/sh", "-c", "$APP_DIR/start_ckan.sh"] \ No newline at end of file From b1ef0b736bbf92e2bca0b0fe1dc49385ce519299 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Fri, 29 Sep 2023 01:34:26 +0200 Subject: [PATCH 13/13] Update Dockerfile.dev --- ckan/Dockerfile.dev | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 490ad8e4..c87c3ad0 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -60,10 +60,6 @@ COPY setup/who.ini ./ COPY setup/start_ckan_development.sh.override start_ckan_development.sh RUN chmod +x start_ckan_development.sh -# Apply any patches needed to CKAN core or any of the built extensions (not the -# runtime mounted ones) -COPY patches patches - # Apply any patches needed to CKAN core or any of the built extensions (not the # runtime mounted ones!) COPY patches patches @@ -71,12 +67,12 @@ COPY patches patches RUN for d in $APP_DIR/patches/*; do \ if [ -d $d ]; then \ for f in `ls $d/*.patch | sort -g`; do \ - if [ -d $SRC_EXTENSIONS_DIR/`basename "$d"` ]; then \ - cd $SRC_EXTENSIONS_DIR/`basename "$d"` && \ - echo "$0: Applying patch $f to $SRC_EXTENSIONS_DIR/`basename $d`" && \ + if [ -d $SRC_DIR/`basename "$d"` ]; then \ + cd $SRC_DIR/`basename "$d"` && \ + echo "$0: Applying patch $f to $SRC_DIR/`basename $d`" && \ patch -p1 < "$f" ; \ else \ - echo "$0: Skipping patch $f because directory $SRC_EXTENSIONS_DIR/`basename $d` does not exist. Built the extension: `basename $d`" ; \ + echo "$0: Skipping patch $f because directory $SRC_DIR/`basename $d` does not exist. Built the extension: `basename $d`" ; \ fi \ done ; \ fi ; \