From 941b7f0411ff2036d881871497dcf00669e7d2d5 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 14 Apr 2022 07:20:42 +0000 Subject: [PATCH 1/5] Remove the get_latest_ref script and use the primary branch. We previously used the get_latest_ref.py script to fetch the latest commit sha for each project that passed CI correctly. Although, that has some challenges due to requiring Gitlab Auth token. It causes issues with building PRs and we don't want to share the Gitlab token with PR authors. Instead, this commit removes the code and sha references and Instead simply uses the primary master branch from each project to build the rolling release container image. --- README.md | 16 ++-------------- build.sh | 30 ----------------------------- core/Dockerfile.dev | 4 ++-- get_latest_ref.py | 41 ---------------------------------------- postorius/Dockerfile.dev | 4 ++-- web/Dockerfile.dev | 6 +++--- 6 files changed, 9 insertions(+), 92 deletions(-) delete mode 100755 get_latest_ref.py diff --git a/README.md b/README.md index 133ebf37..d67df988 100644 --- a/README.md +++ b/README.md @@ -89,36 +89,24 @@ $ docker pull docker.io/maxking/mailman-web:rolling $ docker pull docker.io/maxking/mailman-core:rolling ``` -Rolling releases are built with every commit and also re-generated weekly. You +Rolling releases are built with every commit and also re-generated nightly. You can inspect the images to get which commit it was built using: ```bash $ docker inspect --format '{{json .Config.Labels }}' mailman-core | python -m json.tool { - "version.core": "31f434d0", "version.git_commit": "45a4d7805b2b3d0e7c51679f59682d64ba02f05f", - "version.mm3-hk": "c625bfd2" } $ docker inspect --format '{{json .Config.Labels }}' mailman-web | python -m json.tool { - "version.client": "d9e9cb73", - "version.dj-mm3": "72a7d6c4", "version.git_commit": "45a4d7805b2b3d0e7c51679f59682d64ba02f05f", - "version.hyperkitty": "b67ca8a8", - "version.postorius": "73328ad4" } ``` - `version.git_commit` : This is the commit hash of the Dockerfile in the [Github repo](https://github.com/maxking/docker-mailman) -- `version.core`: The commit hash of Mailman Core -- `version.mm3-hk`: The commit hash of Mailman3-hyperkitty plugin. -- `version.client`: The commit hash of Mailman Client. -- `version.hyperkitty`: The commit hash of Hyperkitty. -- `version.postorius`: The commit hash of Postorius. -- `version.dj-mm3`: The commit hash of Django-Mailman3 project. ## Dependencies @@ -414,7 +402,7 @@ smtp_port: $SMTP_PORT configuration: /etc/postfix-mailman.cfg ``` -So, if you need to update the values, you can set `SMTP_HOST`, `SMTP_PORT`, +So, if you need to update the values, you can set `SMTP_HOST`, `SMTP_PORT`, `MM_HOSTNAME` environment variables in `mailman-core` container. Please verify the output for `[mta]` section to ensure that it points to diff --git a/build.sh b/build.sh index 370d6210..9d8fe7a2 100755 --- a/build.sh +++ b/build.sh @@ -12,49 +12,19 @@ DOCKER=docker if [ "$BUILD_ROLLING" = "yes" ]; then echo "Building rolling releases..." - python3 -m venv venv - source venv/bin/activate - pip install python-gitlab - # Get the latest commit for repositories and set their reference values to be - # used in the development builds. - CORE_REF=$(python get_latest_ref.py mailman/mailman) - CLIENT_REF=$(python get_latest_ref.py mailman/mailmanclient) - POSTORIUS_REF=$(python get_latest_ref.py mailman/postorius) - HYPERKITTY_REF=$(python get_latest_ref.py mailman/hyperkitty) - DJ_MM3_REF=$(python get_latest_ref.py mailman/django-mailman3) - MM3_HK_REF=$(python get_latest_ref.py mailman/mailman-hyperkitty) - # Build the mailman-core image. $DOCKER build -f core/Dockerfile.dev \ - --build-arg CORE_REF=$CORE_REF \ - --build-arg MM3_HK_REF=$MM3_HK_REF \ - --label version.core="$CORE_REF" \ - --label version.mm3-hk="$MM3_HK_REF" \ --label version.git_commit="$COMMIT_ID" \ -t maxking/mailman-core:rolling core/ # Build the mailman-web image. $DOCKER build -f web/Dockerfile.dev \ --label version.git_commit="$COMMIT_ID" \ - --label version.postorius="$POSTORIUS_REF" \ - --label version.hyperkitty="$HYPERKITTY_REF" \ - --label version.client="$CLIENT_REF" \ - --label version.dj-mm3="$DJ_MM3_REF" \ - --build-arg POSTORIUS_REF=$POSTORIUS_REF \ - --build-arg CLIENT_REF=$CLIENT_REF \ - --build-arg HYPERKITTY_REF=$HYPERKITTY_REF \ - --build-arg DJ_MM3_REF=$DJ_MM3_REF \ -t maxking/mailman-web:rolling web/ # build the postorius image. $DOCKER build -f postorius/Dockerfile.dev\ --label version.git_commit="$COMMIT_ID"\ - --label version.postorius="$POSTORIUS_REF" \ - --label version.client="$CLIENT_REF" \ - --label version.dj-mm3="$DJ_MM3_REF" \ - --build-arg POSTORIUS_REF=$POSTORIUS_REF \ - --build-arg CLIENT_REF=$CLIENT_REF \ - --build-arg DJ_MM3_REF=$DJ_MM3_REF \ -t maxking/postorius:rolling postorius/ else echo "Building stable releases..." diff --git a/core/Dockerfile.dev b/core/Dockerfile.dev index 384b1705..bb0c0b83 100644 --- a/core/Dockerfile.dev +++ b/core/Dockerfile.dev @@ -18,8 +18,8 @@ RUN --mount=type=cache,target=/root/.cache \ && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c \ && python3 -m pip install -U psycopg2 pymysql setuptools wheel \ && python3 -m pip install \ - git+https://gitlab.com/mailman/mailman@${CORE_REF} \ - git+https://gitlab.com/mailman/mailman-hyperkitty@${MM3_HK_REF} \ + git+https://gitlab.com/mailman/mailman \ + git+https://gitlab.com/mailman/mailman-hyperkitty \ gunicorn==19.9.0 \ && apk del build-deps \ && adduser -S mailman diff --git a/get_latest_ref.py b/get_latest_ref.py deleted file mode 100755 index a761d7e5..00000000 --- a/get_latest_ref.py +++ /dev/null @@ -1,41 +0,0 @@ -#! /usr/bin/env python -import os -import sys -import gitlab - - -def usage(): - print("{} ") - - -def main(): - if 2 > len(sys.argv) > 3: - usage() - - project_name = sys.argv[1] - if len(sys.argv) > 2: - branch_name = sys.argv[2] - else: - branch_name = 'master' - - gl_token = os.getenv('GITLAB_TOKEN') - if gl_token is None: - print('GITLAB_TOKEN not set!') - exit(1) - gl = gitlab.Gitlab('https://gitlab.com/', gl_token) - - project = gl.projects.get(project_name) - - # Find the last commit in the branch that passed the CI - # successfully and return the reference to it. - for commit in project.commits.list(ref=branch_name): - stasues = list(status.status == 'success' for status in - commit.statuses.list() if status.allow_failure == False) - if len(stasues) == 0: - continue - if all(stasues): - print(commit.short_id) - break - -if __name__ == '__main__': - main() diff --git a/postorius/Dockerfile.dev b/postorius/Dockerfile.dev index 622ccf0f..a5d00410 100644 --- a/postorius/Dockerfile.dev +++ b/postorius/Dockerfile.dev @@ -22,8 +22,8 @@ RUN --mount=type=cache,target=/root/.cache \ python3 py3-pip libffi gettext py-cryptography \ && python3 -m pip install -U pip setuptools wheel \ && python3 -m pip install -U \ - git+https://gitlab.com/mailman/mailmanclient@${CLIENT_REF} \ - git+https://gitlab.com/mailman/postorius@${POSTORIUS_REF} \ + git+https://gitlab.com/mailman/mailmanclient \ + git+https://gitlab.com/mailman/postorius \ uwsgi \ 'psycopg2<2.9' \ dj-database-url \ diff --git a/web/Dockerfile.dev b/web/Dockerfile.dev index 648c980c..6d226885 100644 --- a/web/Dockerfile.dev +++ b/web/Dockerfile.dev @@ -23,9 +23,9 @@ RUN --mount=type=cache,target=/root/.cache \ curl mailcap xapian-core xapian-bindings-python3 libffi gettext py-cryptography \ && python3 -m pip install -U pip setuptools wheel \ && python3 -m pip install -U \ - git+https://gitlab.com/mailman/mailmanclient@${CLIENT_REF} \ - git+https://gitlab.com/mailman/postorius@${POSTORIUS_REF} \ - git+https://gitlab.com/mailman/hyperkitty@${HYPERKITTY_REF} \ + git+https://gitlab.com/mailman/mailmanclient \ + git+https://gitlab.com/mailman/postorius \ + git+https://gitlab.com/mailman/hyperkitty \ whoosh \ uwsgi \ 'psycopg2<2.9' \ From a4906c680fd5168b4a4dd3e135665599b6697de5 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 14 Apr 2022 07:38:07 +0000 Subject: [PATCH 2/5] Update names of jobs --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4078009f..a1816a28 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,15 +45,15 @@ jobs: workflows: version: 2 - build-and-deploy-workflow: + test-stable: jobs: - - build: + - stable-build: rolling: "no" filters: tags: only: /^v\d+\.\d+\.\d+$/ - cron-workflow: + cron-builds: triggers: - schedule: cron: "0 0 * * *" @@ -61,13 +61,13 @@ workflows: branches: only: main jobs: - - build: + - cron-build: rolling: "yes" context: org-global - rolling-workflow: + rolling-builds: jobs: - - build: + - rolling-build: rolling: "yes" context: org-global filters: From c69b5c5d42bd686edadd8f9b9bfc1b62158bb466 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 14 Apr 2022 07:40:05 +0000 Subject: [PATCH 3/5] Fix some missed our references --- postorius/Dockerfile.dev | 2 +- web/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/postorius/Dockerfile.dev b/postorius/Dockerfile.dev index a5d00410..e427a1c4 100644 --- a/postorius/Dockerfile.dev +++ b/postorius/Dockerfile.dev @@ -32,7 +32,7 @@ RUN --mount=type=cache,target=/root/.cache \ django-utils-six \ && python3 -m pip install -U 'Django<3.2' \ && python3 -m pip install -U \ - git+https://gitlab.com/mailman/django-mailman3@${DJ_MM3_REF} \ + git+https://gitlab.com/mailman/django-mailman3 \ && apk del .build-deps \ && addgroup -S mailman \ && adduser -S -G mailman mailman \ diff --git a/web/Dockerfile.dev b/web/Dockerfile.dev index 6d226885..9b2809ce 100644 --- a/web/Dockerfile.dev +++ b/web/Dockerfile.dev @@ -38,7 +38,7 @@ RUN --mount=type=cache,target=/root/.cache \ django-utils-six \ && python3 -m pip install -U 'Django<3.2' \ && python3 -m pip install -U \ - git+https://gitlab.com/mailman/django-mailman3@${DJ_MM3_REF} \ + git+https://gitlab.com/mailman/django-mailman3 \ && apk del .build-deps \ && addgroup -S mailman \ && adduser -S -G mailman mailman \ From 02d9203a8f415283f5c19933221294b9815ad1f7 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 14 Apr 2022 07:42:12 +0000 Subject: [PATCH 4/5] Revert to using the right job names --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1816a28..72992630 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ workflows: version: 2 test-stable: jobs: - - stable-build: + - build: rolling: "no" filters: tags: @@ -61,13 +61,13 @@ workflows: branches: only: main jobs: - - cron-build: + - build: rolling: "yes" context: org-global rolling-builds: jobs: - - rolling-build: + - build: rolling: "yes" context: org-global filters: From 0b470ba8fb716937b81f8c3314c05e1694cc9ece Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 14 Apr 2022 07:43:03 +0000 Subject: [PATCH 5/5] Some more workflow renames --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 72992630..c2aaaa56 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ workflows: rolling: "yes" context: org-global - rolling-builds: + test-rolling: jobs: - build: rolling: "yes"