From 7b14b36ddd70e0fc3bede51f24be290d02016fe8 Mon Sep 17 00:00:00 2001 From: Andreas Heinrich Date: Mon, 8 Jul 2024 11:18:28 +0200 Subject: [PATCH] Refactor docker images and workflows to build and deploy them * Remove workflow `deploy-build-kit.yaml`. It is integrated into `deploy-docker-images.yml`. * Refactor deploy-docker-images.yml workflow * Convert matrix into multiple jobs, one for each docker image, because the images depend on each other, and the matrix strategy does not support this. * Add `env-setup` job to set up the environment for the other jobs, because reusable workflows don't support environment variables. * Add input parameter to build deprecated images. Currently, the deprecated images are built by default, if not specified otherwise. * Refactor build-single-docker-image.yml * Rename input `docker_directory` to `directory`. Instead of setting the directory that contains all images, it is set now to the directory that contains the Dockerfile for the image to build. * Add input parameter `docker_file_name` to specify the Dockerfile to use. * Add input parameter `platforms` to specify the platforms to build the image for. The default is `linux/amd64,linux/arm64` and `linux/arm/v7`. * The name of the deployed image is now set by the input parameter `image_name`: `ghcr.io/everest/`. The directory name is independent of the image name. * Update README.md * Move `ghcr.io/everest/everest-clan-format` to `ghcr.io/everest/everest-ci/everest-clang-format`. The old image is deprecated, but still available and deployed. * Merge `ghcr.io/everest/build-kit-alpine` and `ghcr.io/everest/build-kit-debian` into `ghcr.io/everest/everest-ci/build-kit`. The old images are deprecated, but still available and deployed. * Add `ghcr.io/everest/everest-ci/run-env-base` image * Add `ghcr.io/everest/everest-ci/build-env-base` image * Add `ghcr.io/everest/everest-ci/dev-env-base` image Signed-off-by: Andreas Heinrich --- .github/workflows/deploy-docker-images.yml | 174 ++++++- .../workflows/deploy-single-docker-image.yml | 43 +- README.md | 54 ++- .../deprecated-images/build-kit/.dockerignore | 1 + .../build-kit/alpine.Dockerfile | 9 +- .../build-kit/debian.Dockerfile | 9 +- .../deprecated-images/build-kit/entrypoint.sh | 44 ++ .../build-kit/maven-settings.xml | 4 + .../everest-clang-format/Dockerfile | 15 + .../everest-clang-format/run-clang-format.py | 434 ++++++++++++++++++ docker/images/build-env-base/Dockerfile | 77 ++++ docker/images/build-kit/Dockerfile | 23 + docker/images/dev-env-base/Dockerfile | 40 ++ docker/images/run-env-base/Dockerfile | 20 + 14 files changed, 913 insertions(+), 34 deletions(-) create mode 100644 docker/deprecated-images/build-kit/.dockerignore rename docker/{images => deprecated-images}/build-kit/alpine.Dockerfile (91%) rename docker/{images => deprecated-images}/build-kit/debian.Dockerfile (90%) create mode 100755 docker/deprecated-images/build-kit/entrypoint.sh create mode 100644 docker/deprecated-images/build-kit/maven-settings.xml create mode 100644 docker/deprecated-images/everest-clang-format/Dockerfile create mode 100755 docker/deprecated-images/everest-clang-format/run-clang-format.py create mode 100644 docker/images/build-env-base/Dockerfile create mode 100644 docker/images/build-kit/Dockerfile create mode 100644 docker/images/dev-env-base/Dockerfile create mode 100644 docker/images/run-env-base/Dockerfile diff --git a/.github/workflows/deploy-docker-images.yml b/.github/workflows/deploy-docker-images.yml index 21e124c..64d4891 100644 --- a/.github/workflows/deploy-docker-images.yml +++ b/.github/workflows/deploy-docker-images.yml @@ -7,6 +7,10 @@ on: description: 'Force rebuild of all images' default: false type: boolean + build_deprecated_images: + description: 'Build deprecated images' + default: false + type: boolean push: branches: - '**' @@ -15,21 +19,171 @@ on: env: REGISTRY: ghcr.io + DOCKER_DIRECTORY: docker/images/ + PLATFORMS: | + linux/amd64 + # linux/arm64 + # linux/arm/v7 jobs: - build-and-push-all-images: - name: Build and push all docker images - strategy: - matrix: - image_name: [everest-clang-format] - uses: everest/everest-ci/.github/workflows/deploy-single-docker-image.yml@v1.1.0 + env-setup: + # Since env variables can't be passed to reusable workflows, we need to pass them as outputs + name: Evaluate force rebuild and set env variables as outputs + runs-on: ubuntu-22.04 + outputs: + force_rebuild: ${{ steps.check.outputs.force_rebuild }} + docker_registry: ${{ steps.check.outputs.docker_registry }} + docker_directory: ${{ steps.check.outputs.docker_directory }} + platforms: ${{ steps.check.outputs.platforms }} + repository_name: ${{ steps.check.outputs.repository_name }} + build_deprecated_images: ${{ github.event.inputs.build_deprecated_images }} + steps: + - id: check + run: | + echo "force_rebuild=${{ inputs.force_rebuild || (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/' )) || false }}" >> $GITHUB_OUTPUT + echo "docker_registry=${{ env.REGISTRY }}" >> $GITHUB_OUTPUT + echo "docker_directory=${{ env.DOCKER_DIRECTORY }}" >> $GITHUB_OUTPUT + echo "platforms=${{ env.PLATFORMS }}" >> $GITHUB_OUTPUT + echo "repository_name=${{ github.event.repository.name }}" >> $GITHUB_OUTPUT + # Build deprecated images if not explicitly disabled + #TODO: set default to false, once backwards compatibility is no longer needed + echo "build_deprecated_images=${{ inputs.build_deprecated_images || true }}" >> $GITHUB_OUTPUT + + # One job for each image, since the images build on top of each other a matrix strategy is not possible + everest-clang-format: + needs: + - env-setup + name: Build and push everest-clang-format docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/everest-clang-format + directory: ${{ needs.env-setup.outputs.docker_directory }}/everest-clang-format + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + run-env-base: + needs: + - env-setup + name: Build and push run-env-base docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} with: - force_rebuild: ${{ inputs.force_rebuild || (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/' )) || false }} - image_name: ${{ matrix.image_name }} - docker_directory: docker/images/ - docker_registry: ghcr.io + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/run-env-base + directory: ${{ needs.env-setup.outputs.docker_directory }}/run-env-base + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} github_ref_before: ${{ github.event.before }} github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + build-env-base: + needs: + - env-setup + - run-env-base + name: Build and push build-env-base docker image + uses: ./.github/workflows/deploy-single-docker-image.yml secrets: SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/build-env-base + directory: ${{ needs.env-setup.outputs.docker_directory }}/build-env-base + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + dev-env-base: + needs: + - env-setup + - build-env-base + name: Build and push dev-env-base docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/dev-env-base + directory: ${{ needs.env-setup.outputs.docker_directory }}/dev-env-base + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + build-kit: + needs: + - env-setup + - build-env-base + name: Build and push build-kit docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/build-kit + directory: ${{ needs.env-setup.outputs.docker_directory }}/build-kit + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + # Include deprecated images for backwards compatibility + deprecated-everest-clang-format: + if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }} + needs: + - env-setup + name: Build and push deprecated everest-clang-format docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: everest-clang-format + directory: docker/deprecated-images/everest-clang-format + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + deprecated-build-kit-alpine: + if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }} + needs: + - env-setup + name: Build and push deprecated build-kit-alpine docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: build-kit-alpine + directory: docker/deprecated-images/build-kit + docker_file_name: alpine.Dockerfile + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + deprecated-build-kit-debian: + if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }} + needs: + - env-setup + name: Build and push deprecated build-kit-debian docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: build-kit-debian + directory: docker/deprecated-images/build-kit + docker_file_name: debian.Dockerfile + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} diff --git a/.github/workflows/deploy-single-docker-image.yml b/.github/workflows/deploy-single-docker-image.yml index a1f36cd..786a73c 100644 --- a/.github/workflows/deploy-single-docker-image.yml +++ b/.github/workflows/deploy-single-docker-image.yml @@ -7,11 +7,15 @@ on: description: 'Name of the image to build and push' required: true type: string - docker_directory: - description: 'Directory containing the docker directory' + directory: + description: 'Directory containing the Dockerfile' required: true type: string - default: docker/ + docker_file_name: + description: 'Name of the Dockerfile' + required: false + default: 'Dockerfile' + type: string docker_registry: description: 'Docker registry to push to' required: true @@ -29,6 +33,13 @@ on: description: 'Forces rebuild of docker image, even if no changes are detected' default: false type: boolean + platforms: + description: 'Platforms to build for' + default: | + linux/amd64 + linux/arm64 + linux/arm/v7 + type: string secrets: SA_GITHUB_PAT: description: 'Github PAT with access to the repository' @@ -55,7 +66,7 @@ jobs: - name: Get changed files id: changed-files run: | - echo "changed_files=$(git diff --name-only ${{ inputs.github_ref_before }} ${{ inputs.github_ref_after }} | grep "^${{inputs.docker_directory}}/${{inputs.image_name}}" | wc -l)" >> $GITHUB_OUTPUT + echo "changed_files=$(git diff --name-only ${{ inputs.github_ref_before }} ${{ inputs.github_ref_after }} | grep "^${{inputs.directory}}" | wc -l)" >> $GITHUB_OUTPUT working-directory: source build-and-push: @@ -75,19 +86,28 @@ jobs: - name: Get context / Path of Dockerfile id: get-context run: | - if [ -f source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/Dockerfile ]; then - echo "::set-output name=path::source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/" - elif [ -f source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/.devcontainer/Dockerfile ]; then - echo "::set-output name=path::source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/.devcontainer/" + if [ -f source/${{ inputs.directory }}/${{ inputs.docker_file_name }} ]; then + echo "path=source/${{ inputs.directory }}" >> $GITHUB_OUTPUT + elif [ -f source/${{ inputs.directory }}/.devcontainer/${{ inputs.docker_file_name }} ]; then + echo "path=source/${{ inputs.directory }}/.devcontainer" >> $GITHUB_OUTPUT else - echo "No Dockerfile found for image ${{ inputs.image_name }}!" + echo "No Dockerfile found for image ${{ inputs.image_name }} in dokcer_directory ${{ inputs.directory }} with docker_file_name ${{ inputs.docker_file_name }}!" exit 1 fi + - name: Parse platforms + id: parse-platforms + run: | + # remove all empty lines + result=$(echo "${{ inputs.platforms }}" | sed '/^$/d') + # replace newlines with commas, remove trailing comma + result=$(echo "${result}" | tr '\n' ',' | sed 's/,$//') + echo "platforms=${result}" >> $GITHUB_OUTPUT - name: Extract metadata id: meta uses: docker/metadata-action@v3 with: - images: ${{ inputs.docker_registry }}/${{ github.repository_owner }}/${{ inputs.image_name }} + images: | + ${{ inputs.docker_registry }}/${{ github.repository_owner }}/${{ inputs.image_name }} - name: Set up QEMU uses: docker/setup-qemu-action@v1 with: @@ -105,9 +125,10 @@ jobs: uses: docker/build-push-action@v3 with: context: ${{ steps.get-context.outputs.path }} + file: ${{ steps.get-context.outputs.path }}/${{ inputs.docker_file_name }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: ${{ steps.parse-platforms.outputs.platforms }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/README.md b/README.md index be2480a..02b3421 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,56 @@ -# everest ci/cd +# EVerest CI/CD This repository should share all common used functionality for the everest github ci/cd pipe + +## Docker Images + +### run-env-base + +Based on `debian:12-slim`. + +Deployed as `ghcr.io/everest/everest-ci/run-env-base`. + +Includes all dependencies that are necessary to run EVerest. + +### build-env-base + +Based on `run-env-base`. + +Deployed as `ghcr.io/everest/everest-ci/build-env-base`. + +Includes all dependencies that are necessary to build EVerest. + +### dev-env-base + +Based on `build-env-base`. + +Deployed as `ghcr.io/everest/everest-ci/dev-env-base`. + +Include additional packages to develop EVerest. + +### everest-clang-format + +> [!NOTE] +> This image was moved from `ghcr.io/everest/clang-format` to +> `ghcr.io/everest/everest-ci/everest-clang-format`. The old image is +> deprecated and will be removed in the future. + +Deployed as `ghcr.io/everest/everest-ci/everest-clang-format`. + +Contains a fixed clang-format version in addition to a run-clang-format script. This image can be diretcly executed. +This image is used as base for a custom github action and can also be used to run clang-format locally. + +### build-kit + +> [!NOTE] +> The images `ghcr.io/everest/build-kit-alpine` and `ghcr.io/everest/build-kit-debian` +> are deprecated and will be removed in the future. Please use +> `ghcr.io/everest/everest-ci/> build-kit` instead. + +Based on `build-env-base`. + +Deployed as `ghcr.io/everest/everest-ci/build-kit`. + +This image includes a mechanic to run bash scripts in the build environment. It is used to run the build scripts in the CI/CD pipeline. +It is executed directly. diff --git a/docker/deprecated-images/build-kit/.dockerignore b/docker/deprecated-images/build-kit/.dockerignore new file mode 100644 index 0000000..c3f982e --- /dev/null +++ b/docker/deprecated-images/build-kit/.dockerignore @@ -0,0 +1 @@ +doc.rst diff --git a/docker/images/build-kit/alpine.Dockerfile b/docker/deprecated-images/build-kit/alpine.Dockerfile similarity index 91% rename from docker/images/build-kit/alpine.Dockerfile rename to docker/deprecated-images/build-kit/alpine.Dockerfile index 03a38f8..a9dc399 100644 --- a/docker/images/build-kit/alpine.Dockerfile +++ b/docker/deprecated-images/build-kit/alpine.Dockerfile @@ -70,11 +70,8 @@ RUN python3 -m pip install \ py4j>=0.10.9.5 \ netifaces>=0.11.0 \ python-dateutil>=2.8.2 \ - gcovr==5.0 - -# install ev-cli -ARG EVEREST_UTILS_VERSION=v0.2.3 -RUN python3 -m pip install git+https://github.com/EVerest/everest-utils@${EVEREST_UTILS_VERSION}#subdirectory=ev-dev-tools + gcovr==5.0 \ + build # install edm RUN python3 -m pip install git+https://github.com/EVerest/everest-dev-environment@v0.5.5#subdirectory=dependency_manager @@ -84,7 +81,7 @@ RUN git clone https://github.com/EVerest/everest-cmake.git $EVEREST_CMAKE_PATH RUN ( \ cd $EVEREST_CMAKE_PATH \ - git checkout 329f8db \ + git checkout v0.4.0 \ rm -r .git \ ) diff --git a/docker/images/build-kit/debian.Dockerfile b/docker/deprecated-images/build-kit/debian.Dockerfile similarity index 90% rename from docker/images/build-kit/debian.Dockerfile rename to docker/deprecated-images/build-kit/debian.Dockerfile index ccee54d..2a0287c 100644 --- a/docker/images/build-kit/debian.Dockerfile +++ b/docker/deprecated-images/build-kit/debian.Dockerfile @@ -61,11 +61,8 @@ RUN python3 -m pip install \ py4j>=0.10.9.5 \ netifaces>=0.11.0 \ python-dateutil>=2.8.2 \ - gcovr==5.0 - -# install ev-cli -ARG EVEREST_UTILS_VERSION=v0.2.3 -RUN python3 -m pip install git+https://github.com/EVerest/everest-utils@${EVEREST_UTILS_VERSION}#subdirectory=ev-dev-tools + gcovr==5.0 \ + build # install edm RUN python3 -m pip install git+https://github.com/EVerest/everest-dev-environment@v0.5.5#subdirectory=dependency_manager @@ -75,7 +72,7 @@ RUN git clone https://github.com/EVerest/everest-cmake.git $EVEREST_CMAKE_PATH RUN ( \ cd $EVEREST_CMAKE_PATH \ - git checkout 329f8db \ + git checkout v0.4.0 \ rm -r .git \ ) diff --git a/docker/deprecated-images/build-kit/entrypoint.sh b/docker/deprecated-images/build-kit/entrypoint.sh new file mode 100755 index 0000000..440b811 --- /dev/null +++ b/docker/deprecated-images/build-kit/entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +setup_cache(){ + export CPM_SOURCE_CACHE=$EXT_MOUNT/cache/cpm + export CCACHE_DIR=$EXT_MOUNT/cache/ccache + + maven_cache_dir=$EXT_MOUNT/cache/m2-repo + + # we copy the settings file each time in case the $HOME folder changes + mkdir -p $HOME/.m2 || true + cp $ASSETS_PATH/maven-settings.xml $HOME/.m2/settings.xml + sed -i 's@MAVEN_CACHE_REPOSITORY@'"$maven_cache_dir"'@' $HOME/.m2/settings.xml +} + +# if first cmd is exec we're going to exec what follows +# otherwise check for script +run_script(){ + + if [ "$1" = 'run-script' ]; then + + if [ -z "$2" ]; then + echo "Error: missing script name for run-script mode" + exit 1 + fi + + init_script=$EXT_MOUNT/scripts/$2.sh + + if [ ! -f "$init_script" ]; then + echo "Error: no script found at $init_script" + # bash convention, 127 = not found + exit 127 + fi + + exec $init_script + fi + + # not the run-script mode + exec "$@" + +} + +setup_cache +run_script $@ + diff --git a/docker/deprecated-images/build-kit/maven-settings.xml b/docker/deprecated-images/build-kit/maven-settings.xml new file mode 100644 index 0000000..317aac6 --- /dev/null +++ b/docker/deprecated-images/build-kit/maven-settings.xml @@ -0,0 +1,4 @@ + + MAVEN_CACHE_REPOSITORY + diff --git a/docker/deprecated-images/everest-clang-format/Dockerfile b/docker/deprecated-images/everest-clang-format/Dockerfile new file mode 100644 index 0000000..4e78acd --- /dev/null +++ b/docker/deprecated-images/everest-clang-format/Dockerfile @@ -0,0 +1,15 @@ +ARG BASE_IMAGE=debian:12 +FROM ${BASE_IMAGE} + +RUN apt update && apt upgrade -y +RUN apt update && \ + apt install -y \ + clang-format-15=1:15.0.6* +RUN ln -s /usr/bin/clang-format-15 /usr/bin/clang-format + +RUN apt install -y \ + python-is-python3 + +COPY run-clang-format.py /usr/bin/run-clang-format + +ENTRYPOINT ["/usr/bin/run-clang-format"] diff --git a/docker/deprecated-images/everest-clang-format/run-clang-format.py b/docker/deprecated-images/everest-clang-format/run-clang-format.py new file mode 100755 index 0000000..3f935d6 --- /dev/null +++ b/docker/deprecated-images/everest-clang-format/run-clang-format.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python + +# See original source at https://github.com/Sarcasm/run-clang-format/blob/39081c9c42768ab5e8321127a7494ad1647c6a2f/run-clang-format.py +# +# MIT License +# +# Copyright (c) 2017 Guillaume Papin +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +"""A wrapper script around clang-format, suitable for linting multiple files +and to use for continuous integration. + +This is an alternative API for the clang-format command line. +It runs over multiple files and directories in parallel. +A diff output is produced and a sensible exit code is returned. + +""" + +from __future__ import print_function, unicode_literals + +import argparse +import codecs +import difflib +import fnmatch +import io +import errno +import multiprocessing +import os +import signal +import subprocess +import sys +import traceback + +from functools import partial + +try: + from subprocess import DEVNULL # py3k +except ImportError: + DEVNULL = open(os.devnull, "wb") + + +DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx' +DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore' + + +class ExitStatus: + SUCCESS = 0 + DIFF = 1 + TROUBLE = 2 + +def excludes_from_file(ignore_file): + excludes = [] + try: + with io.open(ignore_file, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('#'): + # ignore comments + continue + pattern = line.rstrip() + if not pattern: + # allow empty lines + continue + excludes.append(pattern) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + return excludes; + +def list_files(files, recursive=False, extensions=None, exclude=None): + if extensions is None: + extensions = [] + if exclude is None: + exclude = [] + + out = [] + for file in files: + if recursive and os.path.isdir(file): + for dirpath, dnames, fnames in os.walk(file): + fpaths = [os.path.join(dirpath, fname) for fname in fnames] + for pattern in exclude: + # os.walk() supports trimming down the dnames list + # by modifying it in-place, + # to avoid unnecessary directory listings. + dnames[:] = [ + x for x in dnames + if + not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) + ] + fpaths = [ + x for x in fpaths if not fnmatch.fnmatch(x, pattern) + ] + for f in fpaths: + ext = os.path.splitext(f)[1][1:] + if ext in extensions: + out.append(f) + else: + out.append(file) + return out + + +def make_diff(file, original, reformatted): + return list( + difflib.unified_diff( + original, + reformatted, + fromfile='{}\t(original)'.format(file), + tofile='{}\t(reformatted)'.format(file), + n=3)) + + +class DiffError(Exception): + def __init__(self, message, errs=None): + super(DiffError, self).__init__(message) + self.errs = errs or [] + + +class UnexpectedError(Exception): + def __init__(self, message, exc=None): + super(UnexpectedError, self).__init__(message) + self.formatted_traceback = traceback.format_exc() + self.exc = exc + + +def run_clang_format_diff_wrapper(args, file): + try: + ret = run_clang_format_diff(args, file) + return ret + except DiffError: + raise + except Exception as e: + raise UnexpectedError('{}: {}: {}'.format(file, e.__class__.__name__, + e), e) + + +def run_clang_format_diff(args, file): + try: + with io.open(file, 'r', encoding='utf-8') as f: + original = f.readlines() + except IOError as exc: + raise DiffError(str(exc)) + + if args.in_place: + invocation = [args.clang_format_executable, '-i', file] + else: + invocation = [args.clang_format_executable, file] + + if args.style: + invocation.extend(['--style', args.style]) + + if args.dry_run: + print(" ".join(invocation)) + return [], [] + + # Use of utf-8 to decode the process output. + # + # Hopefully, this is the correct thing to do. + # + # It's done due to the following assumptions (which may be incorrect): + # - clang-format will returns the bytes read from the files as-is, + # without conversion, and it is already assumed that the files use utf-8. + # - if the diagnostics were internationalized, they would use utf-8: + # > Adding Translations to Clang + # > + # > Not possible yet! + # > Diagnostic strings should be written in UTF-8, + # > the client can translate to the relevant code page if needed. + # > Each translation completely replaces the format string + # > for the diagnostic. + # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation + # + # It's not pretty, due to Python 2 & 3 compatibility. + encoding_py3 = {} + if sys.version_info[0] >= 3: + encoding_py3['encoding'] = 'utf-8' + + try: + proc = subprocess.Popen( + invocation, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + **encoding_py3) + except OSError as exc: + raise DiffError( + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(invocation), exc + ) + ) + proc_stdout = proc.stdout + proc_stderr = proc.stderr + if sys.version_info[0] < 3: + # make the pipes compatible with Python 3, + # reading lines should output unicode + encoding = 'utf-8' + proc_stdout = codecs.getreader(encoding)(proc_stdout) + proc_stderr = codecs.getreader(encoding)(proc_stderr) + # hopefully the stderr pipe won't get full and block the process + outs = list(proc_stdout.readlines()) + errs = list(proc_stderr.readlines()) + proc.wait() + if proc.returncode: + raise DiffError( + "Command '{}' returned non-zero exit status {}".format( + subprocess.list2cmdline(invocation), proc.returncode + ), + errs, + ) + if args.in_place: + return [], errs + return make_diff(file, original, outs), errs + + +def bold_red(s): + return '\x1b[1m\x1b[31m' + s + '\x1b[0m' + + +def colorize(diff_lines): + def bold(s): + return '\x1b[1m' + s + '\x1b[0m' + + def cyan(s): + return '\x1b[36m' + s + '\x1b[0m' + + def green(s): + return '\x1b[32m' + s + '\x1b[0m' + + def red(s): + return '\x1b[31m' + s + '\x1b[0m' + + for line in diff_lines: + if line[:4] in ['--- ', '+++ ']: + yield bold(line) + elif line.startswith('@@ '): + yield cyan(line) + elif line.startswith('+'): + yield green(line) + elif line.startswith('-'): + yield red(line) + else: + yield line + + +def print_diff(diff_lines, use_color): + if use_color: + diff_lines = colorize(diff_lines) + if sys.version_info[0] < 3: + sys.stdout.writelines((l.encode('utf-8') for l in diff_lines)) + else: + sys.stdout.writelines(diff_lines) + + +def print_trouble(prog, message, use_colors): + error_text = 'error:' + if use_colors: + error_text = bold_red(error_text) + print("{}: {} {}".format(prog, error_text, message), file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--clang-format-executable', + metavar='EXECUTABLE', + help='path to the clang-format executable', + default='clang-format') + parser.add_argument( + '--extensions', + help='comma separated list of file extensions (default: {})'.format( + DEFAULT_EXTENSIONS), + default=DEFAULT_EXTENSIONS) + parser.add_argument( + '-r', + '--recursive', + action='store_true', + help='run recursively over directories') + parser.add_argument( + '-d', + '--dry-run', + action='store_true', + help='just print the list of files') + parser.add_argument( + '-i', + '--in-place', + action='store_true', + help='format file instead of printing differences') + parser.add_argument('files', metavar='file', nargs='+') + parser.add_argument( + '-q', + '--quiet', + action='store_true', + help="disable output, useful for the exit code") + parser.add_argument( + '-j', + metavar='N', + type=int, + default=0, + help='run N clang-format jobs in parallel' + ' (default number of cpus + 1)') + parser.add_argument( + '--color', + default='auto', + choices=['auto', 'always', 'never'], + help='show colored diff (default: auto)') + parser.add_argument( + '-e', + '--exclude', + metavar='PATTERN', + action='append', + default=[], + help='exclude paths matching the given glob-like pattern(s)' + ' from recursive search') + parser.add_argument( + '--style', + help='formatting style to apply (LLVM, Google, Chromium, Mozilla, WebKit)') + + args = parser.parse_args() + + # use default signal handling, like diff return SIGINT value on ^C + # https://bugs.python.org/issue14229#msg156446 + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + signal.SIGPIPE + except AttributeError: + # compatibility, SIGPIPE does not exist on Windows + pass + else: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + colored_stdout = False + colored_stderr = False + if args.color == 'always': + colored_stdout = True + colored_stderr = True + elif args.color == 'auto': + colored_stdout = sys.stdout.isatty() + colored_stderr = sys.stderr.isatty() + + version_invocation = [args.clang_format_executable, str("--version")] + try: + subprocess.check_call(version_invocation, stdout=DEVNULL) + except subprocess.CalledProcessError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + return ExitStatus.TROUBLE + except OSError as e: + print_trouble( + parser.prog, + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(version_invocation), e + ), + use_colors=colored_stderr, + ) + return ExitStatus.TROUBLE + + retcode = ExitStatus.SUCCESS + + excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE) + excludes.extend(args.exclude) + + files = list_files( + args.files, + recursive=args.recursive, + exclude=excludes, + extensions=args.extensions.split(',')) + + if not files: + return + + njobs = args.j + if njobs == 0: + njobs = multiprocessing.cpu_count() + 1 + njobs = min(len(files), njobs) + + if njobs == 1: + # execute directly instead of in a pool, + # less overhead, simpler stacktraces + it = (run_clang_format_diff_wrapper(args, file) for file in files) + pool = None + else: + pool = multiprocessing.Pool(njobs) + it = pool.imap_unordered( + partial(run_clang_format_diff_wrapper, args), files) + pool.close() + while True: + try: + outs, errs = next(it) + except StopIteration: + break + except DiffError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + retcode = ExitStatus.TROUBLE + sys.stderr.writelines(e.errs) + except UnexpectedError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + sys.stderr.write(e.formatted_traceback) + retcode = ExitStatus.TROUBLE + # stop at the first unexpected error, + # something could be very wrong, + # don't process all files unnecessarily + if pool: + pool.terminate() + break + else: + sys.stderr.writelines(errs) + if outs == []: + continue + if not args.quiet: + print_diff(outs, use_color=colored_stdout) + if retcode == ExitStatus.SUCCESS: + retcode = ExitStatus.DIFF + if pool: + pool.join() + return retcode + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docker/images/build-env-base/Dockerfile b/docker/images/build-env-base/Dockerfile new file mode 100644 index 0000000..bc7738f --- /dev/null +++ b/docker/images/build-env-base/Dockerfile @@ -0,0 +1,77 @@ +# syntax=docker/dockerfile:1 +# LTODO: pin version +FROM ghcr.io/everest/everest-ci/run-env-base:refactor-docker-images + +RUN apt update \ + && apt install --no-install-recommends -y \ + # basic command line tools + git=1:2.39.2-1.1 \ + curl=7.88.1-10+deb12u5 \ + rsync=3.2.7-1 \ + # build tools + ninja-build=1.11.1-1 \ + make=4.3-4.1 \ + cmake=3.25.1-1 \ + # compilers + binutils=2.40-2 \ + gcc=4:12.2.0-3 \ + g++=4:12.2.0-3 \ + # compiler tools + ccache=4.8+really4.7.5-1 \ + # LTODO not used but required + lcov=1.16-1 + +# additional packages +RUN apt update \ + && apt install --no-install-recommends -y \ + # required by some everest libraries + libboost-all-dev=1.74.0.3 \ + # required by libocpp + libsqlite3-dev=3.40.1-2 \ + libssl-dev=3.0.13-1~deb12u1 \ + # required by everest-framework + libnode-dev=18.19.0+dfsg-6~deb12u2 \ + # required by packet sniffer module + pkg-config=1.8.1-1 \ + libpcap-dev=1.10.3-1 \ + libcap-dev=1:2.66-4 \ + # LTODO: move to build-env-everest-core + # required by RiseV2G + maven=3.8.7-1 \ + # LTODO only everest-core + python3-build=0.9.0-1 \ + libxml2-dev=2.9.14+dfsg-1.3~deb12u1 \ + libxslt1-dev=1.1.35-1 + +# Install Python packages +RUN apt update \ + && apt install --no-install-recommends -y \ + python3-venv=3.11.2-1+b1 \ + python3-pydantic=1.10.4-1 \ + python3-psutil=5.9.4-1+b1 \ + python3-cryptography=38.0.4-3 \ + python3-netifaces=0.11.0-2+b1 \ + python3-dateutil=2.8.2-2 + +RUN apt clean \ + && rm -rf /var/lib/apt/lists/* + +# Install additional Python packages +RUN python3 -m pip install --break-system-packages \ + environs==11.0.0 \ + aiofile==3.8.8 \ + py4j==0.10.9.7 \ + gcovr==7.2 + +# Install EVerest dependency manager +ARG EDM_VERSION=v0.5.5 +RUN python3 -m pip install --break-system-packages \ + git+https://github.com/EVerest/everest-dev-environment@${EDM_VERSION}#subdirectory=dependency_manager + +# Install everest-cmake +ARG EVEREST_CMAKE_PATH=/usr/lib/cmake/everest-cmake +ARG EVEREST_CMAKE_VERSION=v0.4.2 +RUN git clone https://github.com/EVerest/everest-cmake.git ${EVEREST_CMAKE_PATH} \ + && cd ${EVEREST_CMAKE_PATH} \ + && git checkout ${EVEREST_CMAKE_VERSION} \ + && rm -r .git diff --git a/docker/images/build-kit/Dockerfile b/docker/images/build-kit/Dockerfile new file mode 100644 index 0000000..da67910 --- /dev/null +++ b/docker/images/build-kit/Dockerfile @@ -0,0 +1,23 @@ +# syntax=docker/dockerfile:1 +# LTODO: pin version +FROM ghcr.io/everest/everest-ci/build-env-base:refactor-docker-images + + +ENV WORKSPACE_PATH=/workspace +ENV ASSETS_PATH=/assets +ARG EXT_MOUNT=/ext +ENV EXT_MOUNT=$EXT_MOUNT + +RUN mkdir $ASSETS_PATH +COPY maven-settings.xml $ASSETS_PATH/ + +COPY ./entrypoint.sh / + +# LTODO: why is this needed? +# Disable ownership checks +RUN git config --global --add safe.directory '*' + +WORKDIR $WORKSPACE_PATH + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["run-script", "init"] diff --git a/docker/images/dev-env-base/Dockerfile b/docker/images/dev-env-base/Dockerfile new file mode 100644 index 0000000..0df3085 --- /dev/null +++ b/docker/images/dev-env-base/Dockerfile @@ -0,0 +1,40 @@ +# syntax=docker/dockerfile:1 +# LTODO: pin version +FROM ghcr.io/everest/everest-ci/build-env-base:refactor-docker-images + +ARG USERNAME=docker +ARG USER_UID=1000 +RUN useradd -ms /bin/bash -u ${USER_UID} -U ${USERNAME} +# Extend the timeout +RUN mkdir -p /etc/apt/apt.conf.d/ \ + && echo 'Acquire::http::Timeout "100";' >> /etc/apt/apt.conf.d/99stahp_stahping \ + && echo 'Acquire::ftp::Timeout "100";' >> /etc/apt/apt.conf.d/99stahp_stahping \ + && echo 'Acquire::Retries "10";' >> /etc/apt/apt.conf.d/99stahp_stahping +RUN apt update \ + && apt install --no-install-recommends -y sudo \ + && echo ${USERNAME} ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/${USERNAME} \ + && chmod 0440 /etc/sudoers.d/${USERNAME} + +RUN apt update \ + && apt install --no-install-recommends -y \ + wget=1.21.3-1+b2 \ + doxygen=1.9.4-4 \ + graphviz=2.42.2-7+b3 \ + build-essential=12.9 \ + clang-format-15=1:15.0.6-4+b1 \ + clang-tidy-15=1:15.0.6-4+b1 \ + cppcheck=2.10-2 + +RUN apt clean \ + && rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/bin/clang-format-15 /usr/bin/clang-format \ + && ln -s /usr/bin/clang-tidy-15 /usr/bin/clang-tidy + +USER ${USERNAME} + +RUN mkdir ~/.ssh \ + && ssh-keyscan github.com > ~/.ssh/known_hosts + +ENV PATH="/home/$USERNAME/.local/bin:$PATH" +ENV CPM_SOURCE_CACHE="/home/$USERNAME/.cache/CPM" diff --git a/docker/images/run-env-base/Dockerfile b/docker/images/run-env-base/Dockerfile new file mode 100644 index 0000000..85ae6a9 --- /dev/null +++ b/docker/images/run-env-base/Dockerfile @@ -0,0 +1,20 @@ +# syntax=docker/dockerfile:1 +FROM debian:12-slim + +RUN apt update \ + && apt install --no-install-recommends -y \ + openjdk-17-jre=17.0.11+9-1~deb12u1 \ + nodejs=18.19.0+dfsg-6~deb12u2 \ + npm=9.2.0~ds1-1 \ + python3-pip=23.0.1+dfsg-1 \ + sqlite3=3.40.1-2 \ + libboost-program-options1.74.0=1.74.0+ds1-21 \ + libboost-log1.74.0=1.74.0+ds1-21 \ + libboost-chrono1.74.0=1.74.0+ds1-21 \ + libboost-system1.74.0=1.74.0+ds1-21 \ + libssl3=3.0.13-1~deb12u1 \ + libcurl4=7.88.1-10+deb12u5 \ + libcap2=1:2.66-4 \ + less=590-2.1~deb12u2 \ + && apt clean \ + && rm -rf /var/lib/apt/lists/*