From 58530ef9a2f8ec2a2379d0825bba4e4eed379258 Mon Sep 17 00:00:00 2001 From: Andreas Heinrich Date: Tue, 16 Jul 2024 09:46:26 +0200 Subject: [PATCH] Refactor docker images and workflows to build and deploy them * Remove workflow `deploy-build-kit.yaml`. It is integrated into `deploy-docker-images.yml`. * Refactor deploy-docker-images.yml workflow * Convert matrix into multiple jobs, one for each docker image, because the images depend on each other, and the matrix strategy does not support this. * Add `env-setup` job to set up the environment for the other jobs, because reusable workflows don't support environment variables. * Add input parameter to build deprecated images. Currently, the deprecated images are built by default, if not specified otherwise. * Use output `one_image_tag_short` as input for the dependent images * Refactor build-single-docker-image.yml * Rename input `docker_directory` to `directory`. Instead of setting the directory that contains all images, it is set now to the directory that contains the Dockerfile for the image to build. * Add input parameter `docker_file_name` to specify the Dockerfile to use. * Add input parameter `platforms` to specify the platforms to build the image for. The default is `linux/amd64,linux/arm64` and `linux/arm/v7`. * Add input parameter `depends_on_paths` to specify the paths that the image depends on. If the paths change, the image is rebuilt. * Add input parameter `build_args` to specify the build arguments for the image. * The name of the deployed image is now set by the input parameter `image_name`: `ghcr.io/everest/`. The directory name is independent of the image name. * Add output parameter `rebuild` to indicate if the image was rebuilt. * Add output parameter `image_tag_short` to pass the image tag to the dependent images. * Add output parameter `image_tag_long` to pass the full image tag to the dependent images. * Add output parameter `image_tags` to pass all full image tags. * Fix rebuild conditions * Update README.md * Move `ghcr.io/everest/everest-clan-format` to `ghcr.io/everest/everest-ci/everest-clang-format`. The old image is deprecated, but still available and deployed. * Merge `ghcr.io/everest/build-kit-alpine` and `ghcr.io/everest/build-kit-debian` into `ghcr.io/everest/everest-ci/build-kit-base`. The old images are deprecated, but still available and deployed. * Add `ghcr.io/everest/everest-ci/run-env-base` image * Add `ghcr.io/everest/everest-ci/build-env-base` image * Add `ghcr.io/everest/everest-ci/dev-env-base` image * Add `run-clang-format.py` script to `build-env-base` image * Refactor `everest-clang-format` image to use `build-env-base` as base image * Update github action `run-glang-format` to use new `everest-clang-format` image Signed-off-by: Andreas Heinrich --- .github/workflows/deploy-build-kit.yaml | 44 -- .github/workflows/deploy-docker-images.yml | 203 +++++++- .../workflows/deploy-single-docker-image.yml | 181 +++++++- README.md | 54 ++- .../build-kit/.dockerignore | 0 .../build-kit/alpine.Dockerfile | 3 +- .../build-kit/debian.Dockerfile | 3 +- .../build-kit/deprecated_wrapper | 20 + .../build-kit/entrypoint.sh | 0 .../build-kit/maven-settings.xml | 0 .../everest-clang-format/Dockerfile | 15 + .../everest-clang-format/deprecated_wrapper | 20 + .../everest-clang-format/run-clang-format.py | 0 docker/images/build-env-base/Dockerfile | 81 ++++ .../images/build-env-base/run-clang-format.py | 434 ++++++++++++++++++ docker/images/build-kit-base/.dockerignore | 1 + docker/images/build-kit-base/Dockerfile | 21 + docker/images/build-kit-base/entrypoint.sh | 44 ++ .../images/build-kit-base/maven-settings.xml | 4 + docker/images/dev-env-base/Dockerfile | 53 +++ docker/images/everest-clang-format/Dockerfile | 15 +- docker/images/run-env-base/Dockerfile | 20 + github-actions/run-clang-format/action.yaml | 2 +- 23 files changed, 1133 insertions(+), 85 deletions(-) delete mode 100644 .github/workflows/deploy-build-kit.yaml rename docker/{images => deprecated-images}/build-kit/.dockerignore (100%) rename docker/{images => deprecated-images}/build-kit/alpine.Dockerfile (97%) rename docker/{images => deprecated-images}/build-kit/debian.Dockerfile (97%) create mode 100755 docker/deprecated-images/build-kit/deprecated_wrapper rename docker/{images => deprecated-images}/build-kit/entrypoint.sh (100%) rename docker/{images => deprecated-images}/build-kit/maven-settings.xml (100%) create mode 100644 docker/deprecated-images/everest-clang-format/Dockerfile create mode 100755 docker/deprecated-images/everest-clang-format/deprecated_wrapper rename docker/{images => deprecated-images}/everest-clang-format/run-clang-format.py (100%) create mode 100644 docker/images/build-env-base/Dockerfile create mode 100755 docker/images/build-env-base/run-clang-format.py create mode 100644 docker/images/build-kit-base/.dockerignore create mode 100644 docker/images/build-kit-base/Dockerfile create mode 100755 docker/images/build-kit-base/entrypoint.sh create mode 100644 docker/images/build-kit-base/maven-settings.xml create mode 100644 docker/images/dev-env-base/Dockerfile create mode 100644 docker/images/run-env-base/Dockerfile diff --git a/.github/workflows/deploy-build-kit.yaml b/.github/workflows/deploy-build-kit.yaml deleted file mode 100644 index 57c86ba..0000000 --- a/.github/workflows/deploy-build-kit.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: Deploy build-kit docker image - -on: - workflow_dispatch: - inputs: - base-system: - type: choice - description: Base system type - default: 'alpine' - required: true - options: - - alpine - - debian - tag: - type: string - description: Image tag - default: 'latest' - -jobs: - deploy-build-kit-image: - name: Build and push build-kit docker image - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup docker buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: docker/images/build-kit - file: docker/images/build-kit/${{ inputs.base-system }}.Dockerfile - tags: ghcr.io/everest/build-kit-${{ inputs.base-system }}:${{ inputs.tag }} - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - diff --git a/.github/workflows/deploy-docker-images.yml b/.github/workflows/deploy-docker-images.yml index 21e124c..3c78d5e 100644 --- a/.github/workflows/deploy-docker-images.yml +++ b/.github/workflows/deploy-docker-images.yml @@ -7,6 +7,10 @@ on: description: 'Force rebuild of all images' default: false type: boolean + build_deprecated_images: + description: 'Build deprecated images' + default: false + type: boolean push: branches: - '**' @@ -15,21 +19,200 @@ on: env: REGISTRY: ghcr.io + DOCKER_DIRECTORY: docker/images + PLATFORMS: | + linux/amd64 + PATH_TO_DEPLOY_SINGLE_DOCKER_IMAGE_WORKFLOW: .github/workflows/deploy-single-docker-image.yml + PATH_TO_DEPLOY_DOCKER_IMAGES_WORKFLOW: .github/workflows/deploy-docker-images.yml jobs: - build-and-push-all-images: - name: Build and push all docker images - strategy: - matrix: - image_name: [everest-clang-format] - uses: everest/everest-ci/.github/workflows/deploy-single-docker-image.yml@v1.1.0 + env-setup: + # Since env variables can't be passed to reusable workflows, we need to pass them as outputs + name: Evaluate force rebuild and set env variables as outputs + runs-on: ubuntu-22.04 + outputs: + force_rebuild: ${{ steps.check.outputs.force_rebuild }} + docker_registry: ${{ env.REGISTRY }} + docker_directory: ${{ env.DOCKER_DIRECTORY }} + platforms: ${{ env.PLATFORMS }} + repository_name: ${{ github.event.repository.name }} + build_deprecated_images: ${{ steps.check.outputs.build_deprecated_images }} + path_to_deploy_single_docker_image_workflow: ${{ env.PATH_TO_DEPLOY_SINGLE_DOCKER_IMAGE_WORKFLOW }} + path_to_deploy_docker_images_workflow: ${{ env.PATH_TO_DEPLOY_DOCKER_IMAGES_WORKFLOW }} + steps: + - id: check + run: | + echo "force_rebuild=${{ inputs.force_rebuild || (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/' )) || false }}" >> $GITHUB_OUTPUT + # Build deprecated images if not explicitly disabled + #TODO: set default to false, once backwards compatibility is no longer needed + echo "build_deprecated_images=${{ inputs.build_deprecated_images || true }}" >> $GITHUB_OUTPUT + + # One job for each image, since the images build on top of each other a matrix strategy is not possible + everest-clang-format: + needs: + - env-setup + name: Build and push everest-clang-format docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/everest-clang-format + directory: ${{ needs.env-setup.outputs.docker_directory }}/everest-clang-format + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + depends_on_paths: | + ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }} + ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }} + run-env-base: + needs: + - env-setup + name: Build and push run-env-base docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} with: - force_rebuild: ${{ inputs.force_rebuild || (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/' )) || false }} - image_name: ${{ matrix.image_name }} - docker_directory: docker/images/ - docker_registry: ghcr.io + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/run-env-base + directory: ${{ needs.env-setup.outputs.docker_directory }}/run-env-base + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} github_ref_before: ${{ github.event.before }} github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + depends_on_paths: | + ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }} + ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }} + build-env-base: + needs: + - env-setup + - run-env-base + name: Build and push build-env-base docker image + uses: ./.github/workflows/deploy-single-docker-image.yml secrets: SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' || needs.run-env-base.outputs.rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/build-env-base + directory: ${{ needs.env-setup.outputs.docker_directory }}/build-env-base + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + depends_on_paths: | + ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }} + ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }} + build_args: | + BASE_IMAGE_TAG=${{ needs.run-env-base.outputs.one_image_tag_short }} + + dev-env-base: + needs: + - env-setup + - build-env-base + name: Build and push dev-env-base docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' || needs.build-env-base.outputs.rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/dev-env-base + directory: ${{ needs.env-setup.outputs.docker_directory }}/dev-env-base + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + depends_on_paths: | + ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }} + ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }} + build_args: | + BASE_IMAGE_TAG=${{ needs.build-env-base.outputs.one_image_tag_short }} + build-kit-base: + needs: + - env-setup + - build-env-base + name: Build and push build-kit docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' || needs.build-env-base.outputs.rebuild == 'true' }} + image_name: ${{ needs.env-setup.outputs.repository_name }}/build-kit-base + directory: ${{ needs.env-setup.outputs.docker_directory }}/build-kit-base + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + depends_on_paths: | + ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }} + ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }} + build_args: | + BASE_IMAGE_TAG=${{ needs.build-env-base.outputs.one_image_tag_short }} + # Include deprecated images for backwards compatibility + deprecated-everest-clang-format: + if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }} + needs: + - env-setup + name: Build and push deprecated everest-clang-format docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: everest-clang-format + directory: docker/deprecated-images/everest-clang-format + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + depends_on_paths: | + ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }} + ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }} + deprecated-build-kit-alpine: + if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }} + needs: + - env-setup + name: Build and push deprecated build-kit-alpine docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: build-kit-alpine + directory: docker/deprecated-images/build-kit + docker_file_name: alpine.Dockerfile + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + depends_on_paths: | + ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }} + ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }} + deprecated-build-kit-debian: + if: ${{ needs.env-setup.outputs.build_deprecated_images == 'true' }} + needs: + - env-setup + name: Build and push deprecated build-kit-debian docker image + uses: ./.github/workflows/deploy-single-docker-image.yml + secrets: + SA_GITHUB_PAT: ${{ secrets.SA_GITHUB_PAT }} + SA_GITHUB_USERNAME: ${{ secrets.SA_GITHUB_USERNAME }} + with: + force_rebuild: ${{ needs.env-setup.outputs.force_rebuild == 'true' }} + image_name: build-kit-debian + directory: docker/deprecated-images/build-kit + docker_file_name: debian.Dockerfile + docker_registry: ${{ needs.env-setup.outputs.docker_registry }} + github_ref_before: ${{ github.event.before }} + github_ref_after: ${{ github.event.after }} + platforms: ${{ needs.env-setup.outputs.platforms }} + depends_on_paths: | + ${{ needs.env-setup.outputs.path_to_deploy_single_docker_image_workflow }} + ${{ needs.env-setup.outputs.path_to_deploy_docker_images_workflow }} diff --git a/.github/workflows/deploy-single-docker-image.yml b/.github/workflows/deploy-single-docker-image.yml index a1f36cd..63c778b 100644 --- a/.github/workflows/deploy-single-docker-image.yml +++ b/.github/workflows/deploy-single-docker-image.yml @@ -7,11 +7,15 @@ on: description: 'Name of the image to build and push' required: true type: string - docker_directory: - description: 'Directory containing the docker directory' + directory: + description: 'Directory containing the Dockerfile' required: true type: string - default: docker/ + docker_file_name: + description: 'Name of the Dockerfile' + required: false + default: 'Dockerfile' + type: string docker_registry: description: 'Docker registry to push to' required: true @@ -29,6 +33,34 @@ on: description: 'Forces rebuild of docker image, even if no changes are detected' default: false type: boolean + platforms: + description: 'Platforms to build for' + default: | + linux/amd64 + linux/arm64 + linux/arm/v7 + type: string + depends_on_paths: + description: 'Directories that need to be checked for changes before building this image' + required: false + type: string + build_args: + description: 'Build arguments for the Dockerfile' + required: false + type: string + outputs: + rebuild: + description: 'Whether the image is rebuilt or not' + value: ${{ jobs.check.outputs.rebuild }} + image_tags: + description: 'Tags of the deployed image with image name' + value: ${{ jobs.build-and-push.outputs.image_tags }} + one_image_tag_short: + description: 'One tag of the deployed image without image name' + value: ${{ jobs.build-and-push.outputs.one_image_tag_short }} + one_image_tag_long: + description: 'One tag of the deployed image with image name' + value: ${{ jobs.build-and-push.outputs.one_image_tag_long }} secrets: SA_GITHUB_PAT: description: 'Github PAT with access to the repository' @@ -41,10 +73,11 @@ jobs: check: name: Check for changes outputs: - changed_files: ${{ steps.changed-files.outputs.changed_files }} + rebuild: ${{ steps.set-outputs.outputs.rebuild }} runs-on: ubuntu-22.04 steps: - name: Checkout Dockerfile + if: ${{ inputs.force_rebuild == false }} uses: actions/checkout@v3 with: repository: ${{ github.repository }} @@ -52,17 +85,87 @@ jobs: ref: ${{ inputs.github_ref_after }} token: ${{secrets.SA_GITHUB_PAT}} fetch-depth: 0 + - name: Validate github_ref_before and github_ref_after + if: ${{ inputs.force_rebuild == false }} + id: validate-inputs + run: | + echo "Check validity of github_ref_before.." + if git cat-file -e ${{ inputs.github_ref_before }}^{commit}; then + echo "github_ref_before='${{ inputs.github_ref_before }}' is a commit!✅" + else + echo "github_ref_before='${{ inputs.github_ref_before }}' is not a commit!❌ -> Rebuild image!" + echo "validation_failed=true" >> $GITHUB_OUTPUT + exit 0 + fi + echo "Check validity of github_ref_after.." + if git cat-file -e ${{ inputs.github_ref_after }}^{commit}; then + echo "github_ref_after='${{ inputs.github_ref_after }}' is a commit!✅" + else + echo "github_ref_after='${{ inputs.github_ref_after }}' is not a commit!❌ -> Rebuild image!" + echo "validation_failed=true" >> $GITHUB_OUTPUT + exit 0 + fi + echo "validation_failed=false" >> $GITHUB_OUTPUT + working-directory: source + - name: Parse depends_on_paths + if: ${{ inputs.force_rebuild == false && steps.validate-inputs.outputs.validation_failed == 'false' }} + id: setup-regex + shell: python3 {0} + working-directory: source + run: | + import os + list_of_paths = """${{ inputs.depends_on_paths }}""".split("\n") + # remove all empty lines + list_of_paths = list(filter(None, list_of_paths)) + for path in list_of_paths: + if os.path.isdir(f"{path}"): + path = f"{path}/*" + elif os.path.isfile(f"{path}"): + path = f"{path}" + else: + print(f"Path {path} does not exist!❌") + exit(1) + if not "${{ inputs.directory }}" in list_of_paths: + list_of_paths.append("${{ inputs.directory }}/*") + regex = "|".join(list_of_paths) + # set the regex as an output + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"regex={regex}\n") + print(f"Set regex={regex}") - name: Get changed files - id: changed-files + if: ${{ inputs.force_rebuild == false && steps.validate-inputs.outputs.validation_failed == 'false' }} + id: check run: | - echo "changed_files=$(git diff --name-only ${{ inputs.github_ref_before }} ${{ inputs.github_ref_after }} | grep "^${{inputs.docker_directory}}/${{inputs.image_name}}" | wc -l)" >> $GITHUB_OUTPUT + echo "Checking for changes in paths ${{ steps.setup-regex.outputs.regex }}.." + EXIT_CODE=0 + changed_files="$(git diff --name-only ${{ inputs.github_ref_before }} ${{ inputs.github_ref_after }} | grep -E '${{ steps.setup-regex.outputs.regex }}')" || EXIT_CODE=$? + if [ $EXIT_CODE -eq 1 ]; then + echo "No changes detected in ${{ steps.setup-regex.outputs.regex }}!🔍" + echo "changes_detected=false" >> $GITHUB_OUTPUT + exit 0 + elif [ $EXIT_CODE -ne 0 ]; then + echo "Error while checking for changes in ${{ steps.setup-regex.outputs.regex }}!❌" + exit $EXIT_CODE + else + echo "🔍 Changes detected with regex ${{ steps.setup-regex.outputs.regex }} in the following files:" + echo "$changed_files" + echo "changes_detected=true" >> $GITHUB_OUTPUT + fi working-directory: source + - name: Set outputs + if: ${{ inputs.force_rebuild == false }} + id: set-outputs + run: | + echo "rebuild=${{ steps.check.outputs.changes_detected == 'true' || steps.validate-inputs.outputs.validation_failed == 'true' }}" >> $GITHUB_OUTPUT build-and-push: name: Build and push needs: check - if: ${{ needs.check.outputs.changed_files > 0 || inputs.force_rebuild }} runs-on: ubuntu-22.04 + outputs: + image_tags: ${{ steps.meta.outputs.tags }} + one_image_tag_short: ${{ steps.extract-one-tag.outputs.tag_short }} + one_image_tag_long: ${{ steps.extract-one-tag.outputs.tag_long }} steps: - name: Checkout Dockerfile uses: actions/checkout@v3 @@ -75,39 +178,89 @@ jobs: - name: Get context / Path of Dockerfile id: get-context run: | - if [ -f source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/Dockerfile ]; then - echo "::set-output name=path::source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/" - elif [ -f source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/.devcontainer/Dockerfile ]; then - echo "::set-output name=path::source/${{ inputs.docker_directory }}/${{ inputs.image_name }}/.devcontainer/" + if [ -f source/${{ inputs.directory }}/${{ inputs.docker_file_name }} ]; then + echo "path=source/${{ inputs.directory }}" >> $GITHUB_OUTPUT + elif [ -f source/${{ inputs.directory }}/.devcontainer/${{ inputs.docker_file_name }} ]; then + echo "path=source/${{ inputs.directory }}/.devcontainer" >> $GITHUB_OUTPUT else - echo "No Dockerfile found for image ${{ inputs.image_name }}!" + echo "No Dockerfile found for image ${{ inputs.image_name }} in dokcer_directory ${{ inputs.directory }} with docker_file_name ${{ inputs.docker_file_name }}!" exit 1 fi + - name: Parse platforms + id: parse-platforms + run: | + # remove all empty lines + result=$(echo "${{ inputs.platforms }}" | sed '/^$/d') + # replace newlines with commas, remove trailing comma + result=$(echo "${result}" | tr '\n' ',' | sed 's/,$//') + echo "platforms=${result}" >> $GITHUB_OUTPUT - name: Extract metadata id: meta uses: docker/metadata-action@v3 with: - images: ${{ inputs.docker_registry }}/${{ github.repository_owner }}/${{ inputs.image_name }} + sep-tags: ',' + images: | + ${{ inputs.docker_registry }}/${{ github.repository_owner }}/${{ inputs.image_name }} - name: Set up QEMU uses: docker/setup-qemu-action@v1 + if: ${{ needs.check.outputs.rebuild == 'true' || inputs.force_rebuild }} with: image: tonistiigi/binfmt:latest platforms: all - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + if: ${{ needs.check.outputs.rebuild == 'true' || inputs.force_rebuild }} - name: Login to DockerHub uses: docker/login-action@v2 + if: ${{ needs.check.outputs.rebuild == 'true' || inputs.force_rebuild }} with: registry: ${{ inputs.docker_registry }} username: ${{ secrets.SA_GITHUB_USERNAME }} password: ${{ secrets.SA_GITHUB_PAT }} - name: Build and push uses: docker/build-push-action@v3 + if: ${{ needs.check.outputs.rebuild == 'true' || inputs.force_rebuild }} with: context: ${{ steps.get-context.outputs.path }} + file: ${{ steps.get-context.outputs.path }}/${{ inputs.docker_file_name }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: ${{ steps.parse-platforms.outputs.platforms }} cache-from: type=gha cache-to: type=gha,mode=max + build-args: ${{ inputs.build_args }} + - name: Extract one image tag + id: extract-one-tag + shell: python3 {0} + run: | + import os + tags = "${{ steps.meta.outputs.tags }}".split(",") + tags_short = [tag.split(":")[-1] for tag in tags] + if len(tags_short) == 0: + print("No tags found!❌") + exit(1) + tag_short = None + for t in tags_short: + if t.startswith("v"): + tag_short = t + break + if tag_short is None: + for t in tags_short: + if not t.startswith("latest"): + tag_short = t + break + if tag_short is None: + tag_short = tags_short[0] + if tag_short is None: + print("No tag found!❌") + exit(1) + for t in tags: + if t.endswith(tag_short): + tag_long = t + break + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"tag_short={tag_short}\n") + print(f"Set tag_short={tag_short}") + f.write(f"tag_long={tag_long}\n") + print(f"Set tag_long={tag_long}") diff --git a/README.md b/README.md index be2480a..0aec86c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,56 @@ -# everest ci/cd +# EVerest CI/CD This repository should share all common used functionality for the everest github ci/cd pipe + +## Docker Images + +### run-env-base + +Based on `debian:12-slim`. + +Deployed as `ghcr.io/everest/everest-ci/run-env-base`. + +Includes all dependencies that are necessary to run EVerest. + +### build-env-base + +Based on `run-env-base`. + +Deployed as `ghcr.io/everest/everest-ci/build-env-base`. + +Includes all dependencies that are necessary to build EVerest. + +### dev-env-base + +Based on `build-env-base`. + +Deployed as `ghcr.io/everest/everest-ci/dev-env-base`. + +Include additional packages to develop EVerest. + +### everest-clang-format + +> [!NOTE] +> This image was moved from `ghcr.io/everest/clang-format` to +> `ghcr.io/everest/everest-ci/everest-clang-format`. The old image is +> deprecated and will be removed in the future. + +Deployed as `ghcr.io/everest/everest-ci/everest-clang-format`. + +Contains a fixed clang-format version in addition to a run-clang-format script. This image can be diretcly executed. +This image is used as base for a custom github action and can also be used to run clang-format locally. + +### build-kit-base + +> [!NOTE] +> The images `ghcr.io/everest/build-kit-alpine` and `ghcr.io/everest/build-kit-debian` +> are deprecated and will be removed in the future. Please use +> `ghcr.io/everest/everest-ci/build-kit-base` instead. + +Based on `build-env-base`. + +Deployed as `ghcr.io/everest/everest-ci/build-kit-base`. + +This image includes a mechanic to run bash scripts in the build environment. It is used to run the build scripts in the CI/CD pipeline. +It is executed directly. diff --git a/docker/images/build-kit/.dockerignore b/docker/deprecated-images/build-kit/.dockerignore similarity index 100% rename from docker/images/build-kit/.dockerignore rename to docker/deprecated-images/build-kit/.dockerignore diff --git a/docker/images/build-kit/alpine.Dockerfile b/docker/deprecated-images/build-kit/alpine.Dockerfile similarity index 97% rename from docker/images/build-kit/alpine.Dockerfile rename to docker/deprecated-images/build-kit/alpine.Dockerfile index a9dc399..ef6cd65 100644 --- a/docker/images/build-kit/alpine.Dockerfile +++ b/docker/deprecated-images/build-kit/alpine.Dockerfile @@ -100,5 +100,6 @@ COPY ./entrypoint.sh / WORKDIR $WORKSPACE_PATH -ENTRYPOINT ["/entrypoint.sh"] +COPY deprecated_wrapper / +ENTRYPOINT ["/deprecated_wrapper"] CMD ["run-script", "init"] diff --git a/docker/images/build-kit/debian.Dockerfile b/docker/deprecated-images/build-kit/debian.Dockerfile similarity index 97% rename from docker/images/build-kit/debian.Dockerfile rename to docker/deprecated-images/build-kit/debian.Dockerfile index 2a0287c..1814d66 100644 --- a/docker/images/build-kit/debian.Dockerfile +++ b/docker/deprecated-images/build-kit/debian.Dockerfile @@ -91,5 +91,6 @@ COPY ./entrypoint.sh / WORKDIR $WORKSPACE_PATH -ENTRYPOINT ["/entrypoint.sh"] +COPY deprecated_wrapper / +ENTRYPOINT ["/deprecated_wrapper"] CMD ["run-script", "init"] diff --git a/docker/deprecated-images/build-kit/deprecated_wrapper b/docker/deprecated-images/build-kit/deprecated_wrapper new file mode 100755 index 0000000..eb23ea7 --- /dev/null +++ b/docker/deprecated-images/build-kit/deprecated_wrapper @@ -0,0 +1,20 @@ +#!/bin/bash + +function print_warning { + echo -e "\033[0;31m" + echo "---------------------------------------------" + echo "WARNING" + echo "This docker image is depreacted." + echo "Please use the debian based build-kit instead 'ghcr.io/everest/everest-ci/build-kit-base' instead." + echo "Fore more information see https://github.com/EVerest/everest-ci/pull/22" + echo "---------------------------------------------" + echo -e "\033[0m" +} + +print_warning + +/entrypoint.sh "$@" +exit_code=$? + +print_warning +exit $exit_code diff --git a/docker/images/build-kit/entrypoint.sh b/docker/deprecated-images/build-kit/entrypoint.sh similarity index 100% rename from docker/images/build-kit/entrypoint.sh rename to docker/deprecated-images/build-kit/entrypoint.sh diff --git a/docker/images/build-kit/maven-settings.xml b/docker/deprecated-images/build-kit/maven-settings.xml similarity index 100% rename from docker/images/build-kit/maven-settings.xml rename to docker/deprecated-images/build-kit/maven-settings.xml diff --git a/docker/deprecated-images/everest-clang-format/Dockerfile b/docker/deprecated-images/everest-clang-format/Dockerfile new file mode 100644 index 0000000..6d7aba4 --- /dev/null +++ b/docker/deprecated-images/everest-clang-format/Dockerfile @@ -0,0 +1,15 @@ +ARG BASE_IMAGE=debian:12 +FROM ${BASE_IMAGE} + +RUN apt update && apt upgrade -y +RUN apt update && \ + apt install -y \ + clang-format-15=1:15.0.6* +RUN ln -s /usr/bin/clang-format-15 /usr/bin/clang-format + +RUN apt install -y \ + python-is-python3 + +COPY run-clang-format.py /usr/bin/run-clang-format +COPY deprecated_wrapper /usr/bin/deprecated_wrapper +ENTRYPOINT ["/usr/bin/deprecated_wrapper"] diff --git a/docker/deprecated-images/everest-clang-format/deprecated_wrapper b/docker/deprecated-images/everest-clang-format/deprecated_wrapper new file mode 100755 index 0000000..dd9cd20 --- /dev/null +++ b/docker/deprecated-images/everest-clang-format/deprecated_wrapper @@ -0,0 +1,20 @@ +#!/bin/bash + +function print_warning { + echo -e "\033[0;31m" + echo "---------------------------------------------" + echo "WARNING" + echo "This docker image is depreacted." + echo "It was moved from 'ghcr.io/everest/everest-clang-format' to 'ghcr.io/everest/everest-ci/everest-clang-format'." + echo "Please use 'ghcr.io/everest/everest-ci/everest-clang-format' instead." + echo "---------------------------------------------" + echo -e "\033[0m" +} + +print_warning + +/usr/bin/clang-format "$@" +exit_code=$? + +print_warning +exit $exit_code diff --git a/docker/images/everest-clang-format/run-clang-format.py b/docker/deprecated-images/everest-clang-format/run-clang-format.py similarity index 100% rename from docker/images/everest-clang-format/run-clang-format.py rename to docker/deprecated-images/everest-clang-format/run-clang-format.py diff --git a/docker/images/build-env-base/Dockerfile b/docker/images/build-env-base/Dockerfile new file mode 100644 index 0000000..f6f8fa2 --- /dev/null +++ b/docker/images/build-env-base/Dockerfile @@ -0,0 +1,81 @@ +# syntax=docker/dockerfile:1 +ARG BASE_IMAGE_TAG=latest +FROM ghcr.io/everest/everest-ci/run-env-base:${BASE_IMAGE_TAG} + +RUN apt update \ + && apt install --no-install-recommends -y \ + # basic command line tools + git=1:2.39.2-1.1 \ + curl=7.88.1-10+deb12u5 \ + rsync=3.2.7-1 \ + # build tools + ninja-build=1.11.1-1 \ + make=4.3-4.1 \ + cmake=3.25.1-1 \ + # compilers + binutils=2.40-2 \ + gcc=4:12.2.0-3 \ + g++=4:12.2.0-3 \ + # compiler tools + ccache=4.8+really4.7.5-1 \ + lcov=1.16-1 \ + clang-format-15=1:15.0.6-4+b1 \ + clang-tidy-15=1:15.0.6-4+b1 + +# Create symlinks for clang-format and clang-tidy +RUN ln -s /usr/bin/clang-format-15 /usr/bin/clang-format \ + && ln -s /usr/bin/clang-tidy-15 /usr/bin/clang-tidy + +COPY run-clang-format.py /usr/bin/run-clang-format + +# additional packages +RUN apt update \ + && apt install --no-install-recommends -y \ + # required by some everest libraries + libboost-all-dev=1.74.0.3 \ + # required by libocpp + libsqlite3-dev=3.40.1-2 \ + libssl-dev=3.0.13-1~deb12u1 \ + # required by everest-framework + libnode-dev=18.19.0+dfsg-6~deb12u2 \ + # required by packet sniffer module + pkg-config=1.8.1-1 \ + libpcap-dev=1.10.3-1 \ + libcap-dev=1:2.66-4 \ + # required by RiseV2G + maven=3.8.7-1 \ + libxml2-dev=2.9.14+dfsg-1.3~deb12u1 \ + libxslt1-dev=1.1.35-1 + +# Install Python packages +RUN apt update \ + && apt install --no-install-recommends -y \ + python3-venv=3.11.2-1+b1 \ + python3-pydantic=1.10.4-1 \ + python3-psutil=5.9.4-1+b1 \ + python3-cryptography=38.0.4-3 \ + python3-netifaces=0.11.0-2+b1 \ + python3-dateutil=2.8.2-2 + +RUN apt clean \ + && rm -rf /var/lib/apt/lists/* + +# Install additional Python packages +RUN python3 -m pip install --break-system-packages \ + environs==11.0.0 \ + aiofile==3.8.8 \ + py4j==0.10.9.7 \ + gcovr==7.2 + +# Install EVerest dependency manager +ARG EDM_VERSION=v0.5.5 +RUN python3 -m pip install --break-system-packages \ + git+https://github.com/EVerest/everest-dev-environment@${EDM_VERSION}#subdirectory=dependency_manager + +# Install everest-cmake +ARG EVEREST_CMAKE_PATH=/usr/lib/cmake/everest-cmake +ARG EVEREST_CMAKE_VERSION=v0.4.2 +RUN git clone https://github.com/EVerest/everest-cmake.git ${EVEREST_CMAKE_PATH} \ + && cd ${EVEREST_CMAKE_PATH} \ + && git checkout ${EVEREST_CMAKE_VERSION} \ + && rm -r .git diff --git a/docker/images/build-env-base/run-clang-format.py b/docker/images/build-env-base/run-clang-format.py new file mode 100755 index 0000000..3f935d6 --- /dev/null +++ b/docker/images/build-env-base/run-clang-format.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python + +# See original source at https://github.com/Sarcasm/run-clang-format/blob/39081c9c42768ab5e8321127a7494ad1647c6a2f/run-clang-format.py +# +# MIT License +# +# Copyright (c) 2017 Guillaume Papin +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +"""A wrapper script around clang-format, suitable for linting multiple files +and to use for continuous integration. + +This is an alternative API for the clang-format command line. +It runs over multiple files and directories in parallel. +A diff output is produced and a sensible exit code is returned. + +""" + +from __future__ import print_function, unicode_literals + +import argparse +import codecs +import difflib +import fnmatch +import io +import errno +import multiprocessing +import os +import signal +import subprocess +import sys +import traceback + +from functools import partial + +try: + from subprocess import DEVNULL # py3k +except ImportError: + DEVNULL = open(os.devnull, "wb") + + +DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx' +DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore' + + +class ExitStatus: + SUCCESS = 0 + DIFF = 1 + TROUBLE = 2 + +def excludes_from_file(ignore_file): + excludes = [] + try: + with io.open(ignore_file, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('#'): + # ignore comments + continue + pattern = line.rstrip() + if not pattern: + # allow empty lines + continue + excludes.append(pattern) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + return excludes; + +def list_files(files, recursive=False, extensions=None, exclude=None): + if extensions is None: + extensions = [] + if exclude is None: + exclude = [] + + out = [] + for file in files: + if recursive and os.path.isdir(file): + for dirpath, dnames, fnames in os.walk(file): + fpaths = [os.path.join(dirpath, fname) for fname in fnames] + for pattern in exclude: + # os.walk() supports trimming down the dnames list + # by modifying it in-place, + # to avoid unnecessary directory listings. + dnames[:] = [ + x for x in dnames + if + not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) + ] + fpaths = [ + x for x in fpaths if not fnmatch.fnmatch(x, pattern) + ] + for f in fpaths: + ext = os.path.splitext(f)[1][1:] + if ext in extensions: + out.append(f) + else: + out.append(file) + return out + + +def make_diff(file, original, reformatted): + return list( + difflib.unified_diff( + original, + reformatted, + fromfile='{}\t(original)'.format(file), + tofile='{}\t(reformatted)'.format(file), + n=3)) + + +class DiffError(Exception): + def __init__(self, message, errs=None): + super(DiffError, self).__init__(message) + self.errs = errs or [] + + +class UnexpectedError(Exception): + def __init__(self, message, exc=None): + super(UnexpectedError, self).__init__(message) + self.formatted_traceback = traceback.format_exc() + self.exc = exc + + +def run_clang_format_diff_wrapper(args, file): + try: + ret = run_clang_format_diff(args, file) + return ret + except DiffError: + raise + except Exception as e: + raise UnexpectedError('{}: {}: {}'.format(file, e.__class__.__name__, + e), e) + + +def run_clang_format_diff(args, file): + try: + with io.open(file, 'r', encoding='utf-8') as f: + original = f.readlines() + except IOError as exc: + raise DiffError(str(exc)) + + if args.in_place: + invocation = [args.clang_format_executable, '-i', file] + else: + invocation = [args.clang_format_executable, file] + + if args.style: + invocation.extend(['--style', args.style]) + + if args.dry_run: + print(" ".join(invocation)) + return [], [] + + # Use of utf-8 to decode the process output. + # + # Hopefully, this is the correct thing to do. + # + # It's done due to the following assumptions (which may be incorrect): + # - clang-format will returns the bytes read from the files as-is, + # without conversion, and it is already assumed that the files use utf-8. + # - if the diagnostics were internationalized, they would use utf-8: + # > Adding Translations to Clang + # > + # > Not possible yet! + # > Diagnostic strings should be written in UTF-8, + # > the client can translate to the relevant code page if needed. + # > Each translation completely replaces the format string + # > for the diagnostic. + # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation + # + # It's not pretty, due to Python 2 & 3 compatibility. + encoding_py3 = {} + if sys.version_info[0] >= 3: + encoding_py3['encoding'] = 'utf-8' + + try: + proc = subprocess.Popen( + invocation, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + **encoding_py3) + except OSError as exc: + raise DiffError( + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(invocation), exc + ) + ) + proc_stdout = proc.stdout + proc_stderr = proc.stderr + if sys.version_info[0] < 3: + # make the pipes compatible with Python 3, + # reading lines should output unicode + encoding = 'utf-8' + proc_stdout = codecs.getreader(encoding)(proc_stdout) + proc_stderr = codecs.getreader(encoding)(proc_stderr) + # hopefully the stderr pipe won't get full and block the process + outs = list(proc_stdout.readlines()) + errs = list(proc_stderr.readlines()) + proc.wait() + if proc.returncode: + raise DiffError( + "Command '{}' returned non-zero exit status {}".format( + subprocess.list2cmdline(invocation), proc.returncode + ), + errs, + ) + if args.in_place: + return [], errs + return make_diff(file, original, outs), errs + + +def bold_red(s): + return '\x1b[1m\x1b[31m' + s + '\x1b[0m' + + +def colorize(diff_lines): + def bold(s): + return '\x1b[1m' + s + '\x1b[0m' + + def cyan(s): + return '\x1b[36m' + s + '\x1b[0m' + + def green(s): + return '\x1b[32m' + s + '\x1b[0m' + + def red(s): + return '\x1b[31m' + s + '\x1b[0m' + + for line in diff_lines: + if line[:4] in ['--- ', '+++ ']: + yield bold(line) + elif line.startswith('@@ '): + yield cyan(line) + elif line.startswith('+'): + yield green(line) + elif line.startswith('-'): + yield red(line) + else: + yield line + + +def print_diff(diff_lines, use_color): + if use_color: + diff_lines = colorize(diff_lines) + if sys.version_info[0] < 3: + sys.stdout.writelines((l.encode('utf-8') for l in diff_lines)) + else: + sys.stdout.writelines(diff_lines) + + +def print_trouble(prog, message, use_colors): + error_text = 'error:' + if use_colors: + error_text = bold_red(error_text) + print("{}: {} {}".format(prog, error_text, message), file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--clang-format-executable', + metavar='EXECUTABLE', + help='path to the clang-format executable', + default='clang-format') + parser.add_argument( + '--extensions', + help='comma separated list of file extensions (default: {})'.format( + DEFAULT_EXTENSIONS), + default=DEFAULT_EXTENSIONS) + parser.add_argument( + '-r', + '--recursive', + action='store_true', + help='run recursively over directories') + parser.add_argument( + '-d', + '--dry-run', + action='store_true', + help='just print the list of files') + parser.add_argument( + '-i', + '--in-place', + action='store_true', + help='format file instead of printing differences') + parser.add_argument('files', metavar='file', nargs='+') + parser.add_argument( + '-q', + '--quiet', + action='store_true', + help="disable output, useful for the exit code") + parser.add_argument( + '-j', + metavar='N', + type=int, + default=0, + help='run N clang-format jobs in parallel' + ' (default number of cpus + 1)') + parser.add_argument( + '--color', + default='auto', + choices=['auto', 'always', 'never'], + help='show colored diff (default: auto)') + parser.add_argument( + '-e', + '--exclude', + metavar='PATTERN', + action='append', + default=[], + help='exclude paths matching the given glob-like pattern(s)' + ' from recursive search') + parser.add_argument( + '--style', + help='formatting style to apply (LLVM, Google, Chromium, Mozilla, WebKit)') + + args = parser.parse_args() + + # use default signal handling, like diff return SIGINT value on ^C + # https://bugs.python.org/issue14229#msg156446 + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + signal.SIGPIPE + except AttributeError: + # compatibility, SIGPIPE does not exist on Windows + pass + else: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + colored_stdout = False + colored_stderr = False + if args.color == 'always': + colored_stdout = True + colored_stderr = True + elif args.color == 'auto': + colored_stdout = sys.stdout.isatty() + colored_stderr = sys.stderr.isatty() + + version_invocation = [args.clang_format_executable, str("--version")] + try: + subprocess.check_call(version_invocation, stdout=DEVNULL) + except subprocess.CalledProcessError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + return ExitStatus.TROUBLE + except OSError as e: + print_trouble( + parser.prog, + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(version_invocation), e + ), + use_colors=colored_stderr, + ) + return ExitStatus.TROUBLE + + retcode = ExitStatus.SUCCESS + + excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE) + excludes.extend(args.exclude) + + files = list_files( + args.files, + recursive=args.recursive, + exclude=excludes, + extensions=args.extensions.split(',')) + + if not files: + return + + njobs = args.j + if njobs == 0: + njobs = multiprocessing.cpu_count() + 1 + njobs = min(len(files), njobs) + + if njobs == 1: + # execute directly instead of in a pool, + # less overhead, simpler stacktraces + it = (run_clang_format_diff_wrapper(args, file) for file in files) + pool = None + else: + pool = multiprocessing.Pool(njobs) + it = pool.imap_unordered( + partial(run_clang_format_diff_wrapper, args), files) + pool.close() + while True: + try: + outs, errs = next(it) + except StopIteration: + break + except DiffError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + retcode = ExitStatus.TROUBLE + sys.stderr.writelines(e.errs) + except UnexpectedError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + sys.stderr.write(e.formatted_traceback) + retcode = ExitStatus.TROUBLE + # stop at the first unexpected error, + # something could be very wrong, + # don't process all files unnecessarily + if pool: + pool.terminate() + break + else: + sys.stderr.writelines(errs) + if outs == []: + continue + if not args.quiet: + print_diff(outs, use_color=colored_stdout) + if retcode == ExitStatus.SUCCESS: + retcode = ExitStatus.DIFF + if pool: + pool.join() + return retcode + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docker/images/build-kit-base/.dockerignore b/docker/images/build-kit-base/.dockerignore new file mode 100644 index 0000000..c3f982e --- /dev/null +++ b/docker/images/build-kit-base/.dockerignore @@ -0,0 +1 @@ +doc.rst diff --git a/docker/images/build-kit-base/Dockerfile b/docker/images/build-kit-base/Dockerfile new file mode 100644 index 0000000..d43d73a --- /dev/null +++ b/docker/images/build-kit-base/Dockerfile @@ -0,0 +1,21 @@ +# syntax=docker/dockerfile:1 +ARG BASE_IMAGE_TAG=latest +FROM ghcr.io/everest/everest-ci/build-env-base:${BASE_IMAGE_TAG} + +ENV WORKSPACE_PATH=/workspace +ENV ASSETS_PATH=/assets +ARG EXT_MOUNT=/ext +ENV EXT_MOUNT=$EXT_MOUNT + +RUN mkdir $ASSETS_PATH +COPY maven-settings.xml $ASSETS_PATH/ + +COPY ./entrypoint.sh / + +# Disable ownership checks +RUN git config --global --add safe.directory '*' + +WORKDIR $WORKSPACE_PATH + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["run-script", "init"] diff --git a/docker/images/build-kit-base/entrypoint.sh b/docker/images/build-kit-base/entrypoint.sh new file mode 100755 index 0000000..440b811 --- /dev/null +++ b/docker/images/build-kit-base/entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +setup_cache(){ + export CPM_SOURCE_CACHE=$EXT_MOUNT/cache/cpm + export CCACHE_DIR=$EXT_MOUNT/cache/ccache + + maven_cache_dir=$EXT_MOUNT/cache/m2-repo + + # we copy the settings file each time in case the $HOME folder changes + mkdir -p $HOME/.m2 || true + cp $ASSETS_PATH/maven-settings.xml $HOME/.m2/settings.xml + sed -i 's@MAVEN_CACHE_REPOSITORY@'"$maven_cache_dir"'@' $HOME/.m2/settings.xml +} + +# if first cmd is exec we're going to exec what follows +# otherwise check for script +run_script(){ + + if [ "$1" = 'run-script' ]; then + + if [ -z "$2" ]; then + echo "Error: missing script name for run-script mode" + exit 1 + fi + + init_script=$EXT_MOUNT/scripts/$2.sh + + if [ ! -f "$init_script" ]; then + echo "Error: no script found at $init_script" + # bash convention, 127 = not found + exit 127 + fi + + exec $init_script + fi + + # not the run-script mode + exec "$@" + +} + +setup_cache +run_script $@ + diff --git a/docker/images/build-kit-base/maven-settings.xml b/docker/images/build-kit-base/maven-settings.xml new file mode 100644 index 0000000..317aac6 --- /dev/null +++ b/docker/images/build-kit-base/maven-settings.xml @@ -0,0 +1,4 @@ + + MAVEN_CACHE_REPOSITORY + diff --git a/docker/images/dev-env-base/Dockerfile b/docker/images/dev-env-base/Dockerfile new file mode 100644 index 0000000..3ee8e53 --- /dev/null +++ b/docker/images/dev-env-base/Dockerfile @@ -0,0 +1,53 @@ +# syntax=docker/dockerfile:1 +ARG BASE_IMAGE_TAG=latest +FROM ghcr.io/everest/everest-ci/build-env-base:${BASE_IMAGE_TAG} + +ARG USERNAME=docker +ARG USER_UID=1000 +RUN useradd -ms /bin/bash -u ${USER_UID} -U ${USERNAME} +# Extend the timeout +RUN mkdir -p /etc/apt/apt.conf.d/ \ + && echo 'Acquire::http::Timeout "100";' >> /etc/apt/apt.conf.d/99stahp_stahping \ + && echo 'Acquire::ftp::Timeout "100";' >> /etc/apt/apt.conf.d/99stahp_stahping \ + && echo 'Acquire::Retries "10";' >> /etc/apt/apt.conf.d/99stahp_stahping +RUN apt update \ + && apt install --no-install-recommends -y sudo \ + && echo ${USERNAME} ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/${USERNAME} \ + && chmod 0440 /etc/sudoers.d/${USERNAME} + +# Additional Build Tools +# Additional Build Tool - Documentation +RUN apt update && apt install --no-install-recommends -y \ + python3-sphinx=5.3.0-4 +RUN python3 -m pip install --break-system-packages \ + # Sphinx extensions + sphinxcontrib-contentui==0.2.5 \ + sphinxcontrib-svg2pdfconverter==1.2.2 + +# Development Tools +# Development Tools - General +RUN apt update \ + && apt install --no-install-recommends -y \ + wget=1.21.3-1+b2 \ + doxygen=1.9.4-4 \ + graphviz=2.42.2-7+b3 \ + build-essential=12.9 \ + cppcheck=2.10-2 \ + bash-completion=1:2.11-6 +# Development Tools - Documentation +RUN python3 -m pip install --break-system-packages \ + # language server RST/Sphinx + esbonio==0.16.4 \ + # Style checker for RST/Sphinx + doc8==1.1.1 + +RUN apt clean \ + && rm -rf /var/lib/apt/lists/* + +USER ${USERNAME} + +RUN mkdir ~/.ssh \ + && ssh-keyscan github.com > ~/.ssh/known_hosts + +ENV PATH="/home/$USERNAME/.local/bin:$PATH" +ENV CPM_SOURCE_CACHE="/home/$USERNAME/.cache/CPM" diff --git a/docker/images/everest-clang-format/Dockerfile b/docker/images/everest-clang-format/Dockerfile index 4e78acd..736e718 100644 --- a/docker/images/everest-clang-format/Dockerfile +++ b/docker/images/everest-clang-format/Dockerfile @@ -1,15 +1,4 @@ -ARG BASE_IMAGE=debian:12 -FROM ${BASE_IMAGE} - -RUN apt update && apt upgrade -y -RUN apt update && \ - apt install -y \ - clang-format-15=1:15.0.6* -RUN ln -s /usr/bin/clang-format-15 /usr/bin/clang-format - -RUN apt install -y \ - python-is-python3 - -COPY run-clang-format.py /usr/bin/run-clang-format +ARG BASE_IMAGE_TAG=latest +FROM ghcr.io/everest/everest-ci/build-env-base:${BASE_IMAGE_TAG} ENTRYPOINT ["/usr/bin/run-clang-format"] diff --git a/docker/images/run-env-base/Dockerfile b/docker/images/run-env-base/Dockerfile new file mode 100644 index 0000000..85ae6a9 --- /dev/null +++ b/docker/images/run-env-base/Dockerfile @@ -0,0 +1,20 @@ +# syntax=docker/dockerfile:1 +FROM debian:12-slim + +RUN apt update \ + && apt install --no-install-recommends -y \ + openjdk-17-jre=17.0.11+9-1~deb12u1 \ + nodejs=18.19.0+dfsg-6~deb12u2 \ + npm=9.2.0~ds1-1 \ + python3-pip=23.0.1+dfsg-1 \ + sqlite3=3.40.1-2 \ + libboost-program-options1.74.0=1.74.0+ds1-21 \ + libboost-log1.74.0=1.74.0+ds1-21 \ + libboost-chrono1.74.0=1.74.0+ds1-21 \ + libboost-system1.74.0=1.74.0+ds1-21 \ + libssl3=3.0.13-1~deb12u1 \ + libcurl4=7.88.1-10+deb12u5 \ + libcap2=1:2.66-4 \ + less=590-2.1~deb12u2 \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/github-actions/run-clang-format/action.yaml b/github-actions/run-clang-format/action.yaml index 58ec5a2..5025362 100644 --- a/github-actions/run-clang-format/action.yaml +++ b/github-actions/run-clang-format/action.yaml @@ -19,7 +19,7 @@ inputs: default: 'always' runs: using: 'docker' - image: 'docker://ghcr.io/everest/everest-clang-format:v1.1.0' + image: 'docker://ghcr.io/everest/everest-ci/everest-clang-format:refactor-docker-images' # LTODO: Update this args: - /github/workspace/${{ inputs.source-dir }} - --extensions