From c9c63bd62a8b60301db9eec46cf8bda75a05c44d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 5 Nov 2024 17:52:23 +0100 Subject: [PATCH 01/62] chore: run lint in a separate build before running the tests --- .github/workflows/ci-lint.yml | 49 ++++++++++++++++++++++++++++++++ .github/workflows/ci-test-go.yml | 22 -------------- .github/workflows/ci.yml | 36 +++++++++++++++++++++-- 3 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/ci-lint.yml diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml new file mode 100644 index 0000000000..b144fd9e56 --- /dev/null +++ b/.github/workflows/ci-lint.yml @@ -0,0 +1,49 @@ +name: Run golangci-lint for a Go project +run-name: "${{ inputs.project-directory }}" + +on: + workflow_call: + inputs: + project-directory: + required: true + type: string + default: "." + description: "The directory where the Go project is located." + +permissions: + contents: read + +jobs: + test-go-project: + name: "${{ inputs.project-directory }}" + runs-on: "ubuntu-latest" + continue-on-error: false + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + with: + go-version-file: '${{ inputs.project-directory }}/go.mod' + cache-dependency-path: '${{ inputs.project-directory }}/go.sum' + id: go + + - name: golangci-lint + uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.61.0 + # Optional: working directory, useful for monorepos + working-directory: ${{ inputs.project-directory }} + # Optional: golangci-lint command line arguments. + args: --verbose + # Optional: if set to true then the all caching functionality will be complete disabled, + # takes precedence over all other caching options. + skip-cache: true + + - name: generate + shell: bash + run: | + make generate + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index a76df75d4d..7a81166d73 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -71,28 +71,6 @@ jobs: cache-dependency-path: '${{ inputs.project-directory }}/go.sum' id: go - - name: golangci-lint - if: ${{ inputs.platform == 'ubuntu-latest' }} - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 - with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.61.0 - # Optional: working directory, useful for monorepos - working-directory: ${{ inputs.project-directory }} - # Optional: golangci-lint command line arguments. - args: --verbose - # Optional: if set to true then the all caching functionality will be complete disabled, - # takes precedence over all other caching options. - skip-cache: true - - - name: generate - if: ${{ inputs.platform == 'ubuntu-latest' }} - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make generate - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - - name: modVerify working-directory: ./${{ inputs.project-directory }} run: go mod verify diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfb4dc273b..09554171be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,13 @@ concurrency: cancel-in-progress: true jobs: + lint: + uses: ./.github/workflows/ci-lint.yml + with: + project-directory: "." + test: + needs: lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -42,6 +48,7 @@ jobs: # It's executed in the first stage to avoid concurrency issues. test-reaper-off: name: "Test with reaper off" + needs: lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -59,6 +66,7 @@ jobs: # It's executed in the first stage to avoid concurrency issues. test-rootless-docker: name: "Test with Rootless Docker" + needs: lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -73,7 +81,13 @@ jobs: run-tests: true ryuk-disabled: false + lint-module-generator: + uses: ./.github/workflows/ci-lint.yml + with: + project-directory: "modulegen" + test-module-generator: + needs: lint-module-generator strategy: matrix: go-version: [1.22.x, 1.x] @@ -88,8 +102,17 @@ jobs: run-tests: true ryuk-disabled: false - test-modules: + lint-modules: needs: test + strategy: + matrix: + module: [artemis, azurite, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, databend, dolt, dynamodb, elasticsearch, etcd, gcloud, grafana-lgtm, inbucket, influxdb, k3s, k6, kafka, localstack, mariadb, meilisearch, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, registry, surrealdb, valkey, vault, vearch, weaviate, yugabytedb] + uses: ./.github/workflows/ci-lint.yml + with: + project-directory: "modules/${{ matrix.module }}" + + test-modules: + needs: lint-modules strategy: matrix: go-version: [1.22.x, 1.x] @@ -105,8 +128,17 @@ jobs: run-tests: ${{ matrix.platform == 'ubuntu-latest' }} ryuk-disabled: false - test-examples: + lint-examples: needs: test-modules + strategy: + matrix: + module: [nginx, toxiproxy] + uses: ./.github/workflows/ci-lint.yml + with: + project-directory: "modules/${{ matrix.module }}" + + test-examples: + needs: lint-examples strategy: matrix: module: [nginx, toxiproxy] From 4bab0fd9b4a4329be826d52dc863b4b2601f24d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 5 Nov 2024 19:30:54 +0100 Subject: [PATCH 02/62] fix: lint work dir --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09554171be..aa5b25b192 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: module: [nginx, toxiproxy] uses: ./.github/workflows/ci-lint.yml with: - project-directory: "modules/${{ matrix.module }}" + project-directory: "examples/${{ matrix.module }}" test-examples: needs: lint-examples From 2f6d74ef0422a9ef0fbd5e0536fd5b93d71ad025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 6 Nov 2024 16:30:52 +0100 Subject: [PATCH 03/62] Revert "fix: lint work dir" This reverts commit 4bab0fd9b4a4329be826d52dc863b4b2601f24d4. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa5b25b192..09554171be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: module: [nginx, toxiproxy] uses: ./.github/workflows/ci-lint.yml with: - project-directory: "examples/${{ matrix.module }}" + project-directory: "modules/${{ matrix.module }}" test-examples: needs: lint-examples From 77b8c25f37688b99280275dabe2370c0d5ea5182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 6 Nov 2024 16:30:59 +0100 Subject: [PATCH 04/62] Revert "chore: run lint in a separate build before running the tests" This reverts commit c9c63bd62a8b60301db9eec46cf8bda75a05c44d. --- .github/workflows/ci-lint.yml | 49 -------------------------------- .github/workflows/ci-test-go.yml | 22 ++++++++++++++ .github/workflows/ci.yml | 36 ++--------------------- 3 files changed, 24 insertions(+), 83 deletions(-) delete mode 100644 .github/workflows/ci-lint.yml diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml deleted file mode 100644 index b144fd9e56..0000000000 --- a/.github/workflows/ci-lint.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Run golangci-lint for a Go project -run-name: "${{ inputs.project-directory }}" - -on: - workflow_call: - inputs: - project-directory: - required: true - type: string - default: "." - description: "The directory where the Go project is located." - -permissions: - contents: read - -jobs: - test-go-project: - name: "${{ inputs.project-directory }}" - runs-on: "ubuntu-latest" - continue-on-error: false - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Set up Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 - with: - go-version-file: '${{ inputs.project-directory }}/go.mod' - cache-dependency-path: '${{ inputs.project-directory }}/go.sum' - id: go - - - name: golangci-lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 - with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.61.0 - # Optional: working directory, useful for monorepos - working-directory: ${{ inputs.project-directory }} - # Optional: golangci-lint command line arguments. - args: --verbose - # Optional: if set to true then the all caching functionality will be complete disabled, - # takes precedence over all other caching options. - skip-cache: true - - - name: generate - shell: bash - run: | - make generate - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 7a81166d73..a76df75d4d 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -71,6 +71,28 @@ jobs: cache-dependency-path: '${{ inputs.project-directory }}/go.sum' id: go + - name: golangci-lint + if: ${{ inputs.platform == 'ubuntu-latest' }} + uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.61.0 + # Optional: working directory, useful for monorepos + working-directory: ${{ inputs.project-directory }} + # Optional: golangci-lint command line arguments. + args: --verbose + # Optional: if set to true then the all caching functionality will be complete disabled, + # takes precedence over all other caching options. + skip-cache: true + + - name: generate + if: ${{ inputs.platform == 'ubuntu-latest' }} + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make generate + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] + - name: modVerify working-directory: ./${{ inputs.project-directory }} run: go mod verify diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09554171be..dfb4dc273b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,13 +23,7 @@ concurrency: cancel-in-progress: true jobs: - lint: - uses: ./.github/workflows/ci-lint.yml - with: - project-directory: "." - test: - needs: lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -48,7 +42,6 @@ jobs: # It's executed in the first stage to avoid concurrency issues. test-reaper-off: name: "Test with reaper off" - needs: lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -66,7 +59,6 @@ jobs: # It's executed in the first stage to avoid concurrency issues. test-rootless-docker: name: "Test with Rootless Docker" - needs: lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -81,13 +73,7 @@ jobs: run-tests: true ryuk-disabled: false - lint-module-generator: - uses: ./.github/workflows/ci-lint.yml - with: - project-directory: "modulegen" - test-module-generator: - needs: lint-module-generator strategy: matrix: go-version: [1.22.x, 1.x] @@ -102,17 +88,8 @@ jobs: run-tests: true ryuk-disabled: false - lint-modules: - needs: test - strategy: - matrix: - module: [artemis, azurite, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, databend, dolt, dynamodb, elasticsearch, etcd, gcloud, grafana-lgtm, inbucket, influxdb, k3s, k6, kafka, localstack, mariadb, meilisearch, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, registry, surrealdb, valkey, vault, vearch, weaviate, yugabytedb] - uses: ./.github/workflows/ci-lint.yml - with: - project-directory: "modules/${{ matrix.module }}" - test-modules: - needs: lint-modules + needs: test strategy: matrix: go-version: [1.22.x, 1.x] @@ -128,17 +105,8 @@ jobs: run-tests: ${{ matrix.platform == 'ubuntu-latest' }} ryuk-disabled: false - lint-examples: - needs: test-modules - strategy: - matrix: - module: [nginx, toxiproxy] - uses: ./.github/workflows/ci-lint.yml - with: - project-directory: "modules/${{ matrix.module }}" - test-examples: - needs: lint-examples + needs: test-modules strategy: matrix: module: [nginx, toxiproxy] From eaef8e1845840299f2ea2ba871a719265f27c8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 11:19:40 +0100 Subject: [PATCH 05/62] chore: detect modified files and select which module to run the tests based on that --- .github/workflows/ci-test-go.yml | 16 ++---- .github/workflows/ci.yml | 85 ++++++++++++++------------------ scripts/changed-modules.sh | 65 ++++++++++++++++++++++++ scripts/list-modules.sh | 41 +++++++++++++++ 4 files changed, 146 insertions(+), 61 deletions(-) create mode 100755 scripts/changed-modules.sh create mode 100755 scripts/list-modules.sh diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index a76df75d4d..1b0aee6c41 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -27,11 +27,6 @@ on: type: boolean default: false description: "Run the test with rootless docker." - run-tests: - required: false - type: boolean - default: true - description: "Run the tests under conditions controlled by the caller workflow." ryuk-disabled: required: false type: boolean @@ -46,7 +41,7 @@ permissions: jobs: test-go-project: name: "${{ inputs.project-directory }}/${{ inputs.platform }}/${{ inputs.go-version }}" - runs-on: ${{ inputs.platform }} + runs-on: [ "${{ inputs.platform }}" ] continue-on-error: ${{ !inputs.fail-fast }} env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" @@ -110,17 +105,14 @@ jobs: run: go build - name: go test - # only run tests on linux, there are a number of things that won't allow the tests to run on anything else - # many (maybe, all?) images used can only be build on Linux, they don't have Windows in their manifest, and - # we can't put Windows Server in "Linux Mode" in Github actions - # another, host mode is only available on Linux, and we have tests around that, do we skip them? - if: ${{ inputs.run-tests }} + # for the core, only run the tests on Linux + if: ${{ (endsWith(inputs.project-directory, 'testcontainers-go') && matrix.platform != 'ubuntu-latest') && false || true }} working-directory: ./${{ inputs.project-directory }} timeout-minutes: 30 run: make test-unit - name: Upload SonarCloud files - if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && inputs.run-tests && !inputs.rootless-docker }} + if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && matrix.platform == 'ubuntu-latest' && !inputs.rootless-docker }} uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3 with: name: sonarcloud diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfb4dc273b..6fb7966049 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,36 +6,58 @@ on: push: branches: - main - paths-ignore: - - '.vscode/**' - - 'mkdocs.yml' - - 'docs/**' - - 'README.md' pull_request: - paths-ignore: - - '.vscode/**' - - 'mkdocs.yml' - - 'docs/**' - - 'README.md' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} cancel-in-progress: true +env: + # if the module is the core, testcontainers-go run the tests on linux and macos + # for modulegen, run in all the platforms + # else, a module or an example, run on linux + CORE_PLATFORMS: "ubuntu-latest, macos-latest" + MODULEGEN_PLATFORMS: "ubuntu-latest, macos-latest, windows-latest" + MODULES_PLATFORMS: "ubuntu-latest" + jobs: + detect-modules: + runs-on: ubuntu-latest + outputs: + modules: ${{ steps.set-modified-modules.outputs.modules }} + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - id: set-available-modules + name: "Detect the available modules" + run: echo "modules=$(./scripts/list-modules.sh)" >> $GITHUB_OUTPUT + + - id: changed-files + name: Get changed files + uses: tj-actions/changed-files@v45 + + - id: set-modified-modules + name: Set all modified modules + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + ALL_AVAILABLE_MODULES: ${{ steps.set-available-modules.outputs.modules }} + run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + test: + needs: detect-modules strategy: matrix: go-version: [1.22.x, 1.x] platform: [ubuntu-latest, macos-latest] + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} uses: ./.github/workflows/ci-test-go.yml with: go-version: ${{ matrix.go-version }} fail-fast: true - platform: ${{ matrix.platform }} - project-directory: "." + platform: ${{ endsWith(matrix.module, 'testcontainers-go') && env.CORE_PLATFORMS || (endsWith(matrix.module, 'modulegen') && env.MODULEGEN_PLATFORMS || env.MODULES_PLATFORMS) }} + project-directory: "${{ matrix.module }}" rootless-docker: false - run-tests: ${{ matrix.platform == 'ubuntu-latest' }} ryuk-disabled: false # The job below is a copy of the job above, but with ryuk disabled. @@ -52,7 +74,6 @@ jobs: platform: "ubuntu-latest" project-directory: "." rootless-docker: false - run-tests: true ryuk-disabled: true # The job below is a copy of the job above, but with Docker rootless. @@ -70,7 +91,6 @@ jobs: platform: "ubuntu-latest" project-directory: "." rootless-docker: true - run-tests: true ryuk-disabled: false test-module-generator: @@ -85,39 +105,6 @@ jobs: platform: ${{ matrix.platform }} project-directory: "modulegen" rootless-docker: false - run-tests: true - ryuk-disabled: false - - test-modules: - needs: test - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest] - module: [artemis, azurite, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, databend, dolt, dynamodb, elasticsearch, etcd, gcloud, grafana-lgtm, inbucket, influxdb, k3s, k6, kafka, localstack, mariadb, meilisearch, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, registry, surrealdb, valkey, vault, vearch, weaviate, yugabytedb] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: ${{ matrix.go-version }} - fail-fast: false - platform: ${{ matrix.platform }} - project-directory: modules/${{ matrix.module }} - rootless-docker: false - run-tests: ${{ matrix.platform == 'ubuntu-latest' }} - ryuk-disabled: false - - test-examples: - needs: test-modules - strategy: - matrix: - module: [nginx, toxiproxy] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: "1.22.x" - fail-fast: true - platform: 'ubuntu-latest' - project-directory: examples/${{ matrix.module }} - rootless-docker: false - run-tests: true ryuk-disabled: false sonarcloud: @@ -125,7 +112,7 @@ jobs: contents: read # for actions/checkout to fetch code pull-requests: read # for sonarsource/sonarcloud-github-action to determine which PR to decorate if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' }} - needs: test-examples + needs: test runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory diff --git a/scripts/changed-modules.sh b/scripts/changed-modules.sh new file mode 100755 index 0000000000..b9d92a4032 --- /dev/null +++ b/scripts/changed-modules.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# How to test this script, run it with the required environment variables: +# 1. A Go file from the core module is modified: +# ALL_CHANGED_FILES="examples/nginx/go.mod, examples/foo/a.txt, a/b/c/d/a.go" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# The output should be: all modules. +# +# 2. A file from a module in the modules dir is modified: +# ALL_CHANGED_FILES="modules/nginx/go.mod" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# The output should be: just the modules/nginx module. +# +# 3. A file from a module in the examples dir is modified: +# ALL_CHANGED_FILES="examples/nginx/go.mod" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# The output should be: just the examples/nginx module. +# +# 4. A Go file from the modulegen dir is modified: +# ALL_CHANGED_FILES="modulegen/a.go" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# The output should be: just the modulegen module. +# +# 5. A non-Go file from the core dir is modified: +# ALL_CHANGED_FILES="docs/README.md" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# The output should be: all modules. +# +# There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. +# But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. + +# ROOT_DIR is the root directory of the repository. +readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +# Get the list of modified files +readonly modified_files=${ALL_CHANGED_FILES[@]} + +# Initialize variables +modified_modules=() + +# Check the modified files and determine which modules to build, following these rules: +# - if the modified files contain any file in the root module, include all modules in the list +# - if the modified files only contain files in one of the modules, include that module in the list +# - if the modified files only contain files in one of the examples, include that example in the list +# - if the modified files only contain files in the modulegen module, include only the modulegen module in the list +for file in $modified_files; do + if [[ $file == modules/* ]]; then + module_name=$(echo $file | cut -d'/' -f2) + if [[ ! " ${modified_modules[@]} " =~ " ${module_name} " ]]; then + modified_modules+=("\"${ROOT_DIR}/modules/$module_name\"") + fi + elif [[ $file == examples/* ]]; then + example_name=$(echo $file | cut -d'/' -f2) + if [[ ! " ${modified_modules[@]} " =~ " ${example_name} " ]]; then + modified_modules+=("\"${ROOT_DIR}/examples/$example_name\"") + fi + elif [[ $file == modulegen/* ]]; then + modified_modules+=("\"${ROOT_DIR}/modulegen\"") + else + # a file from the core module is modified, so include all modules in the list and stop the loop + modified_modules=${ALL_AVAILABLE_MODULES[@]} + break + fi +done + +# print all modules with this format: +# each module will be enclosed in double quotes +# each module will be separated by a comma +# the entire list will be enclosed in square brackets +echo "["$(IFS=,; echo "${modified_modules[*]}" | sed 's/ /,/g')"]" diff --git a/scripts/list-modules.sh b/scripts/list-modules.sh new file mode 100755 index 0000000000..4803974f7a --- /dev/null +++ b/scripts/list-modules.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Find all go.mod files in the repository, building a list of all the available modules. + +# ROOT_DIR is the root directory of the repository. +readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +# modules is an array that will store the paths of all the modules in the repository. +modules=() + +# capture modules +for modFile in $(find "${ROOT_DIR}/modules" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do + modules+=("\"${ROOT_DIR}/modules/$(basename "$(dirname "${modFile}")")\"") +done + +# capture examples +for modFile in $(find "${ROOT_DIR}/examples" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do + modules+=("\"${ROOT_DIR}/examples/$(basename "$(dirname "${modFile}")")\"") +done + +# sort modules array +IFS=$'\n' modules=($(sort <<<"${modules[*]}")) +unset IFS + +# capture the root module +rootModule="\"${ROOT_DIR}\"" + +# capture the modulegen module +modulegenModule="\"${ROOT_DIR}/modulegen\"" + +# merge all modules and examples into a single array +readonly allModules=("${rootModule}" "${modulegenModule}" "${modules[@]}") + +# sort modified_modules array +IFS=$'\n' modified_modules=($(sort <<<"${modified_modules[*]}")) + +# print all modules with this format: +# each module will be enclosed in double quotes +# each module will be separated by a comma +# the entire list will be enclosed in square brackets +echo "["$(IFS=,; echo "${allModules[*]}" | sed 's/ /,/g')"]" From cfcec392092c3b0f2b325c6dab5a5f6326d2ab42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 11:21:53 +0100 Subject: [PATCH 06/62] fix: read env var properly --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb7966049..88bbfcde4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: true - platform: ${{ endsWith(matrix.module, 'testcontainers-go') && env.CORE_PLATFORMS || (endsWith(matrix.module, 'modulegen') && env.MODULEGEN_PLATFORMS || env.MODULES_PLATFORMS) }} + platform: ${{ endsWith(matrix.module, 'testcontainers-go') && $CORE_PLATFORMS || (endsWith(matrix.module, 'modulegen') && $MODULEGEN_PLATFORMS || $MODULES_PLATFORMS) }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false From 4a4482fd958d6e0cfa905cb30bafe6a30a5b1f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 11:25:07 +0100 Subject: [PATCH 07/62] fix: simplify --- .github/workflows/ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88bbfcde4d..b93bc09cd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,14 +12,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} cancel-in-progress: true -env: - # if the module is the core, testcontainers-go run the tests on linux and macos - # for modulegen, run in all the platforms - # else, a module or an example, run on linux - CORE_PLATFORMS: "ubuntu-latest, macos-latest" - MODULEGEN_PLATFORMS: "ubuntu-latest, macos-latest, windows-latest" - MODULES_PLATFORMS: "ubuntu-latest" - jobs: detect-modules: runs-on: ubuntu-latest @@ -55,7 +47,10 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: true - platform: ${{ endsWith(matrix.module, 'testcontainers-go') && $CORE_PLATFORMS || (endsWith(matrix.module, 'modulegen') && $MODULEGEN_PLATFORMS || $MODULES_PLATFORMS) }} + # if the module is the core, testcontainers-go run the tests on linux and macos + # for modulegen, run in all the platforms + # else, a module or an example, run on linux + platform: ${{ endsWith(matrix.module, 'testcontainers-go') && "ubuntu-latest, macos-latest" || (endsWith(matrix.module, 'modulegen') && "ubuntu-latest, macos-latest, windows-latest" || "ubuntu-latest") }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false From 6550d8e2298648a2c4fdbe88a384af448eb3c5cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 11:26:49 +0100 Subject: [PATCH 08/62] fix: single quotes --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b93bc09cd8..d96b82a69a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: # if the module is the core, testcontainers-go run the tests on linux and macos # for modulegen, run in all the platforms # else, a module or an example, run on linux - platform: ${{ endsWith(matrix.module, 'testcontainers-go') && "ubuntu-latest, macos-latest" || (endsWith(matrix.module, 'modulegen') && "ubuntu-latest, macos-latest, windows-latest" || "ubuntu-latest") }} + platform: ${{ endsWith(matrix.module, 'testcontainers-go') && 'ubuntu-latest, macos-latest' || (endsWith(matrix.module, 'modulegen') && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest') }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false From 93934ac105477030b9bbae1a4f90bc9b041aa45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 11:27:54 +0100 Subject: [PATCH 09/62] chore: remove job --- .github/workflows/ci.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d96b82a69a..0ed78698a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,20 +88,6 @@ jobs: rootless-docker: true ryuk-disabled: false - test-module-generator: - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest, macos-latest, windows-latest] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: ${{ matrix.go-version }} - fail-fast: true - platform: ${{ matrix.platform }} - project-directory: "modulegen" - rootless-docker: false - ryuk-disabled: false - sonarcloud: permissions: contents: read # for actions/checkout to fetch code From 2e35ae8d07bf2423f5df01e356d7d21e3d60761c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 11:46:32 +0100 Subject: [PATCH 10/62] fix: proper all_modified_files quotes --- .github/workflows/ci.yml | 2 +- scripts/changed-modules.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ed78698a9..d03156d43c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: - id: set-modified-modules name: Set all modified modules env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" ALL_AVAILABLE_MODULES: ${{ steps.set-available-modules.outputs.modules }} run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT diff --git a/scripts/changed-modules.sh b/scripts/changed-modules.sh index b9d92a4032..17a1417229 100755 --- a/scripts/changed-modules.sh +++ b/scripts/changed-modules.sh @@ -2,7 +2,7 @@ # How to test this script, run it with the required environment variables: # 1. A Go file from the core module is modified: -# ALL_CHANGED_FILES="examples/nginx/go.mod, examples/foo/a.txt, a/b/c/d/a.go" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# ALL_CHANGED_FILES="examples/nginx/go.mod examples/foo/a.txt a/b/c/d/a.go" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh # The output should be: all modules. # # 2. A file from a module in the modules dir is modified: @@ -21,6 +21,10 @@ # ALL_CHANGED_FILES="docs/README.md" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh # The output should be: all modules. # +# 6. A file from two modules in the modules dir are modified: +# ALL_CHANGED_FILES="modules/nginx/go.mod modules/localstack/go.mod" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# The output should be: the modules/nginx and modules/localstack modules. +# # There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. # But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. From 87d9718a20bea7956f109471becc5cac38f3f10d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 11:51:23 +0100 Subject: [PATCH 11/62] chore: print out modules to run --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d03156d43c..3c78c36137 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,9 @@ jobs: ALL_AVAILABLE_MODULES: ${{ steps.set-available-modules.outputs.modules }} run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + - name: Print out the modules to be used + run: echo "${{ steps.set-modified-modules.outputs.modules }}[*]" + test: needs: detect-modules strategy: From 4b6331226ee3b0397f551b8a2404520c812f4499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 12:04:35 +0100 Subject: [PATCH 12/62] fix: do not send an array with all modules, but a comma-separated list instead --- scripts/list-modules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/list-modules.sh b/scripts/list-modules.sh index 4803974f7a..8bed36abe9 100755 --- a/scripts/list-modules.sh +++ b/scripts/list-modules.sh @@ -38,4 +38,4 @@ IFS=$'\n' modified_modules=($(sort <<<"${modified_modules[*]}")) # each module will be enclosed in double quotes # each module will be separated by a comma # the entire list will be enclosed in square brackets -echo "["$(IFS=,; echo "${allModules[*]}" | sed 's/ /,/g')"]" +echo "$(IFS=,; echo "${allModules[*]}" | sed 's/ /,/g')" From 1c17f59075966060d5aff9e978f6d94c65b8f872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 13:02:25 +0100 Subject: [PATCH 13/62] fix: do not prepend the root_dir for the modules --- scripts/changed-modules.sh | 9 +++------ scripts/list-modules.sh | 8 ++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/scripts/changed-modules.sh b/scripts/changed-modules.sh index 17a1417229..8cb19cfb2f 100755 --- a/scripts/changed-modules.sh +++ b/scripts/changed-modules.sh @@ -28,9 +28,6 @@ # There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. # But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. -# ROOT_DIR is the root directory of the repository. -readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) - # Get the list of modified files readonly modified_files=${ALL_CHANGED_FILES[@]} @@ -46,15 +43,15 @@ for file in $modified_files; do if [[ $file == modules/* ]]; then module_name=$(echo $file | cut -d'/' -f2) if [[ ! " ${modified_modules[@]} " =~ " ${module_name} " ]]; then - modified_modules+=("\"${ROOT_DIR}/modules/$module_name\"") + modified_modules+=("\"modules/$module_name\"") fi elif [[ $file == examples/* ]]; then example_name=$(echo $file | cut -d'/' -f2) if [[ ! " ${modified_modules[@]} " =~ " ${example_name} " ]]; then - modified_modules+=("\"${ROOT_DIR}/examples/$example_name\"") + modified_modules+=("\"examples/$example_name\"") fi elif [[ $file == modulegen/* ]]; then - modified_modules+=("\"${ROOT_DIR}/modulegen\"") + modified_modules+=("\"modulegen\"") else # a file from the core module is modified, so include all modules in the list and stop the loop modified_modules=${ALL_AVAILABLE_MODULES[@]} diff --git a/scripts/list-modules.sh b/scripts/list-modules.sh index 8bed36abe9..c87fd99cbd 100755 --- a/scripts/list-modules.sh +++ b/scripts/list-modules.sh @@ -10,12 +10,12 @@ modules=() # capture modules for modFile in $(find "${ROOT_DIR}/modules" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do - modules+=("\"${ROOT_DIR}/modules/$(basename "$(dirname "${modFile}")")\"") + modules+=("\"modules/$(basename "$(dirname "${modFile}")")\"") done # capture examples for modFile in $(find "${ROOT_DIR}/examples" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do - modules+=("\"${ROOT_DIR}/examples/$(basename "$(dirname "${modFile}")")\"") + modules+=("\"examples/$(basename "$(dirname "${modFile}")")\"") done # sort modules array @@ -23,10 +23,10 @@ IFS=$'\n' modules=($(sort <<<"${modules[*]}")) unset IFS # capture the root module -rootModule="\"${ROOT_DIR}\"" +rootModule="\"\"" # capture the modulegen module -modulegenModule="\"${ROOT_DIR}/modulegen\"" +modulegenModule="\"modulegen\"" # merge all modules and examples into a single array readonly allModules=("${rootModule}" "${modulegenModule}" "${modules[@]}") From f68e6aff8410e16f7fba8c48177b14d1f6e026ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 13:47:05 +0100 Subject: [PATCH 14/62] chore: proper runner definition --- .github/workflows/ci-test-go.yml | 64 ++++++++++++++++++-------------- .github/workflows/ci.yml | 7 ---- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 1b0aee6c41..602b9795bb 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -1,5 +1,5 @@ name: Run tests for a Go project -run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platform }}" +run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }}" on: workflow_call: @@ -8,10 +8,6 @@ on: required: true type: string description: "The version of Go to use for the test." - platform: - required: true - type: string - description: "The platform to run the test on." fail-fast: required: false type: boolean @@ -39,23 +35,11 @@ permissions: # pull-requests: read jobs: - test-go-project: - name: "${{ inputs.project-directory }}/${{ inputs.platform }}/${{ inputs.go-version }}" - runs-on: [ "${{ inputs.platform }}" ] - continue-on-error: ${{ !inputs.fail-fast }} - env: - TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" - RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}" - RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" + lint-go-project: + name: "lint: ${{ inputs.project-directory }}/${{ inputs.go-version }}" + runs-on: 'ubuntu-latest' + continue-on-error: false steps: - - name: Setup rootless Docker - if: ${{ inputs.rootless-docker }} - uses: ScribeMD/rootless-docker@6bd157a512c2fafa4e0243a8aa87d964eb890886 # v0.2.2 - - - name: Remove Docker root socket - if: ${{ inputs.rootless-docker }} - run: sudo rm -rf /var/run/docker.sock - - name: Check out code into the Go module directory uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 @@ -67,7 +51,6 @@ jobs: id: go - name: golangci-lint - if: ${{ inputs.platform == 'ubuntu-latest' }} uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version @@ -81,38 +64,63 @@ jobs: skip-cache: true - name: generate - if: ${{ inputs.platform == 'ubuntu-latest' }} working-directory: ./${{ inputs.project-directory }} shell: bash run: | make generate git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - + - name: modVerify working-directory: ./${{ inputs.project-directory }} run: go mod verify - name: modTidy - if: ${{ inputs.platform == 'ubuntu-latest' }} working-directory: ./${{ inputs.project-directory }} shell: bash run: | make tidy git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] + test-go-project: + needs: lint-go-project + name: "test: ${{ inputs.project-directory }}/${{ runner.name }}/${{ inputs.go-version }}" + # Modulegen can run the tests on all platforms + runs-on: [ "${{ inputs.project-directory == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest' }}" ] + continue-on-error: ${{ !inputs.fail-fast }} + env: + TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" + RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}" + RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" + steps: + - name: Setup rootless Docker + if: ${{ inputs.rootless-docker }} + uses: ScribeMD/rootless-docker@6bd157a512c2fafa4e0243a8aa87d964eb890886 # v0.2.2 + + - name: Remove Docker root socket + if: ${{ inputs.rootless-docker }} + run: sudo rm -rf /var/run/docker.sock + + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + with: + go-version: '${{ inputs.go-version }}' + cache-dependency-path: '${{ inputs.project-directory }}/go.sum' + id: go + - name: ensure compilation working-directory: ./${{ inputs.project-directory }} run: go build - name: go test - # for the core, only run the tests on Linux - if: ${{ (endsWith(inputs.project-directory, 'testcontainers-go') && matrix.platform != 'ubuntu-latest') && false || true }} working-directory: ./${{ inputs.project-directory }} timeout-minutes: 30 run: make test-unit - name: Upload SonarCloud files - if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && matrix.platform == 'ubuntu-latest' && !inputs.rootless-docker }} + if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && runner.name == 'ubuntu-latest' && !inputs.rootless-docker }} uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3 with: name: sonarcloud diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c78c36137..0e338d6a15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,16 +44,11 @@ jobs: strategy: matrix: go-version: [1.22.x, 1.x] - platform: [ubuntu-latest, macos-latest] module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} uses: ./.github/workflows/ci-test-go.yml with: go-version: ${{ matrix.go-version }} fail-fast: true - # if the module is the core, testcontainers-go run the tests on linux and macos - # for modulegen, run in all the platforms - # else, a module or an example, run on linux - platform: ${{ endsWith(matrix.module, 'testcontainers-go') && 'ubuntu-latest, macos-latest' || (endsWith(matrix.module, 'modulegen') && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest') }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false @@ -69,7 +64,6 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false - platform: "ubuntu-latest" project-directory: "." rootless-docker: false ryuk-disabled: true @@ -86,7 +80,6 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false - platform: "ubuntu-latest" project-directory: "." rootless-docker: true ryuk-disabled: false From 3272c4355dbf72979764f0d64ddd0ecfb2b963aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 13:48:05 +0100 Subject: [PATCH 15/62] fix: use runner in the right place --- .github/workflows/ci-test-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 602b9795bb..4a4ff0d607 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -83,7 +83,7 @@ jobs: test-go-project: needs: lint-go-project - name: "test: ${{ inputs.project-directory }}/${{ runner.name }}/${{ inputs.go-version }}" + name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms runs-on: [ "${{ inputs.project-directory == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest' }}" ] continue-on-error: ${{ !inputs.fail-fast }} From 5e1f13ac725e9501a77c76abd7dd49786fb0ed3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 14:01:11 +0100 Subject: [PATCH 16/62] chore: separate lint to a different workflow --- .github/workflows/ci-lint-go.yml | 82 ++++++++++++++++++++++++++++++++ .github/workflows/ci-test-go.yml | 49 +------------------ .github/workflows/ci.yml | 18 ++++++- 3 files changed, 100 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/ci-lint-go.yml diff --git a/.github/workflows/ci-lint-go.yml b/.github/workflows/ci-lint-go.yml new file mode 100644 index 0000000000..ec61299e6d --- /dev/null +++ b/.github/workflows/ci-lint-go.yml @@ -0,0 +1,82 @@ +name: Run tests for a Go project +run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }}" + +on: + workflow_call: + inputs: + go-version: + required: true + type: string + description: "The version of Go to use for the test." + fail-fast: + required: false + type: boolean + default: true + description: "Fail the workflow if any of the jobs fail." + project-directory: + required: true + type: string + default: "." + description: "The directory where the Go project is located." + rootless-docker: + required: false + type: boolean + default: false + description: "Run the test with rootless docker." + ryuk-disabled: + required: false + type: boolean + default: false + description: "Disable the ryuk container for the test." + +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read + +jobs: + lint-go-project: + name: "lint: ${{ inputs.project-directory }}/${{ inputs.go-version }}" + runs-on: 'ubuntu-latest' + continue-on-error: false + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + with: + go-version: '${{ inputs.go-version }}' + cache-dependency-path: '${{ inputs.project-directory }}/go.sum' + id: go + + - name: golangci-lint + uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.61.0 + # Optional: working directory, useful for monorepos + working-directory: ${{ inputs.project-directory }} + # Optional: golangci-lint command line arguments. + args: --verbose + # Optional: if set to true then the all caching functionality will be complete disabled, + # takes precedence over all other caching options. + skip-cache: true + + - name: generate + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make generate + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] + + - name: modVerify + working-directory: ./${{ inputs.project-directory }} + run: go mod verify + + - name: modTidy + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make tidy + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 4a4ff0d607..e7af72ee39 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -35,57 +35,10 @@ permissions: # pull-requests: read jobs: - lint-go-project: - name: "lint: ${{ inputs.project-directory }}/${{ inputs.go-version }}" - runs-on: 'ubuntu-latest' - continue-on-error: false - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Set up Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 - with: - go-version: '${{ inputs.go-version }}' - cache-dependency-path: '${{ inputs.project-directory }}/go.sum' - id: go - - - name: golangci-lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 - with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.61.0 - # Optional: working directory, useful for monorepos - working-directory: ${{ inputs.project-directory }} - # Optional: golangci-lint command line arguments. - args: --verbose - # Optional: if set to true then the all caching functionality will be complete disabled, - # takes precedence over all other caching options. - skip-cache: true - - - name: generate - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make generate - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - - - name: modVerify - working-directory: ./${{ inputs.project-directory }} - run: go mod verify - - - name: modTidy - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make tidy - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - test-go-project: - needs: lint-go-project name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms - runs-on: [ "${{ inputs.project-directory == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest' }}" ] + runs-on: ${{ inputs.project-directory == 'modulegen' && [ubuntu-latest, macos-latest, windows-latest] || 'ubuntu-latest' }} continue-on-error: ${{ !inputs.fail-fast }} env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e338d6a15..2dab2191f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,8 +39,24 @@ jobs: - name: Print out the modules to be used run: echo "${{ steps.set-modified-modules.outputs.modules }}[*]" - test: + lint: needs: detect-modules + strategy: + matrix: + go-version: [1.22.x, 1.x] + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} + uses: ./.github/workflows/ci-lint-go.yml + with: + go-version: ${{ matrix.go-version }} + fail-fast: true + project-directory: "${{ matrix.module }}" + rootless-docker: false + ryuk-disabled: false + + test: + needs: + - detect-modules + - lint strategy: matrix: go-version: [1.22.x, 1.x] From 8a51fe6fb039bff7e3586ac4f6848518e8f09524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 14:07:04 +0100 Subject: [PATCH 17/62] fix: use fromJson --- .github/workflows/ci-test-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index e7af72ee39..228284048b 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -38,7 +38,7 @@ jobs: test-go-project: name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms - runs-on: ${{ inputs.project-directory == 'modulegen' && [ubuntu-latest, macos-latest, windows-latest] || 'ubuntu-latest' }} + runs-on: ${{ inputs.project-directory == 'modulegen' && fromJSON('[ubuntu-latest, macos-latest, windows-latest]') || 'ubuntu-latest' }} continue-on-error: ${{ !inputs.fail-fast }} env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" From e9e62cc30a02f21b722294d260d6d951a94a25d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 14:08:17 +0100 Subject: [PATCH 18/62] fix: proper jobs chain --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2dab2191f9..632f50c858 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,6 @@ jobs: test: needs: - - detect-modules - lint strategy: matrix: @@ -72,6 +71,7 @@ jobs: # The job below is a copy of the job above, but with ryuk disabled. # It's executed in the first stage to avoid concurrency issues. test-reaper-off: + needs: lint name: "Test with reaper off" strategy: matrix: @@ -87,6 +87,7 @@ jobs: # The job below is a copy of the job above, but with Docker rootless. # It's executed in the first stage to avoid concurrency issues. test-rootless-docker: + needs: lint name: "Test with Rootless Docker" strategy: matrix: From d0cf9abb6599855c3e85f2f55b3c20c665be1348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 14:12:42 +0100 Subject: [PATCH 19/62] fix: no need to lint twice --- .github/workflows/ci-lint-go.yml | 10 +++------- .github/workflows/ci.yml | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-lint-go.yml b/.github/workflows/ci-lint-go.yml index ec61299e6d..20f6e69537 100644 --- a/.github/workflows/ci-lint-go.yml +++ b/.github/workflows/ci-lint-go.yml @@ -1,13 +1,9 @@ name: Run tests for a Go project -run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }}" +run-name: "${{ inputs.project-directory }}" on: workflow_call: inputs: - go-version: - required: true - type: string - description: "The version of Go to use for the test." fail-fast: required: false type: boolean @@ -36,7 +32,7 @@ permissions: jobs: lint-go-project: - name: "lint: ${{ inputs.project-directory }}/${{ inputs.go-version }}" + name: "lint: ${{ inputs.project-directory }}" runs-on: 'ubuntu-latest' continue-on-error: false steps: @@ -46,7 +42,7 @@ jobs: - name: Set up Go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 with: - go-version: '${{ inputs.go-version }}' + go-version-file: '${{ inputs.project-directory }}/go.mod' cache-dependency-path: '${{ inputs.project-directory }}/go.sum' id: go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 632f50c858..1378fdac26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,6 @@ jobs: needs: detect-modules strategy: matrix: - go-version: [1.22.x, 1.x] module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} uses: ./.github/workflows/ci-lint-go.yml with: From afedcd64c10cde57db3ae6c6a714d32158f14d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 14:13:25 +0100 Subject: [PATCH 20/62] fix: remove non-existing input --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1378fdac26..5ad4248dab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,6 @@ jobs: module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} uses: ./.github/workflows/ci-lint-go.yml with: - go-version: ${{ matrix.go-version }} fail-fast: true project-directory: "${{ matrix.module }}" rootless-docker: false From 2352d19212cf414d78e4e8e41fd82cd98d8c56fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 14:16:23 +0100 Subject: [PATCH 21/62] fix: sanitise path --- .github/workflows/ci-lint-go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-lint-go.yml b/.github/workflows/ci-lint-go.yml index 20f6e69537..1fbf9b789d 100644 --- a/.github/workflows/ci-lint-go.yml +++ b/.github/workflows/ci-lint-go.yml @@ -42,8 +42,8 @@ jobs: - name: Set up Go uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 with: - go-version-file: '${{ inputs.project-directory }}/go.mod' - cache-dependency-path: '${{ inputs.project-directory }}/go.sum' + go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod" + cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum" id: go - name: golangci-lint From ff1a5cf379281b4113d4215ab55e26b34494b8bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 14:24:55 +0100 Subject: [PATCH 22/62] chore: only run the reaper if the core module is modified --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ad4248dab..b48403fb4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,8 @@ jobs: # The job below is a copy of the job above, but with ryuk disabled. # It's executed in the first stage to avoid concurrency issues. test-reaper-off: + # the core module is identified by the empty string (the root path) + if: ${{ contains(needs.detect-modules.outputs.modules, '') }} needs: lint name: "Test with reaper off" strategy: @@ -85,6 +87,8 @@ jobs: # The job below is a copy of the job above, but with Docker rootless. # It's executed in the first stage to avoid concurrency issues. test-rootless-docker: + # the core module is identified by the empty string (the root path) + if: ${{ contains(needs.detect-modules.outputs.modules, '') }} needs: lint name: "Test with Rootless Docker" strategy: From e30d9a0ff850b80f6a7d9b51d27ec56751e28767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 14:38:48 +0100 Subject: [PATCH 23/62] revert: simple lint --- .github/workflows/ci.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b48403fb4e..93efe171fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,12 +44,9 @@ jobs: strategy: matrix: module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} - uses: ./.github/workflows/ci-lint-go.yml - with: - fail-fast: true - project-directory: "${{ matrix.module }}" - rootless-docker: false - ryuk-disabled: false + runs-on: 'ubuntu-latest' + steps: + - run: echo "hello lint" test: needs: From 84d2fd95b200b9df6c67cc7c94c7f5080c38f813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 15:55:13 +0100 Subject: [PATCH 24/62] Revert "revert: simple lint" This reverts commit e30d9a0ff850b80f6a7d9b51d27ec56751e28767. --- .github/workflows/ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93efe171fa..b48403fb4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,9 +44,12 @@ jobs: strategy: matrix: module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} - runs-on: 'ubuntu-latest' - steps: - - run: echo "hello lint" + uses: ./.github/workflows/ci-lint-go.yml + with: + fail-fast: true + project-directory: "${{ matrix.module }}" + rootless-docker: false + ryuk-disabled: false test: needs: From d66f2667667e3f4244767d6f0f02565a90073d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 15:55:44 +0100 Subject: [PATCH 25/62] revet: remove lint --- .github/workflows/ci.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b48403fb4e..9f568455cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,21 +39,9 @@ jobs: - name: Print out the modules to be used run: echo "${{ steps.set-modified-modules.outputs.modules }}[*]" - lint: - needs: detect-modules - strategy: - matrix: - module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} - uses: ./.github/workflows/ci-lint-go.yml - with: - fail-fast: true - project-directory: "${{ matrix.module }}" - rootless-docker: false - ryuk-disabled: false - test: needs: - - lint + - detect-modules strategy: matrix: go-version: [1.22.x, 1.x] @@ -71,7 +59,7 @@ jobs: test-reaper-off: # the core module is identified by the empty string (the root path) if: ${{ contains(needs.detect-modules.outputs.modules, '') }} - needs: lint + needs: detect-modules name: "Test with reaper off" strategy: matrix: @@ -89,7 +77,7 @@ jobs: test-rootless-docker: # the core module is identified by the empty string (the root path) if: ${{ contains(needs.detect-modules.outputs.modules, '') }} - needs: lint + needs: detect-modules name: "Test with Rootless Docker" strategy: matrix: From e6636acad90a9b956a75a94381c59ad453791324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 16:07:35 +0100 Subject: [PATCH 26/62] Revert "revet: remove lint" This reverts commit d66f2667667e3f4244767d6f0f02565a90073d81. --- .github/workflows/ci.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f568455cb..b48403fb4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,21 @@ jobs: - name: Print out the modules to be used run: echo "${{ steps.set-modified-modules.outputs.modules }}[*]" + lint: + needs: detect-modules + strategy: + matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} + uses: ./.github/workflows/ci-lint-go.yml + with: + fail-fast: true + project-directory: "${{ matrix.module }}" + rootless-docker: false + ryuk-disabled: false + test: needs: - - detect-modules + - lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -59,7 +71,7 @@ jobs: test-reaper-off: # the core module is identified by the empty string (the root path) if: ${{ contains(needs.detect-modules.outputs.modules, '') }} - needs: detect-modules + needs: lint name: "Test with reaper off" strategy: matrix: @@ -77,7 +89,7 @@ jobs: test-rootless-docker: # the core module is identified by the empty string (the root path) if: ${{ contains(needs.detect-modules.outputs.modules, '') }} - needs: detect-modules + needs: lint name: "Test with Rootless Docker" strategy: matrix: From 02fe23f05073c3aa64289283dfcfe76d55fdcd0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 16:11:49 +0100 Subject: [PATCH 27/62] chore: try with input platform --- .github/workflows/ci-test-go.yml | 7 ++++++- .github/workflows/ci.yml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 228284048b..f032f0f128 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -13,6 +13,11 @@ on: type: boolean default: true description: "Fail the workflow if any of the jobs fail." + platform: + required: true + type: string + default: "." + description: "The platform where to run the project." project-directory: required: true type: string @@ -38,7 +43,7 @@ jobs: test-go-project: name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms - runs-on: ${{ inputs.project-directory == 'modulegen' && fromJSON('[ubuntu-latest, macos-latest, windows-latest]') || 'ubuntu-latest' }} + runs-on: ${{ inputs.platform }} continue-on-error: ${{ !inputs.fail-fast }} env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b48403fb4e..bd0a0c6625 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: true + platform: ${{ matrix.module == 'modulegen' && fromJSON('[ubuntu-latest, macos-latest, windows-latest]') || 'ubuntu-latest' }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false From c5db8ab86237dea8a2b15f432771ef76d77fd9c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 16:33:45 +0100 Subject: [PATCH 28/62] fix: forgotten platform --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd0a0c6625..4cbdb8602c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false + platform: 'ubuntu-latest' project-directory: "." rootless-docker: false ryuk-disabled: true @@ -100,6 +101,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false + platform: 'ubuntu-latest' project-directory: "." rootless-docker: true ryuk-disabled: false From 49f7473ee907a76fc474acc5b588a5f79655890d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 16:50:17 +0100 Subject: [PATCH 29/62] fix: pass a string to the reusable workflow --- .github/workflows/ci-test-go.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index f032f0f128..176eb24357 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -43,7 +43,7 @@ jobs: test-go-project: name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms - runs-on: ${{ inputs.platform }} + runs-on: ${{ fromJson(inputs.platform) }} continue-on-error: ${{ !inputs.fail-fast }} env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cbdb8602c..85aeb46659 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: true - platform: ${{ matrix.module == 'modulegen' && fromJSON('[ubuntu-latest, macos-latest, windows-latest]') || 'ubuntu-latest' }} + platform: ${{ matrix.module == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest') || 'ubuntu-latest' }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false From b4480a8ea70ec70b08b2ece975225cd1ecd5aafe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 16:51:34 +0100 Subject: [PATCH 30/62] fix: remove missing parenthesis --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85aeb46659..faa8371875 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: true - platform: ${{ matrix.module == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest') || 'ubuntu-latest' }} + platform: ${{ matrix.module == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest' }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false From 2638f2ad774025f12bbf101bb076adcdd9e9a8dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 17:20:32 +0100 Subject: [PATCH 31/62] fix: proper input value as string --- .github/workflows/ci-test-go.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 176eb24357..f60370fb28 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -16,7 +16,7 @@ on: platform: required: true type: string - default: "." + default: "[ubuntu-latest]" description: "The platform where to run the project." project-directory: required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faa8371875..18f92a9084 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: true - platform: ${{ matrix.module == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest' }} + platform: ${{ matrix.module == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false From c7e5e7e3a7933859363d997d2cd1cc01ba975e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 17:48:08 +0100 Subject: [PATCH 32/62] fix: order properly --- scripts/list-modules.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/list-modules.sh b/scripts/list-modules.sh index c87fd99cbd..f9276331fb 100755 --- a/scripts/list-modules.sh +++ b/scripts/list-modules.sh @@ -29,10 +29,10 @@ rootModule="\"\"" modulegenModule="\"modulegen\"" # merge all modules and examples into a single array -readonly allModules=("${rootModule}" "${modulegenModule}" "${modules[@]}") +allModules=("${rootModule}" "${modulegenModule}" "${modules[@]}") -# sort modified_modules array -IFS=$'\n' modified_modules=($(sort <<<"${modified_modules[*]}")) +# sort allModules array +IFS=$'\n' allModules=($(sort <<<"${allModules[*]}")) # print all modules with this format: # each module will be enclosed in double quotes From 5514edd7b11c5bb65b7e32698a2f746167103721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 7 Nov 2024 17:54:43 +0100 Subject: [PATCH 33/62] chore: simplify merging scripts --- .github/workflows/ci.yml | 5 ---- scripts/changed-modules.sh | 48 +++++++++++++++++++++++++++++++------- scripts/list-modules.sh | 41 -------------------------------- 3 files changed, 40 insertions(+), 54 deletions(-) delete mode 100755 scripts/list-modules.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18f92a9084..672f1419ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,6 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - id: set-available-modules - name: "Detect the available modules" - run: echo "modules=$(./scripts/list-modules.sh)" >> $GITHUB_OUTPUT - - id: changed-files name: Get changed files uses: tj-actions/changed-files@v45 @@ -33,7 +29,6 @@ jobs: name: Set all modified modules env: ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" - ALL_AVAILABLE_MODULES: ${{ steps.set-available-modules.outputs.modules }} run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT - name: Print out the modules to be used diff --git a/scripts/changed-modules.sh b/scripts/changed-modules.sh index 8cb19cfb2f..2f5ed88dc3 100755 --- a/scripts/changed-modules.sh +++ b/scripts/changed-modules.sh @@ -2,33 +2,65 @@ # How to test this script, run it with the required environment variables: # 1. A Go file from the core module is modified: -# ALL_CHANGED_FILES="examples/nginx/go.mod examples/foo/a.txt a/b/c/d/a.go" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# ALL_CHANGED_FILES="examples/nginx/go.mod examples/foo/a.txt a/b/c/d/a.go" ./scripts/changed-modules.sh # The output should be: all modules. # # 2. A file from a module in the modules dir is modified: -# ALL_CHANGED_FILES="modules/nginx/go.mod" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# ALL_CHANGED_FILES="modules/nginx/go.mod" ./scripts/changed-modules.sh # The output should be: just the modules/nginx module. # # 3. A file from a module in the examples dir is modified: -# ALL_CHANGED_FILES="examples/nginx/go.mod" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# ALL_CHANGED_FILES="examples/nginx/go.mod" ./scripts/changed-modules.sh # The output should be: just the examples/nginx module. # # 4. A Go file from the modulegen dir is modified: -# ALL_CHANGED_FILES="modulegen/a.go" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# ALL_CHANGED_FILES="modulegen/a.go" ./scripts/changed-modules.sh # The output should be: just the modulegen module. # # 5. A non-Go file from the core dir is modified: -# ALL_CHANGED_FILES="docs/README.md" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# ALL_CHANGED_FILES="docs/README.md" ./scripts/changed-modules.sh # The output should be: all modules. # # 6. A file from two modules in the modules dir are modified: -# ALL_CHANGED_FILES="modules/nginx/go.mod modules/localstack/go.mod" ALL_AVAILABLE_MODULES="$(./scripts/list-modules.sh)" ./scripts/changed-modules.sh +# ALL_CHANGED_FILES="modules/nginx/go.mod modules/localstack/go.mod" ./scripts/changed-modules.sh # The output should be: the modules/nginx and modules/localstack modules. # # There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. # But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. -# Get the list of modified files +# ROOT_DIR is the root directory of the repository. +readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +# modules is an array that will store the paths of all the modules in the repository. +modules=() + +# Find all go.mod files in the repository, building a list of all the available modules and examples. +for modFile in $(find "${ROOT_DIR}/modules" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do + modules+=("\"modules/$(basename "$(dirname "${modFile}")")\"") +done +for modFile in $(find "${ROOT_DIR}/examples" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do + modules+=("\"examples/$(basename "$(dirname "${modFile}")")\"") +done + +# sort modules array +IFS=$'\n' modules=($(sort <<<"${modules[*]}")) +unset IFS + +# capture the root module +readonly rootModule="\"\"" + +# capture the modulegen module +readonly modulegenModule="\"modulegen\"" + +# merge all modules and examples into a single array +allModules=(${rootModule} ${modulegenModule} "${modules[@]}") + +# sort allModules array +IFS=$'\n' allModules=($(sort <<<"${allModules[*]}")) +unset IFS + +# Get the list of modified files, retrieved from the environment variable ALL_CHANGED_FILES. +# On CI, this value will come from a Github Action retrieving the list of modified files from the pull request. readonly modified_files=${ALL_CHANGED_FILES[@]} # Initialize variables @@ -54,7 +86,7 @@ for file in $modified_files; do modified_modules+=("\"modulegen\"") else # a file from the core module is modified, so include all modules in the list and stop the loop - modified_modules=${ALL_AVAILABLE_MODULES[@]} + modified_modules=${allModules[@]} break fi done diff --git a/scripts/list-modules.sh b/scripts/list-modules.sh deleted file mode 100755 index f9276331fb..0000000000 --- a/scripts/list-modules.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Find all go.mod files in the repository, building a list of all the available modules. - -# ROOT_DIR is the root directory of the repository. -readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) - -# modules is an array that will store the paths of all the modules in the repository. -modules=() - -# capture modules -for modFile in $(find "${ROOT_DIR}/modules" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do - modules+=("\"modules/$(basename "$(dirname "${modFile}")")\"") -done - -# capture examples -for modFile in $(find "${ROOT_DIR}/examples" -name "go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do - modules+=("\"examples/$(basename "$(dirname "${modFile}")")\"") -done - -# sort modules array -IFS=$'\n' modules=($(sort <<<"${modules[*]}")) -unset IFS - -# capture the root module -rootModule="\"\"" - -# capture the modulegen module -modulegenModule="\"modulegen\"" - -# merge all modules and examples into a single array -allModules=("${rootModule}" "${modulegenModule}" "${modules[@]}") - -# sort allModules array -IFS=$'\n' allModules=($(sort <<<"${allModules[*]}")) - -# print all modules with this format: -# each module will be enclosed in double quotes -# each module will be separated by a comma -# the entire list will be enclosed in square brackets -echo "$(IFS=,; echo "${allModules[*]}" | sed 's/ /,/g')" From 718e0bca094cf1440a31018323522c3cab3480b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 07:56:21 +0100 Subject: [PATCH 34/62] fix: proper array syntax --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 672f1419ca..dd83a40eb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false - platform: 'ubuntu-latest' + platform: '[ubuntu-latest]' project-directory: "." rootless-docker: false ryuk-disabled: true @@ -91,12 +91,11 @@ jobs: strategy: matrix: go-version: [1.22.x, 1.x] - platform: [ubuntu-latest] uses: ./.github/workflows/ci-test-go.yml with: go-version: ${{ matrix.go-version }} fail-fast: false - platform: 'ubuntu-latest' + platform: '[ubuntu-latest]' project-directory: "." rootless-docker: true ryuk-disabled: false From a6b0e5323a0a36fb9b11d384b78a3a53e8d26bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 12:58:14 +0100 Subject: [PATCH 35/62] fix: outputs are consumed by downstream jobs that needs the producer --- .github/workflows/ci.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd83a40eb3..7f271448bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,7 @@ jobs: test: needs: + - detect-modules - lint strategy: matrix: @@ -66,8 +67,10 @@ jobs: # It's executed in the first stage to avoid concurrency issues. test-reaper-off: # the core module is identified by the empty string (the root path) - if: ${{ contains(needs.detect-modules.outputs.modules, '') }} - needs: lint + if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }} + needs: + - detect-modules + - lint name: "Test with reaper off" strategy: matrix: @@ -85,8 +88,10 @@ jobs: # It's executed in the first stage to avoid concurrency issues. test-rootless-docker: # the core module is identified by the empty string (the root path) - if: ${{ contains(needs.detect-modules.outputs.modules, '') }} - needs: lint + if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }} + needs: + - detect-modules + - lint name: "Test with Rootless Docker" strategy: matrix: From 3314f8d8000ce58f89b0b50562673fa6170d33cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 12:59:52 +0100 Subject: [PATCH 36/62] fix: remove extra element --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f271448bb..54781484f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT - name: Print out the modules to be used - run: echo "${{ steps.set-modified-modules.outputs.modules }}[*]" + run: echo "${{ steps.set-modified-modules.outputs.modules }}" lint: needs: detect-modules From 459de9f5186e2ac50058ba29c62cc1e1db99118e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 13:11:59 +0100 Subject: [PATCH 37/62] chore: detemine platform in downstream job --- .github/workflows/ci-test-go.yml | 8 ++------ .github/workflows/ci.yml | 3 --- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index f60370fb28..644270ed8a 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -13,11 +13,6 @@ on: type: boolean default: true description: "Fail the workflow if any of the jobs fail." - platform: - required: true - type: string - default: "[ubuntu-latest]" - description: "The platform where to run the project." project-directory: required: true type: string @@ -43,12 +38,13 @@ jobs: test-go-project: name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms - runs-on: ${{ fromJson(inputs.platform) }} continue-on-error: ${{ !inputs.fail-fast }} env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}" RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" + PLATFORM: ${{ inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} + runs-on: ${{ fromJson(env.PLATFORM) }} steps: - name: Setup rootless Docker if: ${{ inputs.rootless-docker }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54781484f5..ed8082322c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,6 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: true - platform: ${{ matrix.module == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false @@ -79,7 +78,6 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false - platform: '[ubuntu-latest]' project-directory: "." rootless-docker: false ryuk-disabled: true @@ -100,7 +98,6 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false - platform: '[ubuntu-latest]' project-directory: "." rootless-docker: true ryuk-disabled: false From e23c1cd0fc9c74091d2f5536b70acdafe301349c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 13:14:35 +0100 Subject: [PATCH 38/62] fix: read env properly --- .github/workflows/ci-test-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 644270ed8a..cf898ffce7 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -44,7 +44,7 @@ jobs: RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}" RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" PLATFORM: ${{ inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} - runs-on: ${{ fromJson(env.PLATFORM) }} + runs-on: ${{ fromJson($PLATFORM) }} steps: - name: Setup rootless Docker if: ${{ inputs.rootless-docker }} From c7476de421bc2e2c4ce53d5b9e4188281b76dccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 13:22:01 +0100 Subject: [PATCH 39/62] chore: define runner in a previos step --- .github/workflows/ci-test-go.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index cf898ffce7..0b721bf7d9 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -35,7 +35,18 @@ permissions: # pull-requests: read jobs: + set-runners: + outputs: + runners: ${{ steps.set-runners.outputs.runners }} + runs-on: 'ubuntu-latest' + steps: + - id: set-runners + name: "Set runners for ${{ inputs.project-directory }}" + run: echo "runners=${{ inputs.project-directory == 'modulegen' && [ubuntu-latest, macos-latest, windows-latest] || [ubuntu-latest] }}" >> $GITHUB_OUTPUT + test-go-project: + needs: + - set-runners name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms continue-on-error: ${{ !inputs.fail-fast }} @@ -43,8 +54,7 @@ jobs: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}" RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" - PLATFORM: ${{ inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} - runs-on: ${{ fromJson($PLATFORM) }} + runs-on: ${{ fromJSON(needs.set-runners.outputs.runners) }} steps: - name: Setup rootless Docker if: ${{ inputs.rootless-docker }} From 9a3f6f9c034255a3f336b7a30d61cd701285cf48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 13:23:55 +0100 Subject: [PATCH 40/62] fix: proper output format --- .github/workflows/ci-test-go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 0b721bf7d9..898563d859 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -42,13 +42,13 @@ jobs: steps: - id: set-runners name: "Set runners for ${{ inputs.project-directory }}" - run: echo "runners=${{ inputs.project-directory == 'modulegen' && [ubuntu-latest, macos-latest, windows-latest] || [ubuntu-latest] }}" >> $GITHUB_OUTPUT + # Modulegen can run the tests on all platforms + run: echo "runners=[${{ inputs.project-directory == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest' }}]" >> $GITHUB_OUTPUT test-go-project: needs: - set-runners name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" - # Modulegen can run the tests on all platforms continue-on-error: ${{ !inputs.fail-fast }} env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" From b75cf6573109beff16455cff7ab599252faa1ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 13:38:48 +0100 Subject: [PATCH 41/62] Revert "fix: proper output format" This reverts commit 9a3f6f9c034255a3f336b7a30d61cd701285cf48. --- .github/workflows/ci-test-go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 898563d859..0b721bf7d9 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -42,13 +42,13 @@ jobs: steps: - id: set-runners name: "Set runners for ${{ inputs.project-directory }}" - # Modulegen can run the tests on all platforms - run: echo "runners=[${{ inputs.project-directory == 'modulegen' && 'ubuntu-latest, macos-latest, windows-latest' || 'ubuntu-latest' }}]" >> $GITHUB_OUTPUT + run: echo "runners=${{ inputs.project-directory == 'modulegen' && [ubuntu-latest, macos-latest, windows-latest] || [ubuntu-latest] }}" >> $GITHUB_OUTPUT test-go-project: needs: - set-runners name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" + # Modulegen can run the tests on all platforms continue-on-error: ${{ !inputs.fail-fast }} env: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" From 607fb1faa07bd2927ec66f6af22141c387094384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 13:38:52 +0100 Subject: [PATCH 42/62] Revert "chore: define runner in a previos step" This reverts commit c7476de421bc2e2c4ce53d5b9e4188281b76dccf. --- .github/workflows/ci-test-go.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 0b721bf7d9..cf898ffce7 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -35,18 +35,7 @@ permissions: # pull-requests: read jobs: - set-runners: - outputs: - runners: ${{ steps.set-runners.outputs.runners }} - runs-on: 'ubuntu-latest' - steps: - - id: set-runners - name: "Set runners for ${{ inputs.project-directory }}" - run: echo "runners=${{ inputs.project-directory == 'modulegen' && [ubuntu-latest, macos-latest, windows-latest] || [ubuntu-latest] }}" >> $GITHUB_OUTPUT - test-go-project: - needs: - - set-runners name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms continue-on-error: ${{ !inputs.fail-fast }} @@ -54,7 +43,8 @@ jobs: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}" RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" - runs-on: ${{ fromJSON(needs.set-runners.outputs.runners) }} + PLATFORM: ${{ inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} + runs-on: ${{ fromJson($PLATFORM) }} steps: - name: Setup rootless Docker if: ${{ inputs.rootless-docker }} From f8dd7b90c71f50c1e2bd21f9702bd76c0464a601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 13:42:13 +0100 Subject: [PATCH 43/62] chore: define matrix for the downstream job --- .github/workflows/ci-test-go.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index cf898ffce7..3569a85259 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -43,8 +43,10 @@ jobs: TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}" RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}" RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" - PLATFORM: ${{ inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} - runs-on: ${{ fromJson($PLATFORM) }} + strategy: + matrix: + platform: ${{ fromJSON(inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]') }} + runs-on: ${{ matrix.platform }} steps: - name: Setup rootless Docker if: ${{ inputs.rootless-docker }} From f7828460a3c39d17e97f752cc3a95696d32329b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 13:51:32 +0100 Subject: [PATCH 44/62] fix: try with string --- .github/workflows/ci-test-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 3569a85259..a013d1515d 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -45,7 +45,7 @@ jobs: RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" strategy: matrix: - platform: ${{ fromJSON(inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]') }} + platform: ${{ inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} runs-on: ${{ matrix.platform }} steps: - name: Setup rootless Docker From 9bc5a52ba2ebf39d74ef42605c5bd7b63647ef32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 14:30:06 +0100 Subject: [PATCH 45/62] fix: use fromJSON --- .github/workflows/ci-test-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index a013d1515d..679e554098 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -45,7 +45,7 @@ jobs: RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" strategy: matrix: - platform: ${{ inputs.project-directory == 'modulegen' && '[ubuntu-latest, macos-latest, windows-latest]' || '[ubuntu-latest]' }} + platform: ${{ inputs.project-directory == 'modulegen' && fromJSON('ubuntu-latest', 'macos-latest', 'windows-latest]') || fromJSON('ubuntu-latest') }} runs-on: ${{ matrix.platform }} steps: - name: Setup rootless Docker From 963de87d9a56440bae3c1c95fcf58e37bd440c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 14:57:13 +0100 Subject: [PATCH 46/62] fix: items --- .github/workflows/ci-test-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 679e554098..9c87421dc1 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -45,7 +45,7 @@ jobs: RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" strategy: matrix: - platform: ${{ inputs.project-directory == 'modulegen' && fromJSON('ubuntu-latest', 'macos-latest', 'windows-latest]') || fromJSON('ubuntu-latest') }} + platform: ${{ inputs.project-directory == 'modulegen' && fromJSON(ubuntu-latest, macos-latest, windows-latest) || fromJSON(ubuntu-latest) }} runs-on: ${{ matrix.platform }} steps: - name: Setup rootless Docker From fb339249e8592604f863e1974b13ff310aa9f436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 18:22:28 +0100 Subject: [PATCH 47/62] chore: default as string --- .github/workflows/ci-test-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index 9c87421dc1..f68b9587e3 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -45,7 +45,7 @@ jobs: RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" strategy: matrix: - platform: ${{ inputs.project-directory == 'modulegen' && fromJSON(ubuntu-latest, macos-latest, windows-latest) || fromJSON(ubuntu-latest) }} + platform: ${{ inputs.project-directory == 'modulegen' && fromJSON(ubuntu-latest, macos-latest, windows-latest) || 'ubuntu-latest' }} runs-on: ${{ matrix.platform }} steps: - name: Setup rootless Docker From 7bc38ccd238d9adb7a72c955724d68e2739f985b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 18:52:59 +0100 Subject: [PATCH 48/62] chore: pass platforms in the parent workflow --- .github/workflows/ci-test-go.yml | 7 ++++++- .github/workflows/ci.yml | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index f68b9587e3..c7cdfc56a9 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -13,6 +13,11 @@ on: type: boolean default: true description: "Fail the workflow if any of the jobs fail." + platforms: + required: true + type: string + default: "ubuntu-latest" + description: "The platforms in which the project will be run" project-directory: required: true type: string @@ -45,7 +50,7 @@ jobs: RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}" strategy: matrix: - platform: ${{ inputs.project-directory == 'modulegen' && fromJSON(ubuntu-latest, macos-latest, windows-latest) || 'ubuntu-latest' }} + platform: ${{ fromJSON(inputs.platforms) }} runs-on: ${{ matrix.platform }} steps: - name: Setup rootless Docker diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed8082322c..6f4c54a5ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: true + platforms: ${{ matrix.module == 'modulegen' && '["ubuntu-latest", "macos-latest", "windows-latest"]' || '["ubuntu-latest"]' }} project-directory: "${{ matrix.module }}" rootless-docker: false ryuk-disabled: false @@ -78,6 +79,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false + platforms: 'ubuntu-latest' project-directory: "." rootless-docker: false ryuk-disabled: true @@ -98,6 +100,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false + platforms: 'ubuntu-latest' project-directory: "." rootless-docker: true ryuk-disabled: false From c12c7a6c5a680bd8ece96901809f7f6f82765ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 19:49:52 +0100 Subject: [PATCH 49/62] fix: single platforms must follow the array-instring format --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f4c54a5ce..32bece8899 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false - platforms: 'ubuntu-latest' + platforms: '["ubuntu-latest"]' project-directory: "." rootless-docker: false ryuk-disabled: true @@ -100,7 +100,7 @@ jobs: with: go-version: ${{ matrix.go-version }} fail-fast: false - platforms: 'ubuntu-latest' + platforms: '["ubuntu-latest"]' project-directory: "." rootless-docker: true ryuk-disabled: false From bfddbc698e48f716ca38d4377d2bf741b6a07473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 20:48:34 +0100 Subject: [PATCH 50/62] chore: exclude certain locations from the root dir --- .github/workflows/ci.yml | 4 ++++ scripts/changed-modules.sh | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32bece8899..c6cc5f92ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,8 @@ jobs: run: echo "${{ steps.set-modified-modules.outputs.modules }}" lint: + # only run if there are modules to lint + if: ${{ needs.detect-modules.outputs.modules[0] != null }} needs: detect-modules strategy: matrix: @@ -47,6 +49,8 @@ jobs: ryuk-disabled: false test: + # only run if there are modules to test + if: ${{ needs.detect-modules.outputs.modules[0] != null }} needs: - detect-modules - lint diff --git a/scripts/changed-modules.sh b/scripts/changed-modules.sh index 2f5ed88dc3..cad62f62f6 100755 --- a/scripts/changed-modules.sh +++ b/scripts/changed-modules.sh @@ -18,19 +18,26 @@ # The output should be: just the modulegen module. # # 5. A non-Go file from the core dir is modified: -# ALL_CHANGED_FILES="docs/README.md" ./scripts/changed-modules.sh +# ALL_CHANGED_FILES="README.md" ./scripts/changed-modules.sh # The output should be: all modules. # # 6. A file from two modules in the modules dir are modified: # ALL_CHANGED_FILES="modules/nginx/go.mod modules/localstack/go.mod" ./scripts/changed-modules.sh # The output should be: the modules/nginx and modules/localstack modules. # +# 7. Files from the excluded dirs are modified: +# ALL_CHANGED_FILES="docs/a.md .vscode/a.json .devcontainer/a.json scripts/a.sh" ./scripts/changed-modules.sh +# The output should be: no modules. +# # There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. # But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. # ROOT_DIR is the root directory of the repository. readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +# define an array of modules that won't be included in the list +readonly excluded_modules=(".devcontainer" ".vscode" "docs" "scripts") + # modules is an array that will store the paths of all the modules in the repository. modules=() @@ -86,6 +93,15 @@ for file in $modified_files; do modified_modules+=("\"modulegen\"") else # a file from the core module is modified, so include all modules in the list and stop the loop + # check if the file is in one of the excluded modules + for exclude_module in ${excluded_modules[@]}; do + if [[ $file == $exclude_module/* ]]; then + # continue skips to the next iteration of an enclosing for, select, until, or while loop in a shell script. + # Execution continues at the loop control of the nth enclosing loop, in this case two levels up. + continue 2 + fi + done + modified_modules=${allModules[@]} break fi From 4b8d2fa1acee9f8bcc033e002d3d49ffbfc01b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 20:58:13 +0100 Subject: [PATCH 51/62] fix: use jq to get modules count --- .github/workflows/ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6cc5f92ba..fe5b2b5703 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-latest outputs: modules: ${{ steps.set-modified-modules.outputs.modules }} + modulesCount: ${{ steps.set-modified-modules.outputs.modulesCount }} steps: - name: Check out code into the Go module directory uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 @@ -29,14 +30,16 @@ jobs: name: Set all modified modules env: ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" - run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + run: | + echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + echo "modulesCount=$(echo $modules | jq '. | length')" >> GITHUB_OUTPUT - name: Print out the modules to be used run: echo "${{ steps.set-modified-modules.outputs.modules }}" lint: # only run if there are modules to lint - if: ${{ needs.detect-modules.outputs.modules[0] != null }} + if: ${{ needs.detect-modules.outputs.modulesCount > 0 }} needs: detect-modules strategy: matrix: @@ -50,7 +53,7 @@ jobs: test: # only run if there are modules to test - if: ${{ needs.detect-modules.outputs.modules[0] != null }} + if: ${{ needs.detect-modules.outputs.modulesCount > 0 }} needs: - detect-modules - lint From 8e5691cfb999f59b2c94a958b42f0d1f8fa9247a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 21:00:14 +0100 Subject: [PATCH 52/62] chore: do not run sonarcloud for zero modules --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe5b2b5703..6e37c33c97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,8 +116,10 @@ jobs: permissions: contents: read # for actions/checkout to fetch code pull-requests: read # for sonarsource/sonarcloud-github-action to determine which PR to decorate - if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' }} - needs: test + if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && needs.detect-modules.outputs.modulesCount > 0 }} + needs: + - detect-modules + - test runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory From 7a2fe265c0bb3ee6d69312bd35234f98b72301d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 21:08:46 +0100 Subject: [PATCH 53/62] chore: separate concerns --- .github/workflows/ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e37c33c97..4b17a14211 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest outputs: modules: ${{ steps.set-modified-modules.outputs.modules }} - modulesCount: ${{ steps.set-modified-modules.outputs.modulesCount }} + modulesCount: ${{ steps.set-modified-modules-count.outputs.modulesCount }} steps: - name: Check out code into the Go module directory uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 @@ -30,12 +30,16 @@ jobs: name: Set all modified modules env: ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" - run: | - echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT - echo "modulesCount=$(echo $modules | jq '. | length')" >> GITHUB_OUTPUT + run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + + - id: set-modified-modules-count + name: Set all modified modules count + run: echo "modulesCount=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> GITHUB_OUTPUT - name: Print out the modules to be used - run: echo "${{ steps.set-modified-modules.outputs.modules }}" + run: | + echo "${{ steps.set-modified-modules-count.outputs.modulesCount }} modules in the build" + echo "${{ steps.set-modified-modules.outputs.modules }}" lint: # only run if there are modules to lint From 0a387e902857ff87b20e35e68c52303b927a14ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 21:34:27 +0100 Subject: [PATCH 54/62] chore: format variable name --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b17a14211..233b37aae7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest outputs: modules: ${{ steps.set-modified-modules.outputs.modules }} - modulesCount: ${{ steps.set-modified-modules-count.outputs.modulesCount }} + modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }} steps: - name: Check out code into the Go module directory uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 @@ -34,16 +34,16 @@ jobs: - id: set-modified-modules-count name: Set all modified modules count - run: echo "modulesCount=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> GITHUB_OUTPUT + run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT - name: Print out the modules to be used run: | - echo "${{ steps.set-modified-modules-count.outputs.modulesCount }} modules in the build" + echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build" echo "${{ steps.set-modified-modules.outputs.modules }}" lint: # only run if there are modules to lint - if: ${{ needs.detect-modules.outputs.modulesCount > 0 }} + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} needs: detect-modules strategy: matrix: @@ -57,7 +57,7 @@ jobs: test: # only run if there are modules to test - if: ${{ needs.detect-modules.outputs.modulesCount > 0 }} + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} needs: - detect-modules - lint @@ -120,7 +120,7 @@ jobs: permissions: contents: read # for actions/checkout to fetch code pull-requests: read # for sonarsource/sonarcloud-github-action to determine which PR to decorate - if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && needs.detect-modules.outputs.modulesCount > 0 }} + if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && needs.detect-modules.outputs.modules_count > 0 }} needs: - detect-modules - test From b638baa1df6ffd631560866f19d21a0352d456a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 8 Nov 2024 21:52:58 +0100 Subject: [PATCH 55/62] chore: no need to generate the ci.yml anymore --- .github/workflows/ci.yml | 2 - modulegen/_template/ci.yml.tmpl | 145 --------------------------- modulegen/internal/main.go | 6 +- modulegen/internal/workflow/main.go | 40 -------- modulegen/internal/workflow/types.go | 17 ---- modulegen/main_test.go | 34 ------- scripts/bump-go.sh | 1 - 7 files changed, 2 insertions(+), 243 deletions(-) delete mode 100644 modulegen/_template/ci.yml.tmpl delete mode 100644 modulegen/internal/workflow/main.go delete mode 100644 modulegen/internal/workflow/types.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 233b37aae7..04d995c31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,3 @@ -# This file is autogenerated by the 'modulegen' tool. -# Please update the 'ci.yml' template instead. name: Main pipeline on: diff --git a/modulegen/_template/ci.yml.tmpl b/modulegen/_template/ci.yml.tmpl deleted file mode 100644 index 46fc3e3906..0000000000 --- a/modulegen/_template/ci.yml.tmpl +++ /dev/null @@ -1,145 +0,0 @@ -# This file is autogenerated by the 'modulegen' tool. -# Please update the 'ci.yml' template instead. -name: Main pipeline - -on: - push: - branches: - - main - paths-ignore: - - '.vscode/**' - - 'mkdocs.yml' - - 'docs/**' - - 'README.md' - pull_request: - paths-ignore: - - '.vscode/**' - - 'mkdocs.yml' - - 'docs/**' - - 'README.md' - -concurrency: - group: {{ "${{ github.workflow }}-${{ github.head_ref || github.sha }}" }} - cancel-in-progress: true - -jobs: - test: - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest, macos-latest] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: true - platform: {{ "${{ matrix.platform }}" }} - project-directory: "." - rootless-docker: false - run-tests: {{ "${{ matrix.platform == 'ubuntu-latest' }}" }} - ryuk-disabled: false - - # The job below is a copy of the job above, but with ryuk disabled. - # It's executed in the first stage to avoid concurrency issues. - test-reaper-off: - name: "Test with reaper off" - strategy: - matrix: - go-version: [1.22.x, 1.x] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: false - platform: "ubuntu-latest" - project-directory: "." - rootless-docker: false - run-tests: true - ryuk-disabled: true - - # The job below is a copy of the job above, but with Docker rootless. - # It's executed in the first stage to avoid concurrency issues. - test-rootless-docker: - name: "Test with Rootless Docker" - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: false - platform: "ubuntu-latest" - project-directory: "." - rootless-docker: true - run-tests: true - ryuk-disabled: false - - test-module-generator: - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest, macos-latest, windows-latest] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: true - platform: {{ "${{ matrix.platform }}" }} - project-directory: "modulegen" - rootless-docker: false - run-tests: true - ryuk-disabled: false - - test-modules: - needs: test - strategy: - matrix: - go-version: [1.22.x, 1.x] - platform: [ubuntu-latest] - module: [{{ .Modules }}] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: {{ "${{ matrix.go-version }}" }} - fail-fast: false - platform: {{ "${{ matrix.platform }}" }} - project-directory: {{ "modules/${{ matrix.module }}" }} - rootless-docker: false - run-tests: {{ "${{ matrix.platform == 'ubuntu-latest' }}" }} - ryuk-disabled: false - - test-examples: - needs: test-modules - strategy: - matrix: - module: [{{ .Examples }}] - uses: ./.github/workflows/ci-test-go.yml - with: - go-version: "1.22.x" - fail-fast: true - platform: 'ubuntu-latest' - project-directory: {{ "examples/${{ matrix.module }}" }} - rootless-docker: false - run-tests: true - ryuk-disabled: false - - sonarcloud: - permissions: - contents: read # for actions/checkout to fetch code - pull-requests: read # for sonarsource/sonarcloud-github-action to determine which PR to decorate - if: {{ "${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' }}" }} - needs: test-examples - runs-on: ubuntu-latest - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - # Disabling shallow clone is recommended for improving relevancy of reporting - fetch-depth: 0 - - - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: sonarcloud - - - name: Analyze with SonarCloud - uses: sonarsource/sonarcloud-github-action@49e6cd3b187936a73b8280d59ffd9da69df63ec9 # v2.1.1 - env: - GITHUB_TOKEN: {{ "${{ secrets.GITHUB_TOKEN }}" }} - SONAR_TOKEN: {{ "${{ secrets.SONAR_TOKEN }}" }} diff --git a/modulegen/internal/main.go b/modulegen/internal/main.go index 2f64c5836d..21084a5a52 100644 --- a/modulegen/internal/main.go +++ b/modulegen/internal/main.go @@ -11,7 +11,6 @@ import ( "github.com/testcontainers/testcontainers-go/modulegen/internal/sonar" "github.com/testcontainers/testcontainers-go/modulegen/internal/tools" "github.com/testcontainers/testcontainers-go/modulegen/internal/vscode" - "github.com/testcontainers/testcontainers-go/modulegen/internal/workflow" ) func Generate(moduleVar context.TestcontainersModuleVar, isModule bool) error { @@ -82,9 +81,8 @@ func GenerateFiles(ctx context.Context, tcModule context.TestcontainersModule) e // not in the new module to be added, that's why they happen after the actual // module generation projectGenerators := []ProjectGenerator{ - workflow.Generator{}, // update github ci workflow - vscode.Generator{}, // update vscode workspace - sonar.Generator{}, // update sonar-project.properties + vscode.Generator{}, // update vscode workspace + sonar.Generator{}, // update sonar-project.properties } for _, generator := range projectGenerators { diff --git a/modulegen/internal/workflow/main.go b/modulegen/internal/workflow/main.go deleted file mode 100644 index 9cf75d259c..0000000000 --- a/modulegen/internal/workflow/main.go +++ /dev/null @@ -1,40 +0,0 @@ -package workflow - -import ( - "path/filepath" - "text/template" - - "github.com/testcontainers/testcontainers-go/modulegen/internal/context" - internal_template "github.com/testcontainers/testcontainers-go/modulegen/internal/template" -) - -type Generator struct{} - -// Generate updates github ci workflow -func (g Generator) Generate(ctx context.Context) error { - rootCtx, err := context.GetRootContext() - if err != nil { - return err - } - examples, err := rootCtx.GetExamples() - if err != nil { - return err - } - modules, err := rootCtx.GetModules() - if err != nil { - return err - } - - githubWorkflowsDir := ctx.GithubWorkflowsDir() - - projectDirectories := newProjectDirectories(examples, modules) - name := "ci.yml.tmpl" - t, err := template.New(name).ParseFiles(filepath.Join("_template", name)) - if err != nil { - return err - } - - exampleFilePath := filepath.Join(githubWorkflowsDir, "ci.yml") - - return internal_template.GenerateFile(t, exampleFilePath, name, projectDirectories) -} diff --git a/modulegen/internal/workflow/types.go b/modulegen/internal/workflow/types.go deleted file mode 100644 index 77433613fb..0000000000 --- a/modulegen/internal/workflow/types.go +++ /dev/null @@ -1,17 +0,0 @@ -package workflow - -import ( - "strings" -) - -type ProjectDirectories struct { - Examples string - Modules string -} - -func newProjectDirectories(examples []string, modules []string) *ProjectDirectories { - return &ProjectDirectories{ - Examples: strings.Join(examples, ", "), - Modules: strings.Join(modules, ", "), - } -} diff --git a/modulegen/main_test.go b/modulegen/main_test.go index d90c0da5be..9147626804 100644 --- a/modulegen/main_test.go +++ b/modulegen/main_test.go @@ -247,14 +247,11 @@ func TestGenerate(t *testing.T) { tmpCtx := context.New(t.TempDir()) examplesTmp := filepath.Join(tmpCtx.RootDir, "examples") examplesDocTmp := filepath.Join(tmpCtx.DocsDir(), "examples") - githubWorkflowsTmp := tmpCtx.GithubWorkflowsDir() err := os.MkdirAll(examplesTmp, 0o777) require.NoError(t, err) err = os.MkdirAll(examplesDocTmp, 0o777) require.NoError(t, err) - err = os.MkdirAll(githubWorkflowsTmp, 0o777) - require.NoError(t, err) err = copyInitialMkdocsConfig(t, tmpCtx) require.NoError(t, err) @@ -283,12 +280,7 @@ func TestGenerate(t *testing.T) { _, err = os.Stat(moduleDocFile) require.NoError(t, err) // error nil implies the file exist - mainWorkflowFile := filepath.Join(githubWorkflowsTmp, "ci.yml") - _, err = os.Stat(mainWorkflowFile) - require.NoError(t, err) // error nil implies the file exist - assertModuleDocContent(t, module, moduleDocFile) - assertModuleGithubWorkflowContent(t, mainWorkflowFile) generatedTemplatesDir := filepath.Join(examplesTmp, moduleNameLower) // do not generate examples_test.go for examples @@ -303,14 +295,11 @@ func TestGenerateModule(t *testing.T) { tmpCtx := context.New(t.TempDir()) modulesTmp := filepath.Join(tmpCtx.RootDir, "modules") modulesDocTmp := filepath.Join(tmpCtx.DocsDir(), "modules") - githubWorkflowsTmp := tmpCtx.GithubWorkflowsDir() err := os.MkdirAll(modulesTmp, 0o777) require.NoError(t, err) err = os.MkdirAll(modulesDocTmp, 0o777) require.NoError(t, err) - err = os.MkdirAll(githubWorkflowsTmp, 0o777) - require.NoError(t, err) err = copyInitialMkdocsConfig(t, tmpCtx) require.NoError(t, err) @@ -339,12 +328,7 @@ func TestGenerateModule(t *testing.T) { _, err = os.Stat(moduleDocFile) require.NoError(t, err) // error nil implies the file exist - mainWorkflowFile := filepath.Join(githubWorkflowsTmp, "ci.yml") - _, err = os.Stat(mainWorkflowFile) - require.NoError(t, err) // error nil implies the file exist - assertModuleDocContent(t, module, moduleDocFile) - assertModuleGithubWorkflowContent(t, mainWorkflowFile) generatedTemplatesDir := filepath.Join(modulesTmp, moduleNameLower) assertExamplesTestContent(t, module, filepath.Join(generatedTemplatesDir, "examples_test.go")) @@ -439,24 +423,6 @@ func assertModuleContent(t *testing.T, module context.TestcontainersModule, exam require.Equal(t, "\treturn c, nil", data[41]) } -// assert content GitHub workflow for the module -func assertModuleGithubWorkflowContent(t *testing.T, moduleWorkflowFile string) { - t.Helper() - content, err := os.ReadFile(moduleWorkflowFile) - require.NoError(t, err) - - data := sanitiseContent(content) - ctx := getTestRootContext(t) - - modulesList, err := ctx.GetModules() - require.NoError(t, err) - assert.Equal(t, " module: ["+strings.Join(modulesList, ", ")+"]", data[96]) - - examplesList, err := ctx.GetExamples() - require.NoError(t, err) - assert.Equal(t, " module: ["+strings.Join(examplesList, ", ")+"]", data[111]) -} - // assert content go.mod func assertGoModContent(t *testing.T, module context.TestcontainersModule, tcVersion string, goModFile string) { t.Helper() diff --git a/scripts/bump-go.sh b/scripts/bump-go.sh index 5ff33a5e6e..5f65e5aada 100755 --- a/scripts/bump-go.sh +++ b/scripts/bump-go.sh @@ -48,7 +48,6 @@ function main() { for f in $(find "${ROOT_DIR}/.github/workflows" -name "*.yml"); do bumpCIMatrix "${f}" "${escapedCurrentGoVersion}" "${escapedGoVersion}" done - bumpCIMatrix "${ROOT_DIR}/modulegen/_template/ci.yml.tmpl" "${escapedCurrentGoVersion}" "${escapedGoVersion}" # bump devcontainer file bumpDevcontainer "${ROOT_DIR}/.devcontainer/devcontainer.json" "${escapedCurrentGoVersion}" "${escapedGoVersion}" From d4adfd5abf0cc050fc916437872b2bf354595b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 11 Nov 2024 07:44:12 +0100 Subject: [PATCH 56/62] chore: merge lint and test workflow files again It will create a nicer UI render --- .github/workflows/ci-lint-go.yml | 78 -------------------------------- .github/workflows/ci.yml | 17 ------- 2 files changed, 95 deletions(-) delete mode 100644 .github/workflows/ci-lint-go.yml diff --git a/.github/workflows/ci-lint-go.yml b/.github/workflows/ci-lint-go.yml deleted file mode 100644 index 1fbf9b789d..0000000000 --- a/.github/workflows/ci-lint-go.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Run tests for a Go project -run-name: "${{ inputs.project-directory }}" - -on: - workflow_call: - inputs: - fail-fast: - required: false - type: boolean - default: true - description: "Fail the workflow if any of the jobs fail." - project-directory: - required: true - type: string - default: "." - description: "The directory where the Go project is located." - rootless-docker: - required: false - type: boolean - default: false - description: "Run the test with rootless docker." - ryuk-disabled: - required: false - type: boolean - default: false - description: "Disable the ryuk container for the test." - -permissions: - contents: read - # Optional: allow read access to pull request. Use with `only-new-issues` option. - # pull-requests: read - -jobs: - lint-go-project: - name: "lint: ${{ inputs.project-directory }}" - runs-on: 'ubuntu-latest' - continue-on-error: false - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Set up Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 - with: - go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod" - cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum" - id: go - - - name: golangci-lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 - with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.61.0 - # Optional: working directory, useful for monorepos - working-directory: ${{ inputs.project-directory }} - # Optional: golangci-lint command line arguments. - args: --verbose - # Optional: if set to true then the all caching functionality will be complete disabled, - # takes precedence over all other caching options. - skip-cache: true - - - name: generate - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make generate - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - - - name: modVerify - working-directory: ./${{ inputs.project-directory }} - run: go mod verify - - - name: modTidy - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make tidy - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04d995c31a..2bee3f2688 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,26 +39,11 @@ jobs: echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build" echo "${{ steps.set-modified-modules.outputs.modules }}" - lint: - # only run if there are modules to lint - if: ${{ needs.detect-modules.outputs.modules_count > 0 }} - needs: detect-modules - strategy: - matrix: - module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} - uses: ./.github/workflows/ci-lint-go.yml - with: - fail-fast: true - project-directory: "${{ matrix.module }}" - rootless-docker: false - ryuk-disabled: false - test: # only run if there are modules to test if: ${{ needs.detect-modules.outputs.modules_count > 0 }} needs: - detect-modules - - lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -79,7 +64,6 @@ jobs: if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }} needs: - detect-modules - - lint name: "Test with reaper off" strategy: matrix: @@ -100,7 +84,6 @@ jobs: if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }} needs: - detect-modules - - lint name: "Test with Rootless Docker" strategy: matrix: From f9682dfca677c052e886cadeda625e2864cabcf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 11 Nov 2024 07:46:19 +0100 Subject: [PATCH 57/62] fix: missing lint job in workflow --- .github/workflows/ci-test-go.yml | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index c7cdfc56a9..a14936b9fa 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -40,6 +40,52 @@ permissions: # pull-requests: read jobs: + lint-go-project: + name: "lint: ${{ inputs.project-directory }}" + runs-on: 'ubuntu-latest' + continue-on-error: false + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + with: + go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod" + cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum" + id: go + + - name: golangci-lint + uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.61.0 + # Optional: working directory, useful for monorepos + working-directory: ${{ inputs.project-directory }} + # Optional: golangci-lint command line arguments. + args: --verbose + # Optional: if set to true then the all caching functionality will be complete disabled, + # takes precedence over all other caching options. + skip-cache: true + + - name: generate + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make generate + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] + + - name: modVerify + working-directory: ./${{ inputs.project-directory }} + run: go mod verify + + - name: modTidy + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make tidy + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] + test-go-project: name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" # Modulegen can run the tests on all platforms From 730f3e6e4b112e0b979a88296d55682a72dc8667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 11 Nov 2024 08:12:08 +0100 Subject: [PATCH 58/62] chore: run lint first --- .github/workflows/ci-test-go.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index a14936b9fa..bb8d83c5ec 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -88,6 +88,7 @@ jobs: test-go-project: name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" + needs: lint-go-project # Modulegen can run the tests on all platforms continue-on-error: ${{ !inputs.fail-fast }} env: From b939d6278b78ae5fa0a2ed2b0dae52505b6842e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 11 Nov 2024 08:53:21 +0100 Subject: [PATCH 59/62] fix: update name --- .github/workflows/ci-test-go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index bb8d83c5ec..6c4a3057a7 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -1,5 +1,5 @@ name: Run tests for a Go project -run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }}" +run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platforms }}" on: workflow_call: @@ -74,7 +74,7 @@ jobs: run: | make generate git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - + - name: modVerify working-directory: ./${{ inputs.project-directory }} run: go mod verify From d8d62d77fdcce59b63190249bae679412c2c8398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 13 Nov 2024 13:51:19 +0100 Subject: [PATCH 60/62] fix: pin the action commit Co-authored-by: Victor Martinez --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac30891d9c..224916fd3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - id: changed-files name: Get changed files - uses: tj-actions/changed-files@v45 + uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4 - id: set-modified-modules name: Set all modified modules From da8b16140341b21d267af46dc6f74025b75159f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 18 Nov 2024 11:55:31 +0100 Subject: [PATCH 61/62] chore: matrix.platform --- .github/workflows/ci-test-go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index d78a0be54c..4c42ecb771 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -128,10 +128,10 @@ jobs: run: make test-unit - name: Upload SonarCloud files - if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && runner.name == 'ubuntu-latest' && !inputs.rootless-docker }} + if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && matrix.platform == 'ubuntu-latest' && !inputs.rootless-docker }} uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: - name: sonarcloud-${{ inputs.project-directory }}-${{ inputs.go-version }}-${{ inputs.platform }} + name: sonarcloud-${{ inputs.project-directory }}-${{ inputs.go-version }}-${{ matrix.platform }} path: | ./sonar-project.properties ${{ inputs.project-directory }}/TEST-unit.xml From 12a90ab5bdccdaa6914e2cc443adaebaea5112e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 19 Nov 2024 11:52:31 +0100 Subject: [PATCH 62/62] chore: run lint stage before the test one --- .github/workflows/ci-lint-go.yml | 63 ++++++++++++++++++++++++++++++++ .github/workflows/ci-test-go.yml | 47 ------------------------ .github/workflows/ci.yml | 15 ++++++++ 3 files changed, 78 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/ci-lint-go.yml diff --git a/.github/workflows/ci-lint-go.yml b/.github/workflows/ci-lint-go.yml new file mode 100644 index 0000000000..9c3ebab832 --- /dev/null +++ b/.github/workflows/ci-lint-go.yml @@ -0,0 +1,63 @@ +name: Run lint for a Go project +run-name: "${{ inputs.project-directory }}" + +on: + workflow_call: + inputs: + project-directory: + required: true + type: string + default: "." + description: "The directory where the Go project is located." + +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read + +jobs: + lint-go-project: + name: "lint: ${{ inputs.project-directory }}" + runs-on: 'ubuntu-latest' + continue-on-error: false + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 + with: + go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod" + cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum" + id: go + + - name: golangci-lint + uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.61.0 + # Optional: working directory, useful for monorepos + working-directory: ${{ inputs.project-directory }} + # Optional: golangci-lint command line arguments. + args: --verbose + # Optional: if set to true then the all caching functionality will be complete disabled, + # takes precedence over all other caching options. + skip-cache: true + + - name: generate + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make generate + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] + + - name: modVerify + working-directory: ./${{ inputs.project-directory }} + run: go mod verify + + - name: modTidy + working-directory: ./${{ inputs.project-directory }} + shell: bash + run: | + make tidy + git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] diff --git a/.github/workflows/ci-test-go.yml b/.github/workflows/ci-test-go.yml index deba5690f2..ad0f8150d8 100644 --- a/.github/workflows/ci-test-go.yml +++ b/.github/workflows/ci-test-go.yml @@ -40,55 +40,8 @@ permissions: # pull-requests: read jobs: - lint-go-project: - name: "lint: ${{ inputs.project-directory }}" - runs-on: 'ubuntu-latest' - continue-on-error: false - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - - name: Set up Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 - with: - go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod" - cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum" - id: go - - - name: golangci-lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 - with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.61.0 - # Optional: working directory, useful for monorepos - working-directory: ${{ inputs.project-directory }} - # Optional: golangci-lint command line arguments. - args: --verbose - # Optional: if set to true then the all caching functionality will be complete disabled, - # takes precedence over all other caching options. - skip-cache: true - - - name: generate - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make generate - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - - - name: modVerify - working-directory: ./${{ inputs.project-directory }} - run: go mod verify - - - name: modTidy - working-directory: ./${{ inputs.project-directory }} - shell: bash - run: | - make tidy - git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]] - test-go-project: name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}" - needs: lint-go-project # Modulegen can run the tests on all platforms continue-on-error: ${{ !inputs.fail-fast }} env: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c51a9c621..80371362df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,11 +39,24 @@ jobs: echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build" echo "${{ steps.set-modified-modules.outputs.modules }}" + lint: + # only run if there are modules to lint + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + strategy: + matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} + uses: ./.github/workflows/ci-lint-go.yml + with: + project-directory: "${{ matrix.module }}" + test: # only run if there are modules to test if: ${{ needs.detect-modules.outputs.modules_count > 0 }} needs: - detect-modules + - lint strategy: matrix: go-version: [1.22.x, 1.x] @@ -64,6 +77,7 @@ jobs: if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }} needs: - detect-modules + - lint name: "Test with reaper off" strategy: matrix: @@ -84,6 +98,7 @@ jobs: if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }} needs: - detect-modules + - lint name: "Test with Rootless Docker" strategy: matrix: