chore(ci): run lint in a separate build before running the tests

.github/workflows/ci-test-go.yml

@@ -1,5 +1,5 @@
 name: Run tests for a Go project
-run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platform }}"
+run-name: "${{ inputs.project-directory }} ${{ inputs.go-version }} ${{ inputs.platforms }}"
 
 on:
   workflow_call:
@@ -8,15 +8,16 @@ on:
         required: true
         type: string
         description: "The version of Go to use for the test."
-      platform:
-        required: true
-        type: string
-        description: "The platform to run the test on."
       fail-fast:
         required: false
         type: boolean
         default: true
         description: "Fail the workflow if any of the jobs fail."
+      platforms:
+        required: true
+        type: string
+        default: "ubuntu-latest"
+        description: "The platforms in which the project will be run"
       project-directory:
         required: true
         type: string
@@ -27,11 +28,6 @@ on:
         type: boolean
         default: false
         description: "Run the test with rootless docker."
-      run-tests:
-        required: false
-        type: boolean
-        default: true
-        description: "Run the tests under conditions controlled by the caller workflow."
       ryuk-disabled:
         required: false
         type: boolean
@@ -44,35 +40,22 @@ permissions:
   # pull-requests: read
 
 jobs:
-  test-go-project:
-    name: "${{ inputs.project-directory }}/${{ inputs.platform }}/${{ inputs.go-version }}"
-    runs-on: ${{ inputs.platform }}
-    continue-on-error: ${{ !inputs.fail-fast }}
-    env:
-      TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}"
-      RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}"
-      RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}"
+  lint-go-project:
+    name: "lint: ${{ inputs.project-directory }}"
+    runs-on: 'ubuntu-latest'
+    continue-on-error: false
     steps:
-      - name: Setup rootless Docker
-        if: ${{ inputs.rootless-docker }}
-        uses: ScribeMD/rootless-docker@6bd157a512c2fafa4e0243a8aa87d964eb890886  # v0.2.2
-
-      - name: Remove Docker root socket
-        if: ${{ inputs.rootless-docker }}
-        run: sudo rm -rf /var/run/docker.sock
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
       - name: Set up Go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5
         with:
-            go-version: '${{ inputs.go-version }}'
-            cache-dependency-path: '${{ inputs.project-directory }}/go.sum'
+            go-version-file: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.mod"
+            cache-dependency-path: "${{ inputs.project-directory == '' && '.' || inputs.project-directory }}/go.sum"
         id: go
 
       - name: golangci-lint
-        if: ${{ inputs.platform == 'ubuntu-latest' }}
         uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
@@ -86,7 +69,6 @@ jobs:
           skip-cache: true
 
       - name: generate
-        if: ${{ inputs.platform == 'ubuntu-latest' }}
         working-directory: ./${{ inputs.project-directory }}
         shell: bash
         run: |
@@ -98,30 +80,56 @@ jobs:
         run: go mod verify
 
       - name: modTidy
-        if: ${{ inputs.platform == 'ubuntu-latest' }}
         working-directory: ./${{ inputs.project-directory }}
         shell: bash
         run: |
           make tidy
           git --no-pager diff && [[ 0 -eq $(git status --porcelain | wc -l) ]]
 
+  test-go-project:
+    name: "test: ${{ inputs.project-directory }}/${{ inputs.go-version }}"
+    needs: lint-go-project
+    # Modulegen can run the tests on all platforms
+    continue-on-error: ${{ !inputs.fail-fast }}
+    env:
+      TESTCONTAINERS_RYUK_DISABLED: "${{ inputs.ryuk-disabled }}"
+      RYUK_CONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '5m' || '60s' }}"
+      RYUK_RECONNECTION_TIMEOUT: "${{ inputs.project-directory == 'modules/compose' && '30s' || '10s' }}"
+    strategy:
+      matrix:
+        platform: ${{ fromJSON(inputs.platforms) }}
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Setup rootless Docker
+        if: ${{ inputs.rootless-docker }}
+        uses: ScribeMD/rootless-docker@6bd157a512c2fafa4e0243a8aa87d964eb890886  # v0.2.2
+
+      - name: Remove Docker root socket
+        if: ${{ inputs.rootless-docker }}
+        run: sudo rm -rf /var/run/docker.sock
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - name: Set up Go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5
+        with:
+            go-version: '${{ inputs.go-version }}'
+            cache-dependency-path: '${{ inputs.project-directory }}/go.sum'
+        id: go
+
       - name: ensure compilation
         working-directory: ./${{ inputs.project-directory }}
         run: go build
 
       - name: go test
-        # only run tests on linux, there are a number of things that won't allow the tests to run on anything else
-        # many (maybe, all?) images used can only be build on Linux, they don't have Windows in their manifest, and
-        # we can't put Windows Server in "Linux Mode" in Github actions
-        # another, host mode is only available on Linux, and we have tests around that, do we skip them?
-        if: ${{ inputs.run-tests }}
         working-directory: ./${{ inputs.project-directory }}
         timeout-minutes: 30
         run: make test-unit
 
       - name: Upload SonarCloud files
-        if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && inputs.run-tests && !inputs.rootless-docker }}
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v3
+        if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && runner.name == 'ubuntu-latest' && !inputs.rootless-docker }}
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: sonarcloud
           path: |

.github/workflows/ci.yml

@@ -1,46 +1,69 @@
-# This file is autogenerated by the 'modulegen' tool.
-# Please update the 'ci.yml' template instead.
 name: Main pipeline
 
 on:
   push:
     branches:
       - main
-    paths-ignore:
-      - '.vscode/**'
-      - 'mkdocs.yml'
-      - 'docs/**'
-      - 'README.md'
   pull_request:
-    paths-ignore:
-      - '.vscode/**'
-      - 'mkdocs.yml'
-      - 'docs/**'
-      - 'README.md'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
 jobs:
+  detect-modules:
+    runs-on: ubuntu-latest
+    outputs:
+      modules: ${{ steps.set-modified-modules.outputs.modules }}
+      modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }}
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - id: changed-files
+        name: Get changed files
+        uses: tj-actions/changed-files@v45
+
+      - id: set-modified-modules
+        name: Set all modified modules
+        env:
+          ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}"
+        run: echo "modules=$(./scripts/changed-modules.sh)" >> $GITHUB_OUTPUT
+
+      - id: set-modified-modules-count
+        name: Set all modified modules count
+        run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT
+
+      - name: Print out the modules to be used
+        run: |
+          echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build"
+          echo "${{ steps.set-modified-modules.outputs.modules }}"
+
   test:
+    # only run if there are modules to test
+    if: ${{ needs.detect-modules.outputs.modules_count > 0  }}
+    needs:
+      - detect-modules
     strategy:
       matrix:
         go-version: [1.22.x, 1.x]
-        platform: [ubuntu-latest, macos-latest]
+        module: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: ${{ matrix.go-version }}
       fail-fast: true
-      platform: ${{ matrix.platform }}
-      project-directory: "."
+      platforms: ${{ matrix.module == 'modulegen' && '["ubuntu-latest", "macos-latest", "windows-latest"]' || '["ubuntu-latest"]' }}
+      project-directory: "${{ matrix.module }}"
       rootless-docker: false
-      run-tests: ${{ matrix.platform == 'ubuntu-latest' }}
       ryuk-disabled: false
 
   # The job below is a copy of the job above, but with ryuk disabled.
   # It's executed in the first stage to avoid concurrency issues.
   test-reaper-off:
+    # the core module is identified by the empty string (the root path)
+    if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }}
+    needs:
+      - detect-modules
     name: "Test with reaper off"
     strategy:
       matrix:
@@ -49,83 +72,39 @@ jobs:
     with:
       go-version: ${{ matrix.go-version }}
       fail-fast: false
-      platform: "ubuntu-latest"
+      platforms: '["ubuntu-latest"]'
       project-directory: "."
       rootless-docker: false
-      run-tests: true
       ryuk-disabled: true
 
   # The job below is a copy of the job above, but with Docker rootless.
   # It's executed in the first stage to avoid concurrency issues.
   test-rootless-docker:
+    # the core module is identified by the empty string (the root path)
+    if: ${{ contains(fromJSON(needs.detect-modules.outputs.modules), '') }}
+    needs:
+      - detect-modules
     name: "Test with Rootless Docker"
     strategy:
       matrix:
         go-version: [1.22.x, 1.x]
-        platform: [ubuntu-latest]
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: ${{ matrix.go-version }}
       fail-fast: false
-      platform: "ubuntu-latest"
+      platforms: '["ubuntu-latest"]'
       project-directory: "."
       rootless-docker: true
-      run-tests: true
-      ryuk-disabled: false
-
-  test-module-generator:
-    strategy:
-      matrix:
-        go-version: [1.22.x, 1.x]
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-    uses: ./.github/workflows/ci-test-go.yml
-    with:
-      go-version: ${{ matrix.go-version }}
-      fail-fast: true
-      platform: ${{ matrix.platform }}
-      project-directory: "modulegen"
-      rootless-docker: false
-      run-tests: true
-      ryuk-disabled: false
-
-  test-modules:
-    needs: test
-    strategy:
-      matrix:
-        go-version: [1.22.x, 1.x]
-        platform: [ubuntu-latest]
-        module: [artemis, azurite, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, databend, dolt, dynamodb, elasticsearch, etcd, gcloud, grafana-lgtm, inbucket, influxdb, k3s, k6, kafka, localstack, mariadb, meilisearch, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, registry, surrealdb, valkey, vault, vearch, weaviate, yugabytedb]
-    uses: ./.github/workflows/ci-test-go.yml
-    with:
-      go-version: ${{ matrix.go-version }}
-      fail-fast: false
-      platform: ${{ matrix.platform }}
-      project-directory: modules/${{ matrix.module }}
-      rootless-docker: false
-      run-tests: ${{ matrix.platform == 'ubuntu-latest' }}
-      ryuk-disabled: false
-
-  test-examples:
-    needs: test-modules
-    strategy:
-      matrix:
-        module: [nginx, toxiproxy]
-    uses: ./.github/workflows/ci-test-go.yml
-    with:
-      go-version: "1.22.x"
-      fail-fast: true
-      platform: 'ubuntu-latest'
-      project-directory: examples/${{ matrix.module }}
-      rootless-docker: false
-      run-tests: true
       ryuk-disabled: false
 
   sonarcloud:
     permissions:
       contents: read  # for actions/checkout to fetch code
       pull-requests: read  # for sonarsource/sonarcloud-github-action to determine which PR to decorate
-    if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' }}
-    needs: test-examples
+    if: ${{ github.ref_name == 'main' && github.repository_owner == 'testcontainers' && needs.detect-modules.outputs.modules_count > 0 }}
+    needs:
+      - detect-modules
+      - test
     runs-on: ubuntu-latest 
     steps:
       - name: Check out code into the Go module directory