From 4dbb7b9f6e42acb396f049480138c47c47f34d24 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Tue, 3 Dec 2024 12:07:39 +1100 Subject: [PATCH 1/3] chore: move to reusable docker workflows --- .github/actions/save-docker-image/action.yml | 24 ++++++++++++++++++ .../actions/shard-docker-builds/action.yml | 13 ++++++++++ .github/workflows/ci.yml | 25 ++++++++++--------- .github/workflows/release.yml | 22 ++++++---------- 4 files changed, 57 insertions(+), 27 deletions(-) create mode 100644 .github/actions/save-docker-image/action.yml create mode 100644 .github/actions/shard-docker-builds/action.yml diff --git a/.github/actions/save-docker-image/action.yml b/.github/actions/save-docker-image/action.yml new file mode 100644 index 0000000000..f13838d9e2 --- /dev/null +++ b/.github/actions/save-docker-image/action.yml @@ -0,0 +1,24 @@ +name: Save Docker Images +description: Builds all docker images and saves them as artifacts +inputs: + service: + description: The service to build + required: true +runs: + using: "composite" + # Make sure to keep these cache entries in sync with those in writecache.yml + steps: + - name: Init Hermit + uses: cashapp/activate-hermit@v1.1.3 + - name: Build Docker Image + shell: bash + run: | + just build-docker ${{ inputs.service }} + mkdir -p artifacts/ftl-${{ inputs.service }} + docker save -o artifacts/ftl-${{ inputs.service }}/ftl-${{ inputs.service }}.tar ftl0/ftl-${{ inputs.service }}:latest + - name: Temporarily save Docker image + uses: actions/upload-artifact@v4 + with: + name: docker-${{ inputs.service }}-artifact + path: artifacts/ftl-${{ inputs.service }} + retention-days: 1 diff --git a/.github/actions/shard-docker-builds/action.yml b/.github/actions/shard-docker-builds/action.yml new file mode 100644 index 0000000000..051f3ce0d9 --- /dev/null +++ b/.github/actions/shard-docker-builds/action.yml @@ -0,0 +1,13 @@ +name: Shard Docker Builds +description: Outputs a matrix of all services to build +outputs: + matrix: + description: A matrix of all services to build + value: ${{ steps.set-matrix.outputs.matrix }} +runs: + using: "composite" + steps: + - uses: cashapp/activate-hermit@v1.1.3 + - id: set-matrix + shell: bash + run: echo "matrix=$(just list-docker-images | tr -d '\n' | jq -R -s -c 'split(" ")')" >> "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e62d8c50d..6d5ee330c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -224,24 +224,25 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v4 - - uses: cashapp/activate-hermit@v1.1.3 + - name: Checkout code + uses: actions/checkout@v4 - id: set-matrix - run: echo "matrix=$(just list-docker-images | tr -d '\n' | jq -R -s -c 'split(" ")')" >> "$GITHUB_OUTPUT" - docker-build: + name: Shard Docker Builds + uses: ./.github/actions/shard-docker-builds + build-docker-images: name: Build ${{ matrix.service }} Docker Image needs: docker-shard runs-on: ubuntu-latest strategy: - fail-fast: false matrix: service: ${{ fromJson(needs.docker-shard.outputs.matrix) }} steps: - - uses: actions/checkout@v4 - - uses: cashapp/activate-hermit@v1.1.3 - - uses: ./.github/actions/build-cache - - name: Build Docker Image - run: just build-docker ${{ matrix.service }} + - name: Checkout code + uses: actions/checkout@v4 + - name: Build Image and Save + uses: ./.github/actions/save-docker-image + with: + service: ${{ matrix.service }} console-e2e: name: Console e2e # if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all') @@ -394,13 +395,13 @@ jobs: fi docker-success: name: Docker Success - needs: [docker-build] + needs: [build-docker-images] runs-on: ubuntu-latest if: ${{ always() }} steps: - name: Check docker builds result run: | - if [[ "${{ needs.docker-build.result }}" == "failure" ]]; then + if [[ "${{ needs.build-docker-images.result }}" == "failure" ]]; then echo "Docker builds failed" exit 1 else diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2d05f0cfb6..c796fbc379 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,10 +10,11 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v4 - - uses: cashapp/activate-hermit@v1.1.3 + - name: Checkout code + uses: actions/checkout@v4 - id: set-matrix - run: echo "matrix=$(just list-docker-images | tr -d '\n' | jq -R -s -c 'split(" ")')" >> "$GITHUB_OUTPUT" + name: Shard Docker Builds + uses: ./.github/actions/shard-docker-builds build-docker-images: name: Build ${{ matrix.service }} Docker Image needs: docker-shard @@ -24,19 +25,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - name: Init Hermit - uses: cashapp/activate-hermit@v1.1.3 - - name: Build Docker Image - run: | - just build-docker ${{ matrix.service }} - mkdir -p artifacts/ftl-${{ matrix.service }} - docker save -o artifacts/ftl-${{ matrix.service }}/ftl-${{ matrix.service }}.tar ftl0/ftl-${{ matrix.service }}:latest - - name: Temporarily save Docker image - uses: actions/upload-artifact@v4 + - name: Build Image and Save + uses: ./.github/actions/save-docker-image with: - name: docker-${{ matrix.service }}-artifact - path: artifacts/ftl-${{ matrix.service }} - retention-days: 1 + service: ${{ matrix.service }} release-docker: name: Release Docker Images runs-on: ubuntu-latest From 093833edb7ce127e2153c0f8655e8a64f685d41a Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Wed, 4 Dec 2024 08:14:29 +1100 Subject: [PATCH 2/3] tmp --- .github/actions/load-docker-images/action.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/actions/load-docker-images/action.yml diff --git a/.github/actions/load-docker-images/action.yml b/.github/actions/load-docker-images/action.yml new file mode 100644 index 0000000000..75d7159cc7 --- /dev/null +++ b/.github/actions/load-docker-images/action.yml @@ -0,0 +1,22 @@ +name: Load Docker Images +description: Loads all docker images built in previous steps +inputs: + service: + description: The service to build + required: true +runs: + using: "composite" + # Make sure to keep these cache entries in sync with those in writecache.yml + steps: + - name: Init Hermit + uses: cashapp/activate-hermit@v1.1.3 + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + pattern: docker-*-artifact + - name: Load Docker images + run: | + for tar in artifacts/docker-*/ftl-*.tar; do + docker load -i "$tar" + done \ No newline at end of file From a9557932c7ca1a5b16df2fcd7db4c7131a5a13c4 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Wed, 4 Dec 2024 09:16:37 +1100 Subject: [PATCH 3/3] tmp --- .github/actions/load-docker-images/action.yml | 8 +------ .github/actions/save-docker-image/action.yml | 3 --- .../actions/shard-docker-builds/action.yml | 1 - .github/workflows/ci.yml | 24 +++++++++++++------ .github/workflows/release.yml | 14 ++++------- 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/.github/actions/load-docker-images/action.yml b/.github/actions/load-docker-images/action.yml index 75d7159cc7..ed7602ea07 100644 --- a/.github/actions/load-docker-images/action.yml +++ b/.github/actions/load-docker-images/action.yml @@ -1,21 +1,15 @@ name: Load Docker Images description: Loads all docker images built in previous steps -inputs: - service: - description: The service to build - required: true runs: using: "composite" - # Make sure to keep these cache entries in sync with those in writecache.yml steps: - - name: Init Hermit - uses: cashapp/activate-hermit@v1.1.3 - name: Download all artifacts uses: actions/download-artifact@v4 with: path: artifacts pattern: docker-*-artifact - name: Load Docker images + shell: bash run: | for tar in artifacts/docker-*/ftl-*.tar; do docker load -i "$tar" diff --git a/.github/actions/save-docker-image/action.yml b/.github/actions/save-docker-image/action.yml index f13838d9e2..1a57cfa8f2 100644 --- a/.github/actions/save-docker-image/action.yml +++ b/.github/actions/save-docker-image/action.yml @@ -6,10 +6,7 @@ inputs: required: true runs: using: "composite" - # Make sure to keep these cache entries in sync with those in writecache.yml steps: - - name: Init Hermit - uses: cashapp/activate-hermit@v1.1.3 - name: Build Docker Image shell: bash run: | diff --git a/.github/actions/shard-docker-builds/action.yml b/.github/actions/shard-docker-builds/action.yml index 051f3ce0d9..94a22fc82b 100644 --- a/.github/actions/shard-docker-builds/action.yml +++ b/.github/actions/shard-docker-builds/action.yml @@ -7,7 +7,6 @@ outputs: runs: using: "composite" steps: - - uses: cashapp/activate-hermit@v1.1.3 - id: set-matrix shell: bash run: echo "matrix=$(just list-docker-images | tr -d '\n' | jq -R -s -c 'split(" ")')" >> "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d5ee330c2..58efc72693 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,6 +226,7 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - uses: cashapp/activate-hermit@v1.1.3 - id: set-matrix name: Shard Docker Builds uses: ./.github/actions/shard-docker-builds @@ -239,6 +240,7 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - uses: cashapp/activate-hermit@v1.1.3 - name: Build Image and Save uses: ./.github/actions/save-docker-image with: @@ -333,7 +335,9 @@ jobs: infrastructure-run: name: Infrastructure Test #if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all') - needs: infrastructure-shard + needs: + - infrastructure-shard + - build-docker-images runs-on: ubuntu-latest strategy: fail-fast: false @@ -349,16 +353,22 @@ jobs: uses: actions/checkout@v4 - name: Init Hermit uses: cashapp/activate-hermit@v1.1.3 + - name: Load Docker Images + uses: ./.github/actions/load-docker-images + - name: Start Cluster + run: just k8s setup-istio-cluster + - name: Tag and Push Docker Images + run: | + for image in $(just list-docker-images); do + docker tag "ftl0/ftl-${image}:latest" "localhost:5000/ftl-${image}:latest" + docker push "localhost:5000/ftl-${image}:latest" + done - name: Build Cache uses: ./.github/actions/build-cache - - name: Docker Compose - run: just compose-up - - name: Create DB - run: just init-db - name: Download Go Modules run: go mod download - - name: Build Test Images - run: just k8s full-deploy + - name: Start FTL + run: just k8s apply - name: Build Language Plugins run: just build-language-plugins - name: Run ${{ matrix.test }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c796fbc379..7d386babc1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Init Hermit + uses: cashapp/activate-hermit@v1.1.3 - name: Build Image and Save uses: ./.github/actions/save-docker-image with: @@ -43,16 +45,8 @@ jobs: fetch-depth: 0 - name: Init Hermit uses: cashapp/activate-hermit@v1.1.3 - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - pattern: docker-*-artifact - - name: Load Docker images - run: | - for tar in artifacts/docker-*/ftl-*.tar; do - docker load -i "$tar" - done + - name: Load Docker Images + uses: ./.github/actions/load-docker-images - name: Log in to Container registry uses: docker/login-action@v3 with: