From f3d51b51b51af055e0dddcf45c95aea4dcc6587e Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Fri, 12 Feb 2021 00:34:03 +0100 Subject: [PATCH 1/6] Updated checkout action to v2 Updated action to the latest version. --- .github/workflows/client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index a10ea09c23..aefaa045d4 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -12,7 +12,7 @@ jobs: build-and-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: satackey/action-docker-layer-caching@v0.0.11 continue-on-error: true # ignore the failure of a step and avoid terminating the job - name: Run Docker build From 5ff6917c589fc8231c6ea4ce10d47ce39956b860 Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Fri, 12 Feb 2021 00:36:22 +0100 Subject: [PATCH 2/6] Added disk space check command after cache restore Added a step printing disk space after the cache is restored. We had problems with executor running out of space. With this step we can debug disk usage after the cache is restored. We assume that available disk space for `/` before cache restore to be at ~20 GB. --- .github/workflows/client.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index aefaa045d4..66699b2088 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -15,6 +15,13 @@ jobs: - uses: actions/checkout@v2 - uses: satackey/action-docker-layer-caching@v0.0.11 continue-on-error: true # ignore the failure of a step and avoid terminating the job + # TODO: This step was left here intentionally so we can track disk space + # usage for a while. We were trying to fight problems with out of disk space + # that happened due to the size of data restored from cache. The cache size + # was growing linearly with subsequent workflow runs. We want to observe + # available disk space for `/`. Fresh execution starts with 20 GB, we expect + # to have no less than 15 GB after the cache is restored. + - run: sudo df -h - name: Run Docker build run: | docker build \ From 4397a16a675e75fda49726911174d1986727d7c9 Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Fri, 12 Feb 2021 00:40:39 +0100 Subject: [PATCH 3/6] Switched to Docker's build-publish-action for client's CI We experienced some problems with `satackey/action-docker-layer-caching` that caused workflow runners to get out of a disk space. This is related to the way the action stores layer incrementaly along with the previously cached ones. See: https://github.com/satackey/action-docker-layer-caching/issues/55 There is also a solution from Docker that we can use for building images but also on later stage of RFC-18 implementation to publish images to registires. See: https://github.com/marketplace/actions/build-and-push-docker-images With the new solution we can also use caching and hopefully it won't cause problems we had before. --- .github/workflows/client.yml | 47 +++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index 66699b2088..a86ed9962e 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -13,8 +13,19 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.11 - continue-on-error: true # ignore the failure of a step and avoid terminating the job + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-${{ github.ref }}- + ${{ runner.os }}-buildx- + # TODO: This step was left here intentionally so we can track disk space # usage for a while. We were trying to fight problems with out of disk space # that happened due to the size of data restored from cache. The cache size @@ -22,13 +33,19 @@ jobs: # available disk space for `/`. Fresh execution starts with 20 GB, we expect # to have no less than 15 GB after the cache is restored. - run: sudo df -h - - name: Run Docker build - run: | - docker build \ - --target gobuild \ - --tag go-build-env . - docker build \ - --tag keep-client . + + - name: Build Docker Build Image + uses: docker/build-push-action@v2 + with: + target: gobuild + tags: go-build-env + build-args: | + REVISION=${{ github.sha }} + # VERSION= ? TODO: Configure version, sample: 1.7.6 + load: true # load image to local registry to use it in next steps + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + - name: Create test results directory run: | mkdir test-results @@ -47,3 +64,15 @@ jobs: files: ./test-results/unit-tests.xml check_name: Go Test Results # name under which test results will be presented in GitHub (optional) comment_on_pr: false # turns off commenting on Pull Requests + + # This step is executed after the tests as we want to configure it eventually + # as image publication step. + - name: Build Docker Runtime Image + uses: docker/build-push-action@v2 + with: + tags: keep-client + labels: | + revision=${{ github.sha }} + # TODO: Check branch name and publish to a registry accordingly to the + # environment. + # push: true # publish to registry From 8e032fe032fdd704146a2b73933e00cb0e333a0d Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Fri, 12 Feb 2021 01:07:29 +0100 Subject: [PATCH 4/6] Removed branch name from cache key We don't need to include branch name in the cache key as GitHub cache handles the logic for accesing cachess between branches. Cache matches are performed first for the current branch according to `key` and `restore-keys` and in case of no much it checks caches on the parent and upstream branches. See: [LINK 1] Additionally caches are isolated for siblings branches, so cache from feature branch can't be accessed by another feature branch. See [LINK 2] LINK 1:https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#matching-a-cache-key LINK 2:https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache --- .github/workflows/client.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index a86ed9962e..d9d099da93 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -21,9 +21,8 @@ jobs: uses: actions/cache@v2 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.ref }}-${{ github.sha }} + key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx-${{ github.ref }}- ${{ runner.os }}-buildx- # TODO: This step was left here intentionally so we can track disk space From b5092918190576d602fa0c567727dd503d034e57 Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Fri, 12 Feb 2021 16:01:43 +0100 Subject: [PATCH 5/6] Moved arguments for versioning in Dockerfile As we expect the arguments to be updating, especially REVISION to be set to the current commit hash we need to define them closer to the place they will be used. Currently if we update a value for REVISION it causes all subsequent layers to be executed and not fetched from cache. There's actualy nothing changin besides the commit hash we pass to go build command, so all the layers can be reused. --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ed0e82569f..92a6ff63a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,5 @@ FROM golang:1.13.6-alpine3.10 AS gobuild -ARG VERSION -ARG REVISION - ENV GOPATH=/go \ GOBIN=/go/bin \ APP_NAME=keep-client \ @@ -63,6 +60,10 @@ RUN go generate ./.../gen COPY ./ $APP_DIR/ RUN go generate ./pkg/gen +# Client Versioning. +ARG VERSION +ARG REVISION + RUN GOOS=linux go build -ldflags "-X main.version=$VERSION -X main.revision=$REVISION" -a -o $APP_NAME ./ && \ mv $APP_NAME $BIN_PATH From 27d1d87636b2a40c710088525c06c8aa52f1f4bd Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Mon, 15 Feb 2021 11:29:27 +0100 Subject: [PATCH 6/6] Removed rfc-18 branch filter for CircleCI job execution Initally we added ignore filter for branches with rfc-18 prefix which sounds like a good idea taking into account that we want to run GitHub Actions for such branches, so duplication is not necessary. It turned out that we require CircleCI job to be executed as a check before merging to master. That's why we removed the filter and will execute the job anywas. It shouldn't be much of a problem as the duplication will take place only for workflows on rfc-18 branches as we filter them in GiHub Actions configuration. --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 68c1bcd86c..9838945cc6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -339,7 +339,6 @@ workflows: branches: ignore: - master - - /rfc-18\/.*/ context: keep-dev - build_token_dashboard_dapp: filters: