chore: e2e improve docker cache (#16)

- Add multiple CI jobs to run e2e - `e2e-docker-build-*` builds docker and saves it as an artifact - `e2e-run-*` runs each e2e test individually - remove the need of sudo on running e2e tests due to removing error check in ClearResources 🥲 it errors out due to `unlinkat /tmp/bbn-e2e-testnet-2820217771/bbn-test-a/bbn-test-a-node-babylon-default-a-2/ibc_08-wasm/state/wasm: permission denied` - removing the need of sudo to run e2e speeds up the run because it uses the go cache created from job `build` TLDR; It is not a pretty solution, but reduces the CI running time of e2e > Accepting suggestions/inputs to make it better ![image](https://github.com/user-attachments/assets/acfb51e9-d09e-4e60-b384-34fc11c0aa41) :heavy_check_mark: CI time reduced to 8min
babylonlabs-io · Aug 14, 2024 · f6ca3ec · f6ca3ec
1 parent e851c12
commit f6ca3ec
Show file tree

Hide file tree

Showing 15 changed files with 395 additions and 97 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,23 +3,253 @@ name: ci
 on:
   push:
     branches:
-    - '**'
+      - '**'
+
+concurrency:
+  group: ci-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
 
 jobs:
   lint_test:
     uses: babylonlabs-io/.github/.github/workflows/[email protected]
     secrets: inherit
     with:
      run-unit-tests: true
-     run-integration-tests: true
+     run-integration-tests: false
      run-lint: true
-     integration-tests-command: |
-      sudo make test-e2e
-     
+
   docker_pipeline:
     uses: babylonlabs-io/.github/.github/workflows/[email protected]
     secrets: inherit
     with:
      publish: false
      dockerfile: ./contrib/images/babylond/Dockerfile
-     repoName: babylond
+     repoName: babylond
+
+###############################################################################
+###                                   E2E                                   ###
+###############################################################################
+
+  e2e-docker-build-babylon:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build docker babylond
+        run: |
+          make -C contrib/images babylond-e2e
+      - name: Docker save babylon
+        run: |
+          docker save -o /tmp/docker-babylond.tar.gz babylonlabs-io/babylond:latest
+      - name: Upload babylon artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: babylond-${{ github.sha }} # so it renovates at every new sha
+          path: /tmp/docker-babylond.tar.gz
+
+  e2e-docker-build-babylon-before-upgrade:
+    runs-on: ubuntu-22.04
+    steps:
+      # - name: Artifact babylond-before-upgrade exists # TODO: check why it doesn't work to load from older workflow runs .-.
+      #   uses: xSAVIKx/artifact-exists-action@v0
+      #   id: check-babylond-before-upgrade
+      #   with:
+      #     name: babylond-before-upgrade
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build docker babylond-before-upgrade
+        run: |
+          make -C contrib/images babylond-before-upgrade
+      - name: Docker save babylond-before-upgrade
+        run: |
+          docker save -o /tmp/docker-babylond-before-upgrade.tar.gz babylonlabs-io/babylond-before-upgrade:latest
+      - name: Upload babylond-before-upgrade artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: babylond-before-upgrade
+          path: /tmp/docker-babylond-before-upgrade.tar.gz
+          # retention-days: 90 active this back if artifact-exists-action works
+
+  e2e-docker-build-e2e-init-chain:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build docker init-chain
+        run: |
+          make -C contrib/images e2e-init-chain
+      - name: Docker save init-chain
+        run: |
+          docker save -o /tmp/docker-init-chain.tar.gz babylonlabs-io/babylond-e2e-init-chain:latest
+      - name: Upload init-chain artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: init-chain
+          path: /tmp/docker-init-chain.tar.gz
+          # retention-days: 90
+
+  e2e-run-ibc-transfer:
+    needs: [e2e-docker-build-babylon]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Download babylon artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: babylond-${{ github.sha }}
+          path: /tmp
+      - name: Docker load babylond
+        run: |
+          docker load < /tmp/docker-babylond.tar.gz
+      - name: Login to Docker Hub # load hermes
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Cache Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+      - name: Run e2e TestIBCTranferTestSuite
+        run: |
+          make test-e2e-cache-ibc-transfer
+
+  e2e-run-btc-timestamping:
+    needs: [e2e-docker-build-babylon]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Download babylon artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: babylond-${{ github.sha }}
+          path: /tmp
+      - name: Docker load babylond
+        run: |
+          docker load < /tmp/docker-babylond.tar.gz
+      - name: Cache Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+      - name: Run e2e TestBTCTimestampingTestSuite
+        run: |
+          make test-e2e-cache-btc-timestamping
+
+  e2e-run-btc-timestamping-phase-2-hermes:
+    needs: [e2e-docker-build-babylon]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Download babylon artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: babylond-${{ github.sha }}
+          path: /tmp
+      - name: Docker load babylond
+        run: |
+          docker load < /tmp/docker-babylond.tar.gz
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Cache Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+      - name: Run e2e TestBTCTimestampingPhase2HermesTestSuite
+        run: |
+          make test-e2e-cache-btc-timestamping-phase-2-hermes
+
+  e2e-run-btc-timestamping-phase-2-rly:
+    needs: [e2e-docker-build-babylon]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Download babylon artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: babylond-${{ github.sha }}
+          path: /tmp
+      - name: Docker load babylond
+        run: |
+          docker load < /tmp/docker-babylond.tar.gz
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Cache Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+      - name: Run e2e TestBTCTimestampingPhase2RlyTestSuite
+        run: |
+          make test-e2e-cache-btc-timestamping-phase-2-rly
+
+  e2e-run-btc-staking:
+    needs: [e2e-docker-build-babylon]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Download babylon artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: babylond-${{ github.sha }}
+          path: /tmp
+      - name: Docker load babylond
+        run: |
+          docker load < /tmp/docker-babylond.tar.gz
+      - name: Cache Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+      - name: Run e2e TestBTCStakingTestSuite
+        run: |
+          make test-e2e-cache-btc-staking
+
+  e2e-run-upgrade-vanilla:
+    needs: [e2e-docker-build-babylon, e2e-docker-build-babylon-before-upgrade, e2e-docker-build-e2e-init-chain]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Download babylon artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: babylond-${{ github.sha }}
+          path: /tmp
+      - name: Download babylond-before-upgrade artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: babylond-before-upgrade
+          path: /tmp
+      - name: Download init-chain artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: init-chain
+          path: /tmp
+      - name: Docker load babylond
+        run: |
+          docker load < /tmp/docker-babylond.tar.gz
+
+      - name: Docker load babylond-before-upgrade
+        run: |
+          docker load < /tmp/docker-babylond-before-upgrade.tar.gz
+
+      - name: Docker load init chain
+        run: |
+          docker load < /tmp/docker-init-chain.tar.gz
+
+      - name: Cache Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+      - name: Run e2e TestSoftwareUpgradeTestSuite
+        run: |
+          sudo make test-e2e-cache-upgrade-vanilla
diff --git a/Makefile b/Makefile
@@ -147,7 +147,7 @@ build: BUILD_ARGS=-o $(BUILDDIR)/ ## Build babylond binary
 build-linux: ## Build babylond linux version binary
 	GOOS=linux GOARCH=$(if $(findstring aarch64,$(shell uname -m)) || $(findstring arm64,$(shell uname -m)),arm64,amd64) LEDGER_ENABLED=false $(MAKE) build
 
-$(BUILD_TARGETS): go.sum $(BUILDDIR)/
+$(BUILD_TARGETS): $(BUILDDIR)/
 	go $@ -mod=readonly $(BUILD_FLAGS) $(BUILD_ARGS) ./...
 
 $(BUILDDIR)/:
@@ -254,8 +254,28 @@ endif
 
 .PHONY: run-tests test test-all $(TEST_TARGETS)
 
-test-e2e: build-docker-e2e
-	go test -mod=readonly -timeout=25m -v $(PACKAGES_E2E) -count=1 --tags=e2e
+test-e2e: build-docker-e2e test-e2e-cache
+
+test-e2e-cache:
+	go test -mod=readonly -timeout=60m -v $(PACKAGES_E2E) --tags=e2e
+
+test-e2e-cache-ibc-transfer:
+	go test -run TestIBCTranferTestSuite -mod=readonly -timeout=60m -v $(PACKAGES_E2E) --tags=e2e
+
+test-e2e-cache-btc-timestamping:
+	go test -run TestBTCTimestampingTestSuite -mod=readonly -timeout=60m -v $(PACKAGES_E2E) --tags=e2e
+
+test-e2e-cache-btc-timestamping-phase-2-hermes:
+	go test -run TestBTCTimestampingPhase2HermesTestSuite -mod=readonly -timeout=60m -v $(PACKAGES_E2E) --tags=e2e
+
+test-e2e-cache-btc-timestamping-phase-2-rly:
+	go test -run TestBTCTimestampingPhase2RlyTestSuite -mod=readonly -timeout=60m -v $(PACKAGES_E2E) --tags=e2e
+
+test-e2e-cache-btc-staking:
+	go test -run TestBTCStakingTestSuite -mod=readonly -timeout=60m -v $(PACKAGES_E2E) --tags=e2e
+
+test-e2e-cache-upgrade-vanilla:
+	go test -run TestSoftwareUpgradeTestSuite -mod=readonly -timeout=60m -v $(PACKAGES_E2E) --tags=e2e
 
 test-sim-nondeterminism:
 	@echo "Running non-determinism test..."

diff --git a/contrib/images/Makefile b/contrib/images/Makefile
@@ -4,20 +4,26 @@ BABYLON_VERSION_BEFORE_UPGRADE ?= v0.9.0
 
 all: babylond cosmos-relayer
 
-babylond: babylond-rmi
+babylond:
 	docker build --tag babylonlabs-io/babylond -f babylond/Dockerfile ${BABYLON_FULL_PATH}
 
-babylond-e2e: babylond-rmi
+babylond-e2e:
 	docker build --tag babylonlabs-io/babylond -f babylond/Dockerfile ${BABYLON_FULL_PATH} \
 		--build-arg BUILD_TAGS="e2e"
 
+## TODO: once release docker public versions for tags, remove this!
 babylond-before-upgrade:
-	docker rmi babylonlabs-io/babylond-before-upgrade 2>/dev/null; true && \
 	docker build --tag babylonlabs-io/babylond-before-upgrade -f babylond/Dockerfile \
     --build-arg VERSION="${BABYLON_VERSION_BEFORE_UPGRADE}" ${BABYLON_FULL_PATH}
 
 babylond-rmi:
-	docker rmi babylonlabs-io/babylond 2>/dev/null; true
+	docker rmi babylonlabs-io/babylond --force 2>/dev/null; true
+
+babylond-rmi-upgrade:
+	docker rmi babylonlabs-io/babylond-before-upgrade --force 2>/dev/null; true
+
+e2e-init-chain-rmi:
+	docker rmi babylonlabs-io/babylond-e2e-init-chain --force 2>/dev/null; true
 
 e2e-init-chain:
 	@DOCKER_BUILDKIT=1 docker build -t babylonlabs-io/babylond-e2e-init-chain --build-arg E2E_SCRIPT_NAME=chain --platform=linux/x86_64 \
@@ -31,4 +37,4 @@ cosmos-relayer: cosmos-relayer-rmi
 cosmos-relayer-rmi:
 	docker rmi babylonlabs-io/cosmos-relayer 2>/dev/null; true
 
-.PHONY: all babylond babylond-before-upgrade cosmos-relayer e2e-init-chain babylond-rmi cosmos-relayer-rmi
+.PHONY: all babylond babylond-before-upgrade cosmos-relayer e2e-init-chain babylond-rmi babylond-rmi-upgrade cosmos-relayer-rmi
diff --git a/contrib/images/babylond/Dockerfile b/contrib/images/babylond/Dockerfile
@@ -2,38 +2,31 @@ FROM golang:1.21 AS build-env
 
 # Version to build. Default is empty
 ARG VERSION
-
 ARG BUILD_TAGS=""
 ARG LEDGER_ENABLED="false"
-# Cosmos build options
 ARG COSMOS_BUILD_OPTIONS=""
 
-# Install cli tools for building and final image
-RUN apt-get update && apt-get install -y make git bash gcc curl jq
-
-# Build
 WORKDIR /go/src/github.com/babylonlabs-io/babylon
-# First cache dependencies
-COPY go.mod go.sum /go/src/github.com/babylonlabs-io/babylon/
-RUN go mod download
-# Then copy everything else
 COPY ./ /go/src/github.com/babylonlabs-io/babylon/
-# If version is set, then checkout this version
+
+# Handle if version is set
 RUN if [ -n "${VERSION}" ]; then \
-        git fetch --all; \
+        git fetch origin tag ${VERSION} --no-tags; \
         git checkout -f ${VERSION}; \
     fi
 
+# Cache mod dependencies
+RUN go mod download
+
 RUN LEDGER_ENABLED=$LEDGER_ENABLED \
     BUILD_TAGS=$BUILD_TAGS \
     COSMOS_BUILD_OPTIONS=$COSMOS_BUILD_OPTIONS \
     LINK_STATICALLY=false \
     make build
 
-FROM debian:bookworm-slim AS run
-# Create a user
-RUN addgroup --gid 1137 --system babylon && adduser --uid 1137 --gid 1137 --system --home /home/babylon babylon
-RUN apt-get update && apt-get install -y bash curl jq wget
+FROM debian:bookworm-slim AS wasm-link
+
+RUN apt-get update && apt-get install -y wget bash
 
 # Label should match your github repo
 LABEL org.opencontainers.image.source="https://github.com/babylonlabs-io/babylond:${VERSION}"
@@ -50,9 +43,3 @@ RUN WASMVM_VERSION=$(grep github.com/CosmWasm/wasmvm /tmp/go.mod | cut -d' ' -f2
 RUN rm -f /tmp/go.mod
 
 COPY --from=build-env /go/src/github.com/babylonlabs-io/babylon/build/babylond /bin/babylond
-
-# Set home directory and user
-WORKDIR /home/babylon
-RUN chown -R babylon /home/babylon
-RUN chmod g+s /home/babylon
-USER babylon