[ADP-3368] Add a pipeline to control the CI for the release

.buildkite/docker-build-push.nix

@@ -53,7 +53,7 @@ in
   writeScript "docker-build-push" (''
     #!${runtimeShell}
 
-    set -euo pipefail
+    set -euox pipefail
 
     export PATH=${lib.makeBinPath [ docker gnused ]}
 
@@ -78,7 +78,7 @@ in
       tags+=( "${image.imageTag}" )
       tags+=( "latest" )
     elif [[ "$git_tag" = "rc-latest" ]]; then
-      tags+=( "$git_tag" )
+      tags+=( "$git_tag")
     else
       echo 'Not pushing docker image because this is neither a rc-latest nor a v20* tag build.'
     fi

.buildkite/pipeline.yml

@@ -11,14 +11,6 @@ env:
 
 
 steps:
-  - label: 'Merge from staging only'
-    if: 'build.branch == "staging"'
-    command: './.buildkite/check-bors.sh'
-    agents:
-      system: ${linux}
-
-  - wait: ~
-    if: 'build.branch == "staging"'
 
   - label: 'Check nix (linux)'
     # Check whether regenerate.sh was applied when it had to be applied.

.buildkite/release.yml

@@ -0,0 +1,65 @@
+agents:
+  queue: "cardano-wallet"
+
+env:
+  LC_ALL: "C.UTF-8"
+  NIX_PATH: "channel:nixos-21.11"
+  NODE_STATE_DIR: "/var/lib/buildkite-agent/cache/preprod/node_db"
+
+
+steps:
+  - label: Add release commits
+    key: add-release-commits
+    commands:
+      - ./scripts/buildkite/release/release-candidate.sh
+    agents:
+      system: x86_64-linux
+
+  - label: 'Build package (linux)'
+    key: linux-package
+    depends_on: add-release-commits
+    command:
+      - ./scripts/buildkite/release/linux-package.sh
+    artifact_paths: [ "./result/linux/**" ]
+    agents:
+      system: x86_64-linux
+
+  - label: 'Refresh node state'
+    key: refresh-node-state
+    command:
+      - ./scripts/buildkite/release/fetch-preprod-snapshot.sh
+    agents:
+      system: x86_64-linux
+
+  - label: 'Build Docker Image'
+    key: build-docker
+    depends_on:
+      - add-release-commits
+    commands:
+      ./scripts/buildkite/release/docker-build.sh
+    agents:
+      system: x86_64-linux
+
+  - label: 'Smoke test docker-compose'
+    depends_on:
+      - build-docker
+      - refresh-node-state
+    commands:
+      - ./scripts/buildkite/release/docker-smoke-test.sh
+    artifact_paths:
+      - "./logs/**/*"
+    agents:
+      system: x86_64-linux
+
+  - label: 'Run linux e2e tests'
+    depends_on:
+      - add-release-commits
+      - refresh-node-state
+      - linux-package
+    commands:
+      - nix develop -c ./scripts/buildkite/release/linux-e2e.sh
+    artifact_paths:
+      - "./result/linux/**"
+      - "./logs/**/*"
+    agents:
+      system: x86_64-linux

.buildkite/tagly.yml

@@ -9,9 +9,13 @@ env:
 steps:
   - label: 'Build Docker Image (linux)'
     key: build-docker
-    command:
-      - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }'  > config/flake.nix"
-      - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import <nixpkgs> {}).path')\" --override-input customConfig path:./config -o docker-build-push"
-      - "./docker-build-push"
+    command: |
+      mkdir -p config && \
+        echo '{ outputs = _: { dockerHubRepoName = "cardanofoundation/cardano-wallet"; }; }'  \
+          > config/flake.nix
+      nix build .#pushDockerImage \
+           --override-input hostNixpkgs path:$(nix eval --impure -I $NIX_PATH --expr '(import <nixpkgs> {}).path') \
+           --override-input customConfig path:./config -o docker-build-push
+      ./docker-build-push
     agents:
       system: x86_64-linux

.gitignore

@@ -78,3 +78,4 @@ lib/wallet-benchmarks/data/membench-snapshot
 
 ## Local cluster swagger golden actual
 lib/local-cluster/data/swagger.json.actual
+test/e2e/.direnv/flake-profile*

docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.5"
 
 services:
   cardano-node:
-    image: ghcr.io/intersectmbo/cardano-node:${NODE_TAG}
+    image: cardanofoundation/cardano-wallet:${WALLET_TAG}
     environment:
       NETWORK:
       CARDANO_NODE_SOCKET_PATH: /ipc/node.socket

nix/docker.nix

@@ -20,7 +20,7 @@
 , glibcLocales ? null
 
 # Used to generate the docker image names
-, repoName ? "inputoutput/cardano-wallet"
+, repoName ? "cardanofoundation/cardano-wallet"
 }:
 
 

nix/haskell.nix

@@ -110,7 +110,17 @@ CHaP: haskell-nix: nixpkgs-recent: nodePkgs: haskell-nix.cabalProject' [
         filter = haskell-nix.haskellSourceFilter;
       };
 
-      shell = {
+      shell =
+        let
+          # To update gemset.nix, run:
+          #   nix-shell --arg bins false --run bundix
+          gems = pkgs.bundlerEnv {
+            name = "gems-cardano-wallet-e2e";
+            gemdir = ../test/e2e;
+            ruby = pkgs.ruby_3_1;
+          };
+        in
+      {
         name = "cardano-wallet-shell${lib.optionalString config.profiling "-profiled"}";
         packages = ps: builtins.attrValues (haskellLib.selectProjectPackages ps);
         tools = {
@@ -130,6 +140,11 @@ CHaP: haskell-nix: nixpkgs-recent: nodePkgs: haskell-nix.cabalProject' [
           nodePkgs.cardano-node
           cardano-addresses-cli.components.exes.cardano-address
           bech32.components.exes.bech32
+          # for e2e tests
+          gems
+          gems.wrappedRuby
+          pkgs.bundix
+          pkgs.screen
         ]) ++ (with pkgs.buildPackages.buildPackages; [
           just
           pkg-config

scripts/buildkite/release/docker-build.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env -S nix shell --command bash
+# shellcheck shell=bash
+
+set -euox pipefail
+
+git fetch --all
+
+RELEASE_COMMIT=$(buildkite-agent meta-data get "release-commit")
+
+git checkout "$RELEASE_COMMIT"
+
+mkdir -p result
+
+nix build .#dockerImage -o result/docker-image
+
+docker load < result/docker-image

scripts/buildkite/release/docker-smoke-test.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env -S nix shell 'nixpkgs#docker-compose' --command bash
+# shellcheck shell=bash
+
+set -euox pipefail
+
+mkdir -p state
+
+export NETWORK=testnet
+export WALLET_TAG=$(buildkite-agent meta-data get "release-cabal-version")
+export NODE_TAG="8.9.3"
+export NODE_DB="$(pwd)/state"
+export WALLET_DB="$(pwd)/state"
+export WALLET_PORT=8090
+
+docker-compose up -d
+mkdir -p logs
+docker-compose logs > logs/docker-compose.log
+docker-compose down

scripts/buildkite/release/fetch-preprod-snapshot.sh

@@ -0,0 +1,43 @@
+#! /usr/bin/env -S nix shell 'nixpkgs#curl' 'nixpkgs#lz4' --command bash
+# shellcheck shell=bash
+
+set -euox pipefail
+
+mkdir -p "$NODE_STATE_DIR"
+
+cd "$NODE_STATE_DIR"
+
+export failed=false
+
+curl -s https://downloads.csnapshots.io/testnet/testnet-db-snapshot.json \
+    | jq -r .[].file_name > file.new || export failed=true
+
+if [ "$failed" ]; then
+
+    if [ -f file.old ]; then
+        if [ "$(cat file.old)" == "$(cat file.new)" ]; then
+            echo "Preprod snapshot is up to date with the latest snapshot available."
+            exit 0
+        else
+            echo "New snapshot available, Downloading..."
+        fi
+    fi
+
+    export failed2=false
+    curl -o - \
+        "https://downloads.csnapshots.io/testnet/$(cat file.new)" \
+        | lz4 -c -d - | tar -x -C . || export failed2=true
+
+    if [ "$failed2" ]; then
+        echo "Failed to fetch preprod snapshot."
+    else
+        echo "Successfully fetched preprod snapshot."
+        mv file.new file.old
+
+        rm -rf db-new
+
+        mv db db-new
+    fi
+else
+    echo "Failed to fetch preprod snapshot name."
+fi

scripts/buildkite/release/linux-e2e.sh

@@ -0,0 +1,47 @@
+#! /usr/bin/env -S nix shell 'nixpkgs#rsync' 'nixpkgs#gnutar' '.#cardano-wallet'  --command bash
+# shellcheck shell=bash
+
+set -euox pipefail
+
+mkdir -p "$(pwd)/logs"
+
+TESTS_LOGDIR="$(pwd)/logs"
+export TESTS_LOGDIR
+
+CARDANO_NODE_CONFIGS="$(pwd)/configs/cardano"
+export CARDANO_NODE_CONFIGS
+
+VERSION=$(buildkite-agent meta-data get "release-version")
+echo "VERSION=$VERSION"
+
+buildkite-agent artifact \
+    download "result/linux/cardano-wallet-$VERSION-linux64.tar.gz" "."
+
+tar xvzf "result/linux/cardano-wallet-$VERSION-linux64.tar.gz"
+
+TESTS_E2E_BINDIR="$(pwd)/cardano-wallet-$VERSION-linux64"
+export TESTS_E2E_BINDIR
+
+cd test/e2e
+
+TESTS_NODE_DB="$(pwd)/state/node_db"
+export TESTS_NODE_DB
+
+mkdir -p "$TESTS_NODE_DB"/preprod
+rsync -a --delete "$NODE_STATE_DIR/db-new/" "$TESTS_NODE_DB/preprod"
+
+tmpfile=$(mktemp /tmp/node-preprod.XXXXXX)
+
+CARDANO_NODE_SOCKET_PATH="$tmpfile"
+export CARDANO_NODE_SOCKET_PATH
+
+
+TESTS_E2E_STATEDIR=$(pwd)/state
+export TESTS_E2E_STATEDIR
+
+TESTS_E2E_TOKEN_METADATA=https://metadata.world.dev.cardano.org/
+export TESTS_E2E_TOKEN_METADATA
+
+rake "run_on[preprod,sync,true]"
+
+rm "$tmpfile"

scripts/buildkite/release/linux-package.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env -S nix shell --command bash
+# shellcheck shell=bash
+
+set -euox pipefail
+
+RELEASE_COMMIT=$(buildkite-agent meta-data get "release-commit")
+
+git checkout "$RELEASE_COMMIT"
+
+nix build -o result/linux .#ci.artifacts.linux64.release

scripts/buildkite/release/release-candidate.sh

@@ -0,0 +1,64 @@
+#! /usr/bin/env -S nix shell '.#cardano-node' 'nixpkgs#gnused' --command bash
+# shellcheck shell=bash
+
+set -euox pipefail
+
+# date from git tag
+# example v2023-04-04 -> 2023-04-04
+tag_date() {
+  echo "${1##v}"
+}
+# cabal version from git tag
+# example v2023-04-04 -> 2023.4.4
+tag_cabal_ver() {
+  tag_date "$1" | sed -e s/-0/-/g -e s/-/./g
+}
+
+git fetch --all
+
+BASE_COMMIT=$(git rev-parse HEAD)
+
+git checkout "$BASE_COMMIT"
+
+today=$(date +%Y-%m-%d)
+
+NEW_GIT_TAG=v$today
+
+NEW_CABAL_VERSION=$(tag_cabal_ver "$NEW_GIT_TAG")
+
+OLD_GIT_TAG=$( git tag -l "v2*-*-*" | sort | tail -n1)
+
+OLD_CABAL_VERSION=$(tag_cabal_ver "$OLD_GIT_TAG")
+
+CARDANO_NODE_TAG=$(cardano-node version | head -n1 | awk '{print $2}')
+
+RELEASE_CANDIDATE_BRANCH="release-candidate-new/$NEW_GIT_TAG"
+
+git config --global user.email "[email protected]"
+git config --global user.name "Github Action"
+
+git branch -D "$RELEASE_CANDIDATE_BRANCH" || true
+git checkout -b "$RELEASE_CANDIDATE_BRANCH" || true
+
+sed -i "s|version: .*|version: $NEW_GIT_TAG|g" specifications/api/swagger.yaml
+git commit -m "Update wallet version in swagger.yaml" specifications/api/swagger.yaml
+
+git ls-files  '*.cabal' | xargs sed -i "s|$OLD_CABAL_VERSION|$NEW_CABAL_VERSION|g"
+git commit -am "Update cardano-wallet version in *.cabal files"
+
+sed -i "s|NODE_TAG=.*|NODE_TAG=$CARDANO_NODE_TAG|g" README.md
+sed -i "s|WALLET_TAG=.*|WALLET_TAG=$NEW_CABAL_VERSION|g" README.md
+sed -i "s|WALLET_VERSION=.*|WALLET_VERSION=$NEW_GIT_TAG|g" README.md
+git commit -am "Update cardano-wallet version in README.md"
+
+RELEASE_COMMIT=$(git rev-parse HEAD)
+
+git remote set-url origin "[email protected]:cardano-foundation/cardano-wallet.git"
+git remote get-url origin
+
+git push -f origin "$RELEASE_CANDIDATE_BRANCH"
+
+buildkite-agent meta-data set "release-version" "$NEW_GIT_TAG"
+buildkite-agent meta-data set "release-commit" "$RELEASE_COMMIT"
+buildkite-agent meta-data set "release-candidate-branch" "$RELEASE_CANDIDATE_BRANCH"
+buildkite-agent meta-data set "release-cabal-version" "$NEW_CABAL_VERSION"