runtime-upgrade.yml

---
name: Runtime Upgrade

on:
  pull_request:
    branches: [main, testnet]

permissions: read-all

env:
  RUNNER_VM_NAME: "github-runner-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT"
  RESOURCE_GROUP: "github-runner-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT"
  AZ_LOCATION: "westus3"

jobs:
  setup:
    runs-on: ubuntu-22.04
    outputs:
      test_config: ${{ steps.testnet-env.outputs.test_config || steps.mainnet-env.outputs.test_config }}
      target_chain: ${{ steps.testnet-env.outputs.target_chain || steps.mainnet-env.outputs.target_chain }}
      boot_node: ${{ steps.testnet-env.outputs.boot_node || steps.mainnet-env.outputs.boot_node }}
      rpc_url: ${{ steps.testnet-env.outputs.rpc_url || steps.mainnet-env.outputs.rpc_url }}
      https_rpc_url: ${{ steps.testnet-env.https_rpc_url || steps.mainnet-env.outputs.https_rpc_url }}
      release_tag: ${{ steps.testnet-env.outputs.release_tag || steps.mainnet-env.outputs.release_tag }}
      artifact_name: ${{ steps.testnet-env.outputs.artifact_name || steps.mainnet-env.outputs.artifact_name }}
      last_block_hash: ${{ steps.last-block-info.outputs.last_block_hash }}
      last_block_number: ${{ steps.last-block-info.outputs.last_block_number }}

    steps:
      - uses: actions/checkout@v3

      - name: Set-Up
        run: |
          sudo apt-get update
          sudo apt install -y jq

      - name: Testnet ENV
        id: testnet-env
        if: github.base_ref == 'testnet'
        run: |
          # shellcheck disable=SC2129
          echo "target_chain=test" >> "$GITHUB_OUTPUT"
          echo "test_config=testnet.config.ts" >> "$GITHUB_OUTPUT"
          echo "boot_node=/dns4/testnet-bootnode.creditcoin.network/tcp/30333/p2p/12D3KooWG3eEuYxo37LvU1g6SSESu4i9TQ8FrZmJcjvdys7eA3cH" >> "$GITHUB_OUTPUT"
          echo "rpc_url=wss://rpc.testnet.creditcoin.network:443/ws" >> "$GITHUB_OUTPUT"
          echo "https_rpc_url=https://rpc.testnet.creditcoin.network/rpc" >> "$GITHUB_OUTPUT"
          RELEASE_TAG=$(./scripts/extract-release-tag.sh "testnet")
          echo "release_tag=$RELEASE_TAG" >> "$GITHUB_OUTPUT"
          echo "artifact_name=creditcoin-$RELEASE_TAG-$(uname -m)-unknown-linux-gnu.zip" >> "$GITHUB_OUTPUT"

      - name: Mainnet ENV
        id: mainnet-env
        if: github.base_ref == 'main'
        run: |
          # shellcheck disable=SC2129
          echo "target_chain=main" >> "$GITHUB_OUTPUT"
          echo "test_config=mainnet.config.ts" >> "$GITHUB_OUTPUT"
          echo "boot_node=/dns4/bootnode.creditcoin.network/tcp/30333/p2p/12D3KooWAEgDL126EUFxFfdQKiUhmx3BJPdszQHu9PsYsLCuavhb" >> "$GITHUB_OUTPUT"
          echo "rpc_url=wss://rpc.mainnet.creditcoin.network:443/ws" >> "$GITHUB_OUTPUT"
          echo "https_rpc_url=https://rpc.mainnet.creditcoin.network/rpc" >> "$GITHUB_OUTPUT"
          RELEASE_TAG=$(./scripts/extract-release-tag.sh "mainnet")
          echo "release_tag=$RELEASE_TAG" >> "$GITHUB_OUTPUT"
          echo "artifact_name=creditcoin-$RELEASE_TAG-$(uname -m)-unknown-linux-gnu.zip" >> "$GITHUB_OUTPUT"

      - name: Store last block info
        id: last-block-info
        run: |
          # store info about the last finalized block before the fork
          # WARNING: using getBlockHash() instead of getFinalizedHead() b/c PoW doesn't have finalization
          LAST_BLOCK=$(curl --silent -H "Content-Type: application/json" \
               -d '{"id": 1, "jsonrpc": "2.0", "method": "chain_getBlockHash", "params": [] }' \
               ${{ env.HTTPS_RPC_URL }} | jq -r .result)
          echo "$LAST_BLOCK" > last-block.hash
          echo "last_block_hash=$LAST_BLOCK" >> "$GITHUB_OUTPUT"

          while true; do
              curl --silent -H "Content-Type: application/json" \
                   -d "{\"id\": 1, \"jsonrpc\": \"2.0\", \"method\": \"chain_getBlock\", \"params\": [\"$LAST_BLOCK\"] }" \
                   ${{ env.HTTPS_RPC_URL }} | jq -r .result > last-block.json
              LAST_BLOCK_NUMBER=$(jq -r .block.header.number last-block.json)

              if [ "$LAST_BLOCK_NUMBER" != "null" ]; then
                break
              else
                echo "INFO: retry fetching block infor for $LAST_BLOCK"
                sleep 60
              fi
          done
          echo "last_block_number=$LAST_BLOCK_NUMBER" >> "$GITHUB_OUTPUT"
        env:
          HTTPS_RPC_URL: ${{ steps.testnet-env.outputs.https_rpc_url || steps.mainnet-env.outputs.https_rpc_url }}

      - name: Upload last block info
        uses: actions/upload-artifact@v3
        with:
          name: last-block-info
          path: "last-block*"
          if-no-files-found: error

  build-sut:
    runs-on: ubuntu-22.04

    steps:
      - uses: actions/checkout@v3

      - name: Set-Up
        run: |
          sudo apt-get update
          sudo apt install -y cmake pkg-config libssl-dev git build-essential clang libclang-dev curl protobuf-compiler
          sudo apt install -y unzip jq

      - name: Configure rustc version
        run: |
          source ci/env
          echo "RUSTC_VERSION=$RUSTC_VERSION" >> "$GITHUB_ENV"

      - name: Install Rust toolchain
        uses: actions-rs/toolchain@v1
        with:
          toolchain: ${{ env.RUSTC_VERSION }}
          target: wasm32-unknown-unknown
          profile: minimal
          override: true
      - uses: Swatinem/rust-cache@v2

      - name: Build SUT
        uses: actions-rs/cargo@v1
        with:
          command: build
          args: --release --features "fast-runtime try-runtime"

      - name: Upload creditcoin-node binary
        uses: actions/upload-artifact@v3
        with:
          name: creditcoin-node
          path: target/release/creditcoin-node

      - name: Upload WASM runtime
        uses: actions/upload-artifact@v3
        with:
          name: creditcoin_node_runtime.compact.compressed.wasm
          path: target/release/wbuild/creditcoin-node-runtime/creditcoin_node_runtime.compact.compressed.wasm

  fork-creditcoin:
    needs:
      - setup
      - live-sync-creditcoin
      - deploy-github-runner
    runs-on:
      [self-hosted, "${{ needs.deploy-github-runner.outputs.runner_vm_name }}"]

    steps:
      - uses: actions/checkout@v3

      - name: Download creditcoin-node for release ${{ needs.setup.outputs.release_tag }}
        uses: i3h/download-release-asset@v1
        with:
          owner: gluwa
          repo: creditcoin
          tag: ${{ needs.setup.outputs.release_tag }}
          file: ${{ needs.setup.outputs.artifact_name }}

      - name: Download creditcoin-fork
        uses: i3h/download-release-asset@v1
        with:
          owner: gluwa
          repo: creditcoin-fork
          tag: latest
          file: creditcoin-fork

      - name: Start local creditcoin-node for ${{ needs.setup.outputs.target_chain }}
        run: |
          # see https://opensource.com/article/18/5/how-find-ip-address-linux
          IP_ADDRESS=$(curl https://ifconfig.me)
          echo "INFO: IP_ADDRESS=$IP_ADDRESS"

          sudo apt-get update
          sudo apt install -y unzip

          unzip creditcoin-*-unknown-linux-gnu.zip
          chmod a+x ./creditcoin-node

          ./creditcoin-node --version
          ./creditcoin-node \
            --name "test-node-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
            --chain ${{ needs.setup.outputs.target_chain }} \
            --validator --mining-key 5DkPYq8hFiCeGxFBkz6DAwnTrvKevAJfTYrzFtr9hpDsEAU1 \
            --bootnodes "${{ needs.setup.outputs.boot_node }}" \
            --prometheus-external \
            --telemetry-url "wss://telemetry.creditcoin.network/submit/ 0" \
            --public-addr "/dns4/$IP_ADDRESS/tcp/50555" \
            --port 50555 >creditcoin-node-used-for-fork.log 2>&1 &

      - name: Wait for creditcoin-node to initialize
        run: |
          ./integration-tests/wait-for-creditcoin.sh

      - name: Create fork
        run: |
          chmod a+x ./creditcoin-fork

          ./creditcoin-fork --bin ./creditcoin-node --orig ${{ needs.setup.outputs.target_chain }} \
                            --base dev --name Development \
                            -o creditcoin-fork.json --rpc ws://127.0.0.1:9944

      - name: TERM creditcoin-node
        continue-on-error: true
        run: |
          killall -TERM creditcoin-node
          sleep 120

      - name: KILL creditcoin-node
        continue-on-error: true
        run: |
          killall -KILL creditcoin-node
          sleep 120

      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: logs
          path: "*.log"

      - name: Upload creditcoin-fork.json
        uses: actions/upload-artifact@v3
        with:
          name: creditcoin-fork.json
          path: creditcoin-fork.json

  live-sync-creditcoin:
    needs:
      - setup
      - deploy-github-runner
    runs-on:
      [self-hosted, "${{ needs.deploy-github-runner.outputs.runner_vm_name }}"]

    steps:
      - uses: actions/checkout@v3

      - name: Download creditcoin-node for release ${{ needs.setup.outputs.release_tag }}
        uses: i3h/download-release-asset@v1
        with:
          owner: gluwa
          repo: creditcoin
          tag: ${{ needs.setup.outputs.release_tag }}
          file: ${{ needs.setup.outputs.artifact_name }}

      - name: Sync with ${{ needs.setup.outputs.target_chain }} past block number ${{ needs.setup.outputs.last_block_number }}
        run: |
          # see https://opensource.com/article/18/5/how-find-ip-address-linux
          IP_ADDRESS=$(curl https://ifconfig.me)
          echo "INFO: IP_ADDRESS=$IP_ADDRESS"

          sudo apt-get update
          sudo apt install -y unzip

          unzip creditcoin-*-unknown-linux-gnu.zip
          chmod a+x ./creditcoin-node

          ./creditcoin-node --version
          ./creditcoin-node \
            --name "test-node-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
            --chain ${{ needs.setup.outputs.target_chain }} \
            --validator --mining-key 5DkPYq8hFiCeGxFBkz6DAwnTrvKevAJfTYrzFtr9hpDsEAU1 \
            --bootnodes "${{ needs.setup.outputs.boot_node }}" \
            --prometheus-external --pruning archive \
            --telemetry-url "wss://telemetry.creditcoin.network/submit/ 0" \
            --public-addr "/dns4/$IP_ADDRESS/tcp/50555" \
            --port 50555 >creditcoin-node-initial-live-sync.log 2>&1 &

      - name: Wait for creditcoin-node to sync past block number ${{ needs.setup.outputs.last_block_number }}
        run: |
          ./scripts/wait-for-sync.sh ${{ needs.setup.outputs.last_block_number }}

      - name: TERM creditcoin-node
        continue-on-error: true
        run: |
          killall -TERM creditcoin-node
          sleep 120

      - name: KILL creditcoin-node
        continue-on-error: true
        run: |
          killall -KILL creditcoin-node
          sleep 120

      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: logs
          path: "*.log"

  test-against-fork:
    needs:
      - setup
      - build-sut
      - fork-creditcoin
      - deploy-github-runner
    runs-on:
      [self-hosted, "${{ needs.deploy-github-runner.outputs.runner_vm_name }}"]
    steps:
      - uses: actions/checkout@v3

      - name: Download creditcoin-node from current PR
        uses: actions/download-artifact@v3
        with:
          name: creditcoin-node
          path: target/release

      - name: Download creditcoin-fork.json
        uses: actions/download-artifact@v3
        with:
          name: creditcoin-fork.json

      - name: Start a local creditcoin-node from the fork
        run: |
          chmod a+x ./target/release/creditcoin-node

          ./target/release/creditcoin-node --version
          ./target/release/creditcoin-node --chain ./creditcoin-fork.json --validator --alice --pruning archive \
                            --mining-key 5DkPYq8hFiCeGxFBkz6DAwnTrvKevAJfTYrzFtr9hpDsEAU1 \
                            --monitor-nonce auto >creditcoin-node-with-forked-chain.log 2>&1 &

      - name: Wait for blockchain to start
        run: |
          ./integration-tests/wait-for-creditcoin.sh

      - name: Install Docker engine
        run: |
          ./scripts/install-docker-engine-from-upstream.sh

      - name: Start local Ethereum node
        run: |
          sudo docker run --name hardhat-dev --rm -p 8545:8545 -d gluwa/hardhat-dev

      - name: Install Node.js
        uses: actions/setup-node@v3
        with:
          node-version: 16
      - run: npm install -g yarn

      - name: Prepare for integration tests
        run: |
          pushd integration-tests/

          # wait for creditcoin-node to initialize
          ./yarn-install-and-wait-for-creditcoin.sh
          git checkout yarn.lock

          popd

      - name: Configure rustc version
        run: |
          source ci/env
          echo "RUSTC_VERSION=$RUSTC_VERSION" >> "$GITHUB_ENV"
          sudo apt install -y gcc

      - name: Install Rust toolchain
        uses: actions-rs/toolchain@v1
        with:
          toolchain: ${{ env.RUSTC_VERSION }}
          profile: minimal
          override: true

      - name: Install Subwasm
        uses: actions-rs/cargo@v1
        with:
          command: install
          args: --locked --git https://github.com/chevdor/subwasm --tag v0.17.1

      - name: Download WASM runtime from current PR
        id: download-wasm
        uses: actions/download-artifact@v3
        with:
          name: creditcoin_node_runtime.compact.compressed.wasm

      - name: Upgrade WASM
        run: |
          yarn --cwd ./scripts/js upgrade 'creditcoin-js'
          yarn --cwd ./scripts/js runtimeUpgrade ws://127.0.0.1:9944 ../../creditcoin_node_runtime.compact.compressed.wasm //Alice 0

      # TODO: wait & confirm wasm upgrade has finished, incl. migrations
      - name: Execute integration tests
        run: |
          yarn --cwd ./integration-tests/ test --config creditcoin-fork.config.ts

      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: logs
          path: "*.log"

      - name: Kill hardhat-dev
        if: always()
        run: |
          sudo docker kill hardhat-dev

      - name: Kill creditcoin-node
        run: |
          # if all went well kill the node. Otherwise GitHub Actions would exit on the
          # previous step killing everything and we don't have to worry about
          # dangling processes
          killall -9 creditcoin-node

  test-against-disconnected-live-node:
    # execute only against Testnet b/c we don't have sudo key for Mainnet
    if: github.base_ref == 'testnet'
    needs:
      - build-sut
      - setup
      - live-sync-creditcoin
      - deploy-github-runner
    runs-on:
      [self-hosted, "${{ needs.deploy-github-runner.outputs.runner_vm_name }}"]
    steps:
      - uses: actions/checkout@v3

      - name: Download creditcoin-node from current PR
        uses: actions/download-artifact@v3
        with:
          name: creditcoin-node
          path: target/release

      - name: Start creditcoin-node from current PR
        run: |
          # see https://opensource.com/article/18/5/how-find-ip-address-linux
          IP_ADDRESS=$(curl https://ifconfig.me)
          echo "INFO: IP_ADDRESS=$IP_ADDRESS"

          chmod a+x ./target/release/creditcoin-node
          ./target/release/creditcoin-node --version

          ./target/release/creditcoin-node \
            --name "test-node-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
            --chain ${{ needs.setup.outputs.target_chain }} \
            --pruning archive \
            --validator --mining-key 5DkPYq8hFiCeGxFBkz6DAwnTrvKevAJfTYrzFtr9hpDsEAU1 \
            --bootnodes "${{ needs.setup.outputs.boot_node }}" \
            --prometheus-external \
            --telemetry-url "wss://telemetry.creditcoin.network/submit/ 0" \
            --public-addr "/dns4/$IP_ADDRESS/tcp/50555" \
            --port 50555 >creditcoin-node-disconnected-paritydb-sync.log 2>&1 &

      - name: Wait for blockchain to start
        run: |
          ./integration-tests/wait-for-creditcoin.sh

      # wait again because of RocksDB -> ParityDB switch !
      - name: Wait for creditcoin-node to sync past block number ${{ needs.setup.outputs.last_block_number }}
        run: |
          ./scripts/wait-for-sync.sh ${{ needs.setup.outputs.last_block_number }}

      - name: TERM creditcoin-node
        continue-on-error: true
        run: |
          killall -TERM creditcoin-node
          sleep 30

      - name: KILL creditcoin-node
        continue-on-error: true
        run: |
          killall -KILL creditcoin-node
          sleep 30

      - name: Start a disconnected creditcoin-node
        run: |
          chmod a+x ./target/release/creditcoin-node
          ./target/release/creditcoin-node --version

          # WARNING: using different port b/c the network remembers there was a node
          # at this address previously. We don't want to be connected to the chain !!!
          ./target/release/creditcoin-node \
            --port 44444 \
            --name "test-node-disconnected-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
            --chain ${{ needs.setup.outputs.target_chain }} \
            --validator --alice --pruning archive \
            --mining-key 5DkPYq8hFiCeGxFBkz6DAwnTrvKevAJfTYrzFtr9hpDsEAU1 \
            --prometheus-external \
            --telemetry-url "wss://telemetry.creditcoin.network/submit/ 0" \
            --monitor-nonce auto >creditcoin-node-disconnected-live-node.log 2>&1 &

      - name: Wait for blockchain to start
        run: |
          ./integration-tests/wait-for-creditcoin.sh

      - name: Install Docker engine
        run: |
          ./scripts/install-docker-engine-from-upstream.sh

      - name: Start local Ethereum node
        run: |
          sudo docker run --name hardhat-dev --rm -p 8545:8545 -d gluwa/hardhat-dev

      - name: Install Node.js
        uses: actions/setup-node@v3
        with:
          node-version: 16
      - run: npm install -g yarn

      - name: Prepare for integration tests
        run: |
          pushd integration-tests/

          # wait for creditcoin-node to initialize
          ./yarn-install-and-wait-for-creditcoin.sh
          git checkout yarn.lock

          popd

      - name: Configure rustc version
        run: |
          source ci/env
          echo "RUSTC_VERSION=$RUSTC_VERSION" >> "$GITHUB_ENV"
          sudo apt install -y gcc

      - name: Install Rust toolchain
        uses: actions-rs/toolchain@v1
        with:
          toolchain: ${{ env.RUSTC_VERSION }}
          profile: minimal
          override: true

      - name: Install Subwasm
        uses: actions-rs/cargo@v1
        with:
          command: install
          args: --locked --git https://github.com/chevdor/subwasm --tag v0.17.1

      - name: Download WASM runtime
        id: download-wasm
        uses: actions/download-artifact@v3
        with:
          name: creditcoin_node_runtime.compact.compressed.wasm

      - name: Upgrade WASM
        if: env.LENDER_SEED
        run: |
          yarn --cwd ./scripts/js upgrade 'creditcoin-js'
          yarn --cwd ./scripts/js runtimeUpgrade ws://127.0.0.1:9944 ../../creditcoin_node_runtime.compact.compressed.wasm "${{ env.LENDER_SEED }}" 0
          sleep 10
        env:
          LENDER_SEED: ${{ secrets.TESTNET_LENDER_SEED }}

      # TODO: wait & confirm wasm upgrade has finished, incl. migrations
      - name: Execute integration tests
        if: env.ETHEREUM_NODE_URL
        run: |
          yarn --cwd ./integration-tests/ test --config testnet.config.ts
        env:
          ETHEREUM_NODE_URL: ${{ secrets.TESTNET_ETHEREUM_NODE_URL }}
          LENDER_PRIVATE_KEY: ${{ secrets.TESTNET_LENDER_PRIVATE_KEY }}
          LENDER_SEED: "${{ secrets.TESTNET_LENDER_SEED }}"
          BORROWER_PRIVATE_KEY: ${{ secrets.TESTNET_BORROWER_PRIVATE_KEY }}
          BORROWER_SEED: "${{ secrets.TESTNET_BORROWER_SEED }}"

      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: logs
          path: "*.log"

      - name: Kill hardhat-dev
        if: always()
        run: |
          sudo docker kill hardhat-dev

      - name: Kill creditcoin-node
        run: |
          # if all went well kill the node. Otherwise GitHub Actions would exit on the
          # previous step killing everything and we don't have to worry about
          # dangling processes
          killall -9 creditcoin-node

  deploy-github-runner:
    runs-on: ubuntu-22.04
    outputs:
      runner_vm_name: ${{ steps.get-env.outputs.runner_vm_name }}
      resource_group: ${{ steps.get-env.outputs.resource_group }}

    steps:
      - uses: actions/checkout@v3

      - name: Install azure-cli
        run: |
          sudo apt remove azure-cli -y && sudo apt autoremove -y
          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
          sudo apt install -y jq
          az version

      - name: Authorize hosted-runner
        run: |
          mkdir -p ~/.ssh/
          ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa
          cat ~/.ssh/id_rsa.pub >> .github/authorized_keys

      - name: Evaluate env vars
        id: get-env
        run: |
          # WARNING: using env.RUNNER_VM_NAME directly in job outputs above
          # doesn't evaluate the $GITHUB_RUN_ID reference
          echo "runner_vm_name=${{ env.RUNNER_VM_NAME }}" >> "$GITHUB_OUTPUT"
          echo "resource_group=${{ env.RESOURCE_GROUP }}" >> "$GITHUB_OUTPUT"

      - name: Provision VM
        if: env.LC_GITHUB_REPO_ADMIN_TOKEN
        run: |
          echo "INFO: From ENVs: RUNNER_VM_NAME=${{ env.RUNNER_VM_NAME }}"
          echo "INFO: From Step: RUNNER_VM_NAME=${{ steps.get-env.outputs.runner_vm_name }}"

          az login --service-principal --username "${{ secrets.AZURE_APP_ID }}" --password "${{ secrets.AZURE_APP_PASSWORD }}" --tenant "${{ secrets.AZURE_TENANT_ID }}"
          az account set --subscription "Playground Subscription"
          ## az account set -s "${{ secrets.AZURE_SUBSCRIPTION_ID }}"


          # create resource group
          echo "INFO: ${{ steps.get-env.outputs.resource_group }}"
          az group create -n "${{ steps.get-env.outputs.resource_group }}" --location "${{ env.AZ_LOCATION }}"

          # RG Creditcoin-Test is in WestUS and the CPU quota is already full
          # that's why specify a different region here
          az deployment group create -g "${{ steps.get-env.outputs.resource_group }}" -f .github/runner.bicep \
              --parameters location="${{ env.AZ_LOCATION }}" \
              --parameters vmName="${{ steps.get-env.outputs.runner_vm_name }}" \
              --parameters adminPasswordOrKey="$(cat .github/authorized_keys)" > output.json

          # provision the GitHub Runner binary on the VM
          # passing additional ENV values
          SSH_USER_AT_HOSTNAME=$(jq -r '.properties.outputs.sshUserAtHostname.value' < output.json)
          echo "INFO: $SSH_USER_AT_HOSTNAME"

          export LC_RUNNER_VM_NAME="${{ env.RUNNER_VM_NAME }}"
          until ssh -i ~/.ssh/id_rsa \
              -o SendEnv=LC_GITHUB_REPO_ADMIN_TOKEN,LC_RUNNER_VM_NAME \
              -o StrictHostKeyChecking=no "$SSH_USER_AT_HOSTNAME" < ./scripts/provision-github-runner.sh; do
            echo "DEBUG: retrying ssh connection ..."
            sleep 30
          done
        env:
          LC_GITHUB_REPO_ADMIN_TOKEN: ${{ secrets.GH_REPO_ADMIN_TOKEN }}
          LC_RUNNER_EPHEMERAL: false

  test-migrations-via-try-runtime:
    # see https://gist.github.com/jonico/a94d03cac7a858e0613926d9f1bc7f2b
    runs-on:
      [self-hosted, "${{ needs.deploy-github-runner.outputs.runner_vm_name }}"]
    needs:
      - build-sut
      - setup
      - deploy-github-runner
    steps:
      - uses: actions/checkout@v3

      - name: Download creditcoin-node binary from current PR
        uses: actions/download-artifact@v3
        with:
          name: creditcoin-node
          path: target/release

      - name: Download WASM runtime
        uses: actions/download-artifact@v3
        with:
          name: creditcoin_node_runtime.compact.compressed.wasm
          path: target/release

      - name: Restore executable permissions
        run: |
          chmod a+x ./target/release/creditcoin-node

      - name: Check if runner machine meets chain's requirements
        continue-on-error: true
        run: |
          ./scripts/check-hardware.sh

      - name: Start local creditcoin-node for ${{ needs.setup.outputs.target_chain }}
        run: |
          # see https://opensource.com/article/18/5/how-find-ip-address-linux
          IP_ADDRESS=$(curl https://ifconfig.me)
          echo "INFO: IP_ADDRESS=$IP_ADDRESS"

          ./target/release/creditcoin-node --version
          ./target/release/creditcoin-node \
            --name "test-node-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
            --chain ${{ needs.setup.outputs.target_chain }} \
            --validator --mining-key 5DkPYq8hFiCeGxFBkz6DAwnTrvKevAJfTYrzFtr9hpDsEAU1 \
            --bootnodes "${{ needs.setup.outputs.boot_node }}" \
            --rpc-max-request-size 100000 \
            --rpc-max-response-size 100000 \
            --prometheus-external \
            --telemetry-url "wss://telemetry.creditcoin.network/submit/ 0" \
            --public-addr "/dns4/$IP_ADDRESS/tcp/30333" >creditcoin-node-used-for-try-runtime.log 2>&1 &

      - name: Wait for creditcoin-node to initialize
        run: |
          ./integration-tests/wait-for-creditcoin.sh
          ./scripts/wait-for-sync.sh ${{ needs.setup.outputs.last_block_number }}

      - name: Try-runtime migrations
        run: |
          try-runtime \
              --runtime ./target/release/creditcoin_node_runtime.compact.compressed.wasm \
              --dev on-runtime-upgrade live --uri ws://127.0.0.1:9944 \
              --at ${{ needs.setup.outputs.last_block_hash }}

      - name: TERM creditcoin-node
        continue-on-error: true
        run: |
          killall -TERM creditcoin-node
          sleep 60

      - name: KILL creditcoin-node
        if: always()
        continue-on-error: true
        run: |
          killall -KILL creditcoin-node
          sleep 600

      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: logs
          path: "*.log"

  remove-github-runner:
    runs-on: ubuntu-22.04
    needs:
      - deploy-github-runner
      - test-migrations-via-try-runtime
      - test-against-fork
      - test-against-disconnected-live-node
    if: ${{ always() && needs.deploy-github-runner.result != 'skipped' }}

    steps:
      - uses: actions/checkout@v3

      - name: Evaluate env vars
        id: get-env
        run: |
          # WARNING: using env.RUNNER_VM_NAME directly in job outputs above
          # doesn't evaluate the $GITHUB_RUN_ID reference
          echo "resource_group=${{ env.RESOURCE_GROUP }}" >> "$GITHUB_OUTPUT"

      - name: Install azure-cli
        run: |
          sudo apt remove azure-cli -y && sudo apt autoremove -y
          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
          az version

      - name: Remove VM
        run: |
          echo "INFO: RUNNER_VM_NAME=${{ env.RUNNER_VM_NAME }}"

          az login --service-principal --username "${{ secrets.AZURE_APP_ID }}" --password "${{ secrets.AZURE_APP_PASSWORD }}" --tenant "${{ secrets.AZURE_TENANT_ID }}"
          az account set --subscription "Playground Subscription"
          az group delete --yes -n "${{ steps.get-env.outputs.resource_group }}"

      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: "Azure resources"
          path: azure_resource_list.json