Feat: Automated Program Upgrade Testing

.github/actions/build_contract_artifacts/action.yml

@@ -46,45 +46,6 @@ runs:
         docker stop build-container
         docker rm build-container
 
-    # should be used again after moving from projectserum/build to backpackapp/build
-    - name: Install latest Git version (>= 2.18.0) for actions/checkout
-      if: ${{ inputs.image  == '' && inputs.image-version  == '' }}
-      shell: bash
-      run: |
-        apt-get update
-        apt-get install software-properties-common -y
-        add-apt-repository ppa:git-core/ppa
-        apt update
-        apt install git -y
-        git config --global --add safe.directory "$GITHUB_WORKSPACE"
-    - name: Setup go
-      if: ${{ inputs.image  == '' && inputs.image-version  == '' }}
-      uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
-      with:
-        go-version-file: "go.mod"
-        check-latest: true
-    - name: Generate build artifacts for go bindings check
-      if: ${{ inputs.image  == '' && inputs.image-version  == '' }}
-      shell: bash
-      run: anchor build
-      working-directory: contracts
-    - name: Check generated go bindings are up to date
-      if: ${{ inputs.image  == '' && inputs.image-version  == '' }}
-      shell: bash
-      run: |
-        go install github.com/gagliardetto/[email protected]
-        ./scripts/anchor-go-gen.sh
-        git diff --stat --exit-code
-    - name: Generate program_ids
-      if: ${{ inputs.image  == '' && inputs.image-version  == '' }}
-      shell: bash
-      run: ./scripts/programs-keys-gen.sh
-    - name: Generate build artifacts with custom program_ids
-      if: ${{ inputs.image  == '' && inputs.image-version  == '' }}
-      shell: bash
-      run: anchor build
-      working-directory: contracts
-
     #save the contracts artifacts
     - name: Upload Artifacts
       uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3

.github/workflows/e2e_custom_cl.yml

@@ -47,6 +47,21 @@ jobs:
         id: psversion
         uses: ./.github/actions/projectserum_version
 
+  contract-changes:
+    name: Check for contract changes
+    runs-on: ubuntu-latest
+    outputs:
+      changed: ${{ steps.changes.outputs.contracts }}
+    steps:
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: changes
+        with:
+          base: develop
+          list-files: csv
+          filters: |
+            contracts:
+              - 'contracts/**'
+
   solana-test-image-exists:
     environment: integration
     permissions:
@@ -85,6 +100,91 @@ jobs:
           image: backpackapp/build
           image-version: ${{ needs.get_projectserum_version.outputs.projectserum_version }}
 
+  e2e_custom_build_previous_artifacts:
+    name: E2E Build Previous Release Artifacts
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest-32cores-128GB
+    needs: [contract-changes,get_projectserum_version]
+    if: needs.contract-changes.outputs.changed == 'true'
+    steps:
+      - name: Checkout previous release
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        with:
+          ref: v1.0.2 # set to v1.1.0 after v1.1.0 is deployed to mainnet
+      - name: build contracts
+        id: previous
+        run: |
+          # for use with 1.0.2 release
+          docker run -d -v "$(pwd)":/repo --name build-container projectserum/build:v0.25.0 tail -f /dev/null
+          # for use with 1.1.0 release and onward
+          # docker run -d -v "$(pwd)":/repo --name build-container backpackapp/build:${{ needs.get_projectserum_version.outputs.projectserum_version }} tail -f /dev/null
+
+          # build with keys
+          docker exec build-container bash -c "\
+            export RUSTUP_HOME=\"/root/.rustup\" &&\
+            cd /repo &&\
+            ./scripts/programs-keys-gen.sh &&\
+            cd ./contracts &&\
+            anchor build &&\
+            chown -R $(id -u):$(id -g) /repo"
+          # clean up the container
+          docker stop build-container
+          docker rm build-container
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifacts-previous
+          path: ${{ env.CONTRACT_ARTIFACTS_PATH }}
+
+  e2e_program_upgrade_generate_artifacts:
+    name: Combine artifacts for upgrade test
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    needs: [contract-changes,e2e_custom_build_artifacts,e2e_custom_build_previous_artifacts]
+    if: needs.contract-changes.outputs.changed == 'true'
+    outputs:
+      changed: ${{ steps.check.outputs.continue }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: artifacts
+          path: ./temp/artifacts
+      - name: hash current artifacts
+        id: current
+        run: |
+          cd ./temp/artifacts # need to be in directory for hashes to  compare without path differences
+          # shellcheck disable=SC2035
+          echo "hash=$(sha1sum * | sha1sum)" >> "$GITHUB_OUTPUT"
+      - uses: actions/download-artifact@v4
+        with:
+          name: artifacts-previous
+          path: ./temp/artifacts/previous
+      - name: hash previous artifacts
+        id: previous
+        run: |
+          cd ./temp/artifacts/previous # need to be in directory for hashes to  compare without path differences
+          # shellcheck disable=SC2035
+          echo "hash=$(sha1sum * | sha1sum)" >> "$GITHUB_OUTPUT"
+      - name: compare hashes
+        id: check
+        run: |
+          echo "previous: ${{ steps.previous.outputs.hash }}"
+          echo "current:  ${{ steps.current.outputs.hash }}"
+
+          if [ "${{ steps.previous.outputs.hash }}" != "${{ steps.current.outputs.hash }}" ]; then
+            echo "continue=true" >> "$GITHUB_OUTPUT"
+          fi
+      - uses: actions/upload-artifact@v4
+        if: steps.check.outputs.continue
+        with:
+          name: artifacts
+          path: ./temp/artifacts/ # combine artifacts
+          overwrite: true
+
   e2e_custom_build_custom_chainlink_image:
     name: E2E Custom Build Custom CL Image
     runs-on: ubuntu-latest-16cores-64GB
@@ -215,9 +315,88 @@ jobs:
           # shellcheck disable=SC2086
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
       - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
+        with:
+          test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaOCRV2Smoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci=true -singlepackage=true -hidepassingtests=false -hidepassinglogs=false
+          test_download_vendor_packages_command: cd ./integration-tests && go mod download
+          download_contract_artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }} 
+          go_mod_path: ./integration-tests/go.mod
+          cl_repo: ${{ env.CL_ECR }}
+          cl_image_tag: solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          artifacts_name: smoke-test-logs
+          artifacts_location: ./integration-tests/smoke/logs/
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          cache_key_id: solana-e2e-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "false"
+
+  e2e_program_upgrade_tests:
+    name: E2E Program Upgrade Tests
+    environment: integration
+    permissions:
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest-16cores-64GB
+    needs: [contract-changes,e2e_program_upgrade_generate_artifacts,e2e_custom_build_custom_chainlink_image, check_test_compilation]
+    if: needs.contract-changes.outputs.changed == 'true' && needs.e2e_program_upgrade_generate_artifacts.outputs.changed == 'true'
+    env:
+      TEST_SUITE: smoke
+      TEST_ARGS: -test.timeout 30m
+      TEST_LOG_LEVEL: debug
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
+        with:
+          id: solana-e2e-program-upgrade
+          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          this-job-name: E2E Program Upgrade Tests
+          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
+        continue-on-error: true
+      - name: Get core ref from PR body
+        if: github.event_name == 'pull_request'
+        run: |
+          comment="$(gh pr view https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }} --json body -q '.body')"
+          # shellcheck disable=SC2086
+          core_ref="$(echo $comment | grep -oP 'core ref: \K\S+' || true)"
+          if [ -n "$core_ref" ]; then
+            echo "CUSTOM_CORE_REF=${core_ref}" >> "${GITHUB_ENV}"
+          fi
+      - name: Checkout the repo
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+      - name: Install Solana CLI # required for ensuring the local test validator is configured correctly
+        run: ./scripts/install-solana-ci.sh
+      - name: Install gauntlet
+        uses: ./.github/actions/build-gauntlet
+      - name: Generate config overrides
+        run: | # https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/README.md
+          cat << EOF > config.toml
+          [ChainlinkImage]
+          image="${{ env.CL_ECR }}"
+          version="solana.${{ env.CUSTOM_CORE_REF || github.event.inputs.cl_branch_ref || github.sha }}"
+          [Common]
+          user="${{ github.actor }}"
+          internal_docker_repo = "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com"
+          EOF
+
+          # shellcheck disable=SC2002
+          BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)
+          # shellcheck disable=SC2086
+          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
+          # shellcheck disable=SC2086
+          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
+      - name: Run Upgrade Test
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
         with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaOCRV2Smoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
+          test_command_to_run: cd ./integration-tests && go test -timeout 24h -count=1 -run TestSolanaOCRV2UpgradeSmoke -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci=true -singlepackage=true -hidepassingtests=false -hidepassinglogs=false
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           download_contract_artifacts_path: ${{ env.CONTRACT_ARTIFACTS_PATH }} 
           go_mod_path: ./integration-tests/go.mod

.github/workflows/upstream-tracker.yml

@@ -31,18 +31,18 @@ jobs:
           WIKI="${WIKI//$'\n'/\\n}" # preserve new line
           printf -v WIKI "%q " "$WIKI" # escape chars that may cause issues
 
-          echo "open<<EOF" >> $GITHUB_OUTPUT
-          echo "$OPEN" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
+          echo "open<<EOF" >> "$GITHUB_OUTPUT"
+          echo "$OPEN" >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
 
-          echo "closed<<EOF" >> $GITHUB_OUTPUT
-          echo "$CLOSED" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
+          echo "closed<<EOF" >> "$GITHUB_OUTPUT"
+          echo "$CLOSED" >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
 
-          echo "wiki<<EOF" >> $GITHUB_OUTPUT
-          echo "$WIKI" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-          echo "wikirange=$WIKIRANGE" >> $GITHUB_OUTPUT
+          echo "wiki<<EOF" >> "$GITHUB_OUTPUT"
+          echo "$WIKI" >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+          echo "wikirange=$WIKIRANGE" >> "$GITHUB_OUTPUT"
       - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         if: steps.updates.outputs.open || steps.updates.outputs.closed || steps.updates.outputs.wiki
       - name: Open Issue

contracts/README.md

@@ -56,5 +56,5 @@ Install `https://github.com/gagliardetto/anchor-go`
 Current version: [v0.2.3](https://github.com/gagliardetto/anchor-go/tree/v0.2.3)
 
 ```bash
-./scripts/anchor-go-gen.sh
+./scripts/anchor-go-gen.sh 
 ```

integration-tests/README.md

@@ -1,3 +1,64 @@
 # Contents
 
-- [How To Run E2E Tests](../docs/RunningE2eTests.md)
+- [How To Run E2E Tests](../docs/RunningE2eTests.md)
+
+## Automated Program Upgrade Testing
+
+Main Benefit: automatically test that changes made to programs are upgradeable from the deployed version
+
+Previously, program upgrade testing is a manual process where a node cluster is spun up against deployed programs,
+and then steps are manually executed. The following workflow describes automated program testing for any changes
+that occur in the `chainlink-solana/contracts` folder.
+
+This workflow adds additional steps to the base [e2e_custom_cl workflow](https://github.com/smartcontractkit/chainlink-solana/blob/develop/.github/workflows/e2e_custom_cl.yml).
+
+```mermaid
+flowchart TD
+
+    classDef base stroke:#00f
+    testimage[Check Test Image Existence]
+    buildver[Build Container Version]
+    smoketest[Execute Smoke Tests]
+    currentartifacts[Build Current Artifacts]
+    buildtestimage[Build & Push Test Image]
+    compcheck[Test Compilation Check]
+    climage[Build Custom CL Image]
+    class compcheck,buildver,climage,testimage,currentartifacts,smoketest,buildtestimage base
+
+    compcheck --> smoketest
+    buildver --> currentartifacts
+    currentartifacts --> smoketest
+    climage --> smoketest
+    currentartifacts .->|is labeled?| buildtestimage
+
+    classDef upgrade stroke:#0f0
+    changecheck[Check Contract Changes]
+    prevartifacts[Build Previous Artifacts]
+    combineartifacts[Combine Artifacts For Test]
+    upgradetest[Execute Upgrade Test]
+    class changecheck,prevartifacts,combineartifacts,upgradetest upgrade
+
+    changecheck .->|changed?| prevartifacts
+    changecheck .->|changed?| combineartifacts
+    changecheck .->|changed?| upgradetest
+    buildver --> prevartifacts
+    prevartifacts --> combineartifacts
+    currentartifacts --> combineartifacts
+    combineartifacts .->|artifacts different?| upgradetest
+    climage --> upgradetest
+    compcheck --> upgradetest
+```
+
+- 🟦 **BLUE**: represents the base smoke test jobs
+- 🟩 **GREEN**: represents the program upgrade test jobs
+- the program upgrade test is only run when two conditions are true:
+  - files within `./contract` are changed: this prevents running an integration test unless needed
+  - generated program artifacts differ from the *pinned* version in the workflow: this prevents running an integration test unless actual program changes were made, not just trivial changes
+    - note: the pinned version needs to be manually updated to track with the deployed production version
+
+### Workflow Runs for Upgrade Testing
+
+✅ **v1.0.2 -> v1.1.0**
+
+- [commit](https://github.com/smartcontractkit/chainlink-solana/pull/811/commits/0699d595e373e4a3d781b3249f38dda0087421dc)
+- [e2e tests job run with upgrade test](https://github.com/smartcontractkit/chainlink-solana/actions/runs/10292293982/job/28486398103?pr=811)