From f78571be9445df5d990680d552e002c01ec15ddd Mon Sep 17 00:00:00 2001
From: Jon C <me@jonc.dev>
Date: Wed, 27 Nov 2024 11:54:48 +0100
Subject: [PATCH] CI: Fix downstream anchor projects (#3805)

#### Problem

The downstream anchor jobs were disabled because of breaking changes
between the monorepo and SPL.

#### Summary of changes

Now that the v2.1 crates are released, and SPL is using them, re-enable
the job!

The biggest change is the crate patching, which will hopefully be easier
to maintain with this change. The patching functions were essentially
copied to mirror the patching scripts in SPL.
---
 .../workflows/downstream-project-anchor.yml   |   2 +-
 scripts/build-downstream-anchor-projects.sh   |   1 -
 scripts/patch-crates.sh                       | 292 ++++++++++++++++--
 3 files changed, 263 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/downstream-project-anchor.yml b/.github/workflows/downstream-project-anchor.yml
index 9e6b1591b6c23e..da5e3e160052db 100644
--- a/.github/workflows/downstream-project-anchor.yml
+++ b/.github/workflows/downstream-project-anchor.yml
@@ -38,11 +38,11 @@ env:
 
 jobs:
   test:
+    if: github.repository == 'anza-xyz/agave'
     runs-on: ubuntu-latest
     strategy:
       matrix:
         version: ["master"]
-    if: false # Re-enable once new major versions for spl-token-2022 and spl-pod are out
     steps:
       - uses: actions/checkout@v4
 
diff --git a/scripts/build-downstream-anchor-projects.sh b/scripts/build-downstream-anchor-projects.sh
index fe2f5e813ee613..e282a446c7c79a 100755
--- a/scripts/build-downstream-anchor-projects.sh
+++ b/scripts/build-downstream-anchor-projects.sh
@@ -69,7 +69,6 @@ anchor() {
   patch_crates_io_solana Cargo.toml "$solana_dir"
   patch_spl_crates . Cargo.toml "$spl_dir"
 
-  sed -i '/\[patch.crates-io\]/a curve25519-dalek = { git = "https://github.com/anza-xyz/curve25519-dalek.git", rev = "b500cdc2a920cd5bff9e2dd974d7b97349d61464" }' ./Cargo.toml
   $cargo test
   # serum_dex and mpl-token-metadata are using caret versions of solana and SPL dependencies
   # rather pull and patch those as well, ignore for now
diff --git a/scripts/patch-crates.sh b/scripts/patch-crates.sh
index 771dcddbd02fa4..45a2d67caf7f9b 100644
--- a/scripts/patch-crates.sh
+++ b/scripts/patch-crates.sh
@@ -6,24 +6,128 @@ update_solana_dependencies() {
   declare tomls=()
   while IFS='' read -r line; do tomls+=("$line"); done < <(find "$project_root" -name Cargo.toml)
 
-  sed -i -e "s#\(solana-program = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-program = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-program-test = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-program-test = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-sdk = \"\).*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-sdk = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-client = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-client = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-cli-config = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-cli-config = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-clap-utils = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-clap-utils = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-account-decoder = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-account-decoder = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-faucet = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-faucet = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-zk-token-sdk = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-zk-token-sdk = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+  crates=(
+    solana-account-decoder
+    solana-account-decoder-client-types
+    solana-banks-client
+    solana-banks-interface
+    solana-banks-server
+    solana-bloom
+    solana-bucket-map
+    solana-builtins-default-costs
+    solana-clap-utils
+    solana-clap-v3-utils
+    solana-cli-config
+    solana-cli-output
+    solana-client
+    solana-compute-budget
+    solana-connection-cache
+    solana-core
+    solana-entry
+    solana-faucet
+    solana-fee
+    solana-frozen-abi
+    solana-frozen-abi-macro
+    agave-geyser-plugin-interface
+    solana-geyser-plugin-manager
+    solana-gossip
+    solana-lattice-hash
+    solana-ledger
+    solana-log-collector
+    solana-logger
+    solana-measure
+    solana-merkle-tree
+    solana-metrics
+    solana-net-utils
+    solana-perf
+    solana-poh
+    solana-program-runtime
+    solana-program-test
+    solana-address-lookup-table-program
+    solana-bpf-loader-program
+    solana-compute-budget-program
+    solana-config-program
+    solana-stake-program
+    solana-system-program
+    solana-vote-program
+    solana-zk-elgamal-proof-program
+    solana-zk-token-proof-program
+    solana-pubsub-client
+    solana-quic-client
+    solana-rayon-threadlimit
+    solana-remote-wallet
+    solana-rpc
+    solana-rpc-client
+    solana-rpc-client-api
+    solana-rpc-client-nonce-utils
+    solana-runtime
+    solana-runtime-transaction
+    solana-sdk
+    solana-sdk-macro
+    solana-program
+    solana-send-transaction-service
+    solana-storage-bigtable
+    solana-storage-proto
+    solana-streamer
+    solana-svm-rent-collector
+    solana-svm-transaction
+    solana-test-validator
+    solana-thin-client
+    solana-tpu-client
+    solana-transaction-status
+    solana-transaction-status-client-types
+    solana-udp-client
+    solana-version
+    solana-zk-token-sdk
+    solana-zk-sdk
+    solana-bn254
+    solana-curve25519
+    solana-secp256k1-recover
+    solana-account
+    solana-account-info
+    solana-atomic-u64
+    solana-bincode
+    solana-borsh
+    solana-clock
+    solana-cpi
+    solana-decode-error
+    solana-define-syscall
+    solana-derivation-path
+    solana-epoch-schedule
+    solana-feature-set
+    solana-fee-calculator
+    solana-hash
+    solana-inflation
+    solana-instruction
+    solana-last-restart-slot
+    solana-msg
+    solana-native-token
+    solana-packet
+    solana-precompile-error
+    solana-program-entrypoint
+    solana-program-error
+    solana-program-memory
+    solana-program-option
+    solana-program-pack
+    solana-pubkey
+    solana-rent
+    solana-sanitize
+    solana-serde-varint
+    solana-serialize-utils
+    solana-sha256-hasher
+    solana-short-vec
+    solana-signature
+    solana-slot-hashes
+    solana-stable-layout
+    solana-timings
+    solana-transaction-error
+  )
+
+  set -x
+  for crate in "${crates[@]}"; do
+    sed -E -i'' -e "s:(${crate} = \")([=<>]*)[0-9.]+([^\"]*)\".*:\1\2${solana_ver}\3\":" "${tomls[@]}"
+    sed -E -i'' -e "s:(${crate} = \{ version = \")([=<>]*)[0-9.]+([^\"]*)(\".*):\1\2${solana_ver}\3\4:" "${tomls[@]}"
+  done
 }
 
 patch_crates_io_solana() {
@@ -32,21 +136,149 @@ patch_crates_io_solana() {
   cat >> "$Cargo_toml" <<EOF
 [patch.crates-io]
 EOF
-patch_crates_io_solana_no_header "$Cargo_toml" "$solana_dir"
+  patch_crates_io_solana_no_header "$Cargo_toml" "$solana_dir"
 }
 
 patch_crates_io_solana_no_header() {
   declare Cargo_toml="$1"
   declare solana_dir="$2"
-  cat >> "$Cargo_toml" <<EOF
-solana-account-decoder = { path = "$solana_dir/account-decoder" }
-solana-clap-utils = { path = "$solana_dir/clap-utils" }
-solana-client = { path = "$solana_dir/client" }
-solana-cli-config = { path = "$solana_dir/cli-config" }
-solana-program = { path = "$solana_dir/sdk/program" }
-solana-program-test = { path = "$solana_dir/program-test" }
-solana-sdk = { path = "$solana_dir/sdk" }
-solana-faucet = { path = "$solana_dir/faucet" }
-solana-zk-token-sdk = { path = "$solana_dir/zk-token-sdk" }
-EOF
+
+  crates_map=()
+  crates_map+=("solana-account-decoder account-decoder")
+  crates_map+=("solana-account-decoder-client-types account-decoder-client-types")
+  crates_map+=("solana-banks-client banks-client")
+  crates_map+=("solana-banks-interface banks-interface")
+  crates_map+=("solana-banks-server banks-server")
+  crates_map+=("solana-bloom bloom")
+  crates_map+=("solana-bucket-map bucket_map")
+  crates_map+=("solana-builtins-default-costs builtins-default-costs")
+  crates_map+=("solana-clap-utils clap-utils")
+  crates_map+=("solana-clap-v3-utils clap-v3-utils")
+  crates_map+=("solana-cli-config cli-config")
+  crates_map+=("solana-cli-output cli-output")
+  crates_map+=("solana-client client")
+  crates_map+=("solana-compute-budget compute-budget")
+  crates_map+=("solana-connection-cache connection-cache")
+  crates_map+=("solana-core core")
+  crates_map+=("solana-entry entry")
+  crates_map+=("solana-faucet faucet")
+  crates_map+=("solana-fee fee")
+  crates_map+=("agave-geyser-plugin-interface geyser-plugin-interface")
+  crates_map+=("solana-geyser-plugin-manager geyser-plugin-manager")
+  crates_map+=("solana-gossip gossip")
+  crates_map+=("solana-lattice-hash lattice-hash")
+  crates_map+=("solana-ledger ledger")
+  crates_map+=("solana-log-collector log-collector")
+  crates_map+=("solana-measure measure")
+  crates_map+=("solana-merkle-tree merkle-tree")
+  crates_map+=("solana-metrics metrics")
+  crates_map+=("solana-net-utils net-utils")
+  crates_map+=("solana-perf perf")
+  crates_map+=("solana-poh poh")
+  crates_map+=("solana-program-runtime program-runtime")
+  crates_map+=("solana-program-test program-test")
+  crates_map+=("solana-address-lookup-table-program programs/address-lookup-table")
+  crates_map+=("solana-bpf-loader-program programs/bpf_loader")
+  crates_map+=("solana-compute-budget-program programs/compute-budget")
+  crates_map+=("solana-config-program programs/config")
+  crates_map+=("solana-stake-program programs/stake")
+  crates_map+=("solana-system-program programs/system")
+  crates_map+=("solana-vote-program programs/vote")
+  crates_map+=("solana-zk-elgamal-proof-program programs/zk-elgamal-proof")
+  crates_map+=("solana-zk-token-proof-program programs/zk-token-proof")
+  crates_map+=("solana-pubsub-client pubsub-client")
+  crates_map+=("solana-quic-client quic-client")
+  crates_map+=("solana-rayon-threadlimit rayon-threadlimit")
+  crates_map+=("solana-remote-wallet remote-wallet")
+  crates_map+=("solana-rpc rpc")
+  crates_map+=("solana-rpc-client rpc-client")
+  crates_map+=("solana-rpc-client-api rpc-client-api")
+  crates_map+=("solana-rpc-client-nonce-utils rpc-client-nonce-utils")
+  crates_map+=("solana-runtime runtime")
+  crates_map+=("solana-runtime-transaction runtime-transaction")
+  crates_map+=("solana-sdk sdk")
+  crates_map+=("solana-sdk-macro sdk/macro")
+  crates_map+=("solana-program sdk/program")
+  crates_map+=("solana-send-transaction-service send-transaction-service")
+  crates_map+=("solana-storage-bigtable storage-bigtable")
+  crates_map+=("solana-storage-proto storage-proto")
+  crates_map+=("solana-streamer streamer")
+  crates_map+=("solana-svm-rent-collector svm-rent-collector")
+  crates_map+=("solana-svm-transaction svm-transaction")
+  crates_map+=("solana-test-validator test-validator")
+  crates_map+=("solana-thin-client thin-client")
+  crates_map+=("solana-tpu-client tpu-client")
+  crates_map+=("solana-transaction-status transaction-status")
+  crates_map+=("solana-transaction-status-client-types transaction-status-client-types")
+  crates_map+=("solana-udp-client udp-client")
+  crates_map+=("solana-version version")
+  crates_map+=("solana-zk-token-sdk zk-token-sdk")
+  crates_map+=("solana-zk-sdk zk-sdk")
+  crates_map+=("solana-bn254 curves/bn254")
+  crates_map+=("solana-curve25519 curves/curve25519")
+  crates_map+=("solana-secp256k1-recover curves/secp256k1-recover")
+  crates_map+=("solana-account sdk/account")
+  crates_map+=("solana-account-info sdk/account-info")
+  crates_map+=("solana-atomic-u64 sdk/atomic-u64")
+  crates_map+=("solana-bincode sdk/bincode")
+  crates_map+=("solana-borsh sdk/borsh")
+  crates_map+=("solana-clock sdk/clock")
+  crates_map+=("solana-cpi sdk/cpi")
+  crates_map+=("solana-decode-error sdk/decode-error")
+  crates_map+=("solana-define-syscall sdk/define-syscall")
+  crates_map+=("solana-derivation-path sdk/derivation-path")
+  crates_map+=("solana-epoch-schedule sdk/epoch-schedule")
+  crates_map+=("solana-feature-set sdk/feature-set")
+  crates_map+=("solana-fee-calculator sdk/fee-calculator")
+  crates_map+=("solana-frozen-abi sdk/frozen-abi")
+  crates_map+=("solana-frozen-abi-macro sdk/frozen-abi/macro")
+  crates_map+=("solana-hash sdk/hash")
+  crates_map+=("solana-inflation sdk/inflation")
+  crates_map+=("solana-instruction sdk/instruction")
+  crates_map+=("solana-last-restart-slot sdk/last-restart-slot")
+  crates_map+=("solana-logger sdk/logger")
+  crates_map+=("solana-msg sdk/msg")
+  crates_map+=("solana-native-token sdk/native-token")
+  crates_map+=("solana-packet sdk/packet")
+  crates_map+=("solana-precompile-error sdk/precompile-error")
+  crates_map+=("solana-program-entrypoint sdk/program-entrypoint")
+  crates_map+=("solana-program-error sdk/program-error")
+  crates_map+=("solana-program-memory sdk/program-memory")
+  crates_map+=("solana-program-option sdk/program-option")
+  crates_map+=("solana-program-pack sdk/program-pack")
+  crates_map+=("solana-pubkey sdk/pubkey")
+  crates_map+=("solana-rent sdk/rent")
+  crates_map+=("solana-sanitize sdk/sanitize")
+  crates_map+=("solana-serde-varint sdk/serde-varint")
+  crates_map+=("solana-serialize-utils sdk/serialize-utils")
+  crates_map+=("solana-sha256-hasher sdk/sha256-hasher")
+  crates_map+=("solana-short-vec sdk/short-vec")
+  crates_map+=("solana-signature sdk/signature")
+  crates_map+=("solana-slot-hashes sdk/slot-hashes")
+  crates_map+=("solana-stable-layout sdk/stable-layout")
+  crates_map+=("solana-timings sdk/timings")
+  crates_map+=("solana-transaction-error sdk/transaction-error")
+
+  patch_crates=()
+  for map_entry in "${crates_map[@]}"; do
+    read -r crate_name crate_path <<<"$map_entry"
+    full_path="$solana_dir/$crate_path"
+    if [[ -r "$full_path/Cargo.toml" ]]; then
+      patch_crates+=("$crate_name = { path = \"$full_path\" }")
+    fi
+  done
+
+  echo "Patching in $solana_ver from $solana_dir"
+  echo
+  if grep -q "# The following entries are auto-generated by $0" "$Cargo_toml"; then
+    echo "$Cargo_toml is already patched"
+  else
+    if ! grep -q '\[patch.crates-io\]' "$Cargo_toml"; then
+      echo "[patch.crates-io]" >> "$Cargo_toml"
+    fi
+    cat >> "$Cargo_toml" <<PATCH
+# The following entries are auto-generated by $0
+$(printf "%s\n" "${patch_crates[@]}")
+PATCH
+  fi
 }