Add program scripts

.github/workflows/build-programs.yml

@@ -52,17 +52,16 @@ jobs:
         uses: metaplex-foundation/actions/cache-programs@v1
 
       - name: Build programs
+        shell: bash
+        working-directory: configs/program-scripts
+        run: ./build.sh
         env:
           PROGRAMS: ${{ env.PROGRAMS }}
-        run: |
-          for program in $(echo $PROGRAMS | jq -c '.[]' | sed 's/"//g'); do
-            (cd ./programs/$program; cargo build-bpf)
-          done
 
       - name: Upload program builds
         uses: actions/upload-artifact@v3
         with:
           name: program-builds
           # First wildcard ensures exported paths are consistently under the programs folder.
-          path: ./program*/**/target/deploy/*.so
+          path: ./program*/.bin/*.so
           if-no-files-found: error

.github/workflows/test-programs.yml

@@ -54,5 +54,8 @@ jobs:
           args: --all-targets --all-features --no-deps --manifest-path ./programs/${{ matrix.program }}/Cargo.toml
 
       - name: Run tests
-        working-directory: ./programs/${{ matrix.program }}
-        run: cargo test-bpf
+        shell: bash
+        working-directory: configs/program-scripts
+        run: ./test.sh
+        env:
+          PROGRAM: ${{ matrix.program }}

.gitignore

@@ -8,3 +8,4 @@ test-ledger
 dist
 .amman
 .crates
+.bin

configs/program-scripts/build.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+OUTPUT="./programs/.bin"
+# saves external programs binaries to the output directory
+source ${SCRIPT_DIR}/dump.sh ${OUTPUT}
+# go to parent folder
+cd $(dirname $(dirname ${SCRIPT_DIR}))
+
+if [ -z ${PROGRAMS+x} ]; then
+    PROGRAMS="$(cat .github/.env | grep "PROGRAMS" | cut -d '=' -f 2)"
+fi
+
+# default to input from the command-line
+ARGS=$*
+
+# command-line arguments override env variable
+if [ ! -z "$ARGS" ]; then
+    PROGRAMS="[\"${1}\"]"
+    shift
+    ARGS=$*
+fi
+
+PROGRAMS=$(echo ${PROGRAMS} | jq -c '.[]' | sed 's/"//g')
+
+# creates the output directory if it doesn't exist
+if [ ! -d ${OUTPUT} ]; then
+    mkdir ${OUTPUT}
+fi
+
+WORKING_DIR=$(pwd)
+export BPF_OUT_DIR="${WORKING_DIR}/${OUTPUT}"
+
+for p in ${PROGRAMS[@]}; do
+    cd ${WORKING_DIR}/programs/${p}
+    cargo build-bpf --bpf-out-dir ${WORKING_DIR}/${OUTPUT} $ARGS
+done

configs/program-scripts/dump.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# output colours
+RED() { echo $'\e[1;31m'$1$'\e[0m'; }
+GRN() { echo $'\e[1;32m'$1$'\e[0m'; }
+
+CURRENT_DIR=$(pwd)
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+# go to parent folder
+cd $(dirname $(dirname $SCRIPT_DIR))
+
+OUTPUT=$1
+EXTERNAL_ID=("auth9SigNpDKz4sJJ1DfCTuZrZNSAgh9sFD3rboVmgg" "SysExL2WDyJi9aRZrXorrjHJut3JwHQ7R9bTyctbNNG" "TokExjvjJmhKaRBShsBAsbSvEWMA1AgUNK7ps4SAc2p")
+EXTERNAL_SO=("mpl_token_auth_rules.so" "mpl_system_extras.so" "mpl_token_extras.so")
+
+if [ -z ${RPC+x} ]; then
+    RPC="https://api.mainnet-beta.solana.com"
+fi
+
+if [ -z "$OUTPUT" ]; then
+    echo "missing output directory"
+    exit 1
+fi
+
+# creates the output directory if it doesn't exist
+if [ ! -d ${OUTPUT} ]; then
+    mkdir ${OUTPUT}
+fi
+
+# only prints this if we have external programs
+if [ ${#EXTERNAL_ID[@]} -gt 0 ]; then
+    echo "Dumping external programs to: '${OUTPUT}'"
+fi
+
+# dump external programs binaries if needed
+for i in ${!EXTERNAL_ID[@]}; do
+    if [ ! -f "${OUTPUT}/${EXTERNAL_SO[$i]}" ]; then
+        solana program dump -u $RPC ${EXTERNAL_ID[$i]} ${OUTPUT}/${EXTERNAL_SO[$i]}
+    else
+        solana program dump -u $RPC ${EXTERNAL_ID[$i]} ${OUTPUT}/onchain-${EXTERNAL_SO[$i]} > /dev/null
+        ON_CHAIN=`sha256sum -b ${OUTPUT}/onchain-${EXTERNAL_SO[$i]} | cut -d ' ' -f 1`
+        LOCAL=`sha256sum -b ${OUTPUT}/${EXTERNAL_SO[$i]} | cut -d ' ' -f 1`
+
+        if [ "$ON_CHAIN" != "$LOCAL" ]; then
+            echo $(RED "[ WARNING ] on-chain and local binaries are different for '${EXTERNAL_SO[$i]}'")
+        else
+            echo "$(GRN "[ SKIPPED ]") on-chain and local binaries are the same for '${EXTERNAL_SO[$i]}'"
+        fi
+
+        rm ${OUTPUT}/onchain-${EXTERNAL_SO[$i]}
+    fi
+done
+
+# only prints this if we have external programs
+if [ ${#EXTERNAL_ID[@]} -gt 0 ]; then
+    echo ""
+fi
+
+cd ${CURRENT_DIR}

configs/program-scripts/test.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+OUTPUT="./programs/.bin"
+# saves external programs binaries to the output directory
+source ${SCRIPT_DIR}/dump.sh ${OUTPUT}
+# go to parent folder
+cd $(dirname $(dirname $SCRIPT_DIR))
+
+if [ ! -z "$PROGRAM" ]; then
+    PROGRAMS='["'${PROGRAM}'"]'
+fi
+
+if [ -z "$PROGRAMS" ]; then
+    PROGRAMS="$(cat .github/.env | grep "PROGRAMS" | cut -d '=' -f 2)"
+fi
+
+# default to input from the command-line
+ARGS=$*
+
+# command-line arguments override env variable
+if [ ! -z "$ARGS" ]; then
+    PROGRAMS="[\"${1}\"]"
+    shift
+    ARGS=$*
+fi
+
+PROGRAMS=$(echo $PROGRAMS | jq -c '.[]' | sed 's/"//g')
+
+WORKING_DIR=$(pwd)
+SOLFMT="solfmt"
+export BPF_OUT_DIR="${WORKING_DIR}/${OUTPUT}"
+
+for p in ${PROGRAMS[@]}; do
+    cd ${WORKING_DIR}/programs/${p}
+
+    if [ ! "$(command -v $SOLFMT)" = "" ]; then
+        CARGO_TERM_COLOR=always cargo test-bpf --bpf-out-dir ${WORKING_DIR}/${OUTPUT} ${ARGS} 2>&1 | ${SOLFMT}
+    else
+        cargo test-bpf --bpf-out-dir ${WORKING_DIR}/${OUTPUT} ${ARGS}
+    fi
+done

configs/validator.cjs

@@ -1,11 +1,9 @@
 const path = require("path");
 
 const programDir = path.join(__dirname, "..", "programs");
-function getProgram(dir, programName) {
-  return path.join(programDir, dir, "target", "deploy", programName);
-}
-function getExternalProgram(programName) {
-  return path.join(__dirname, "external-programs", programName);
+
+function getProgram(programBinary) {
+  return path.join(programDir, ".bin", programBinary);
 }
 
 module.exports = {
@@ -15,22 +13,22 @@ module.exports = {
       {
         label: "Token Metadata",
         programId: "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s",
-        deployPath: getProgram("token-metadata", "mpl_token_metadata.so"),
+        deployPath: getProgram("mpl_token_metadata.so"),
       },
       {
         label: "Token Auth Rules",
         programId: "auth9SigNpDKz4sJJ1DfCTuZrZNSAgh9sFD3rboVmgg",
-        deployPath: getExternalProgram("mpl_token_auth_rules.so"),
+        deployPath: getProgram("mpl_token_auth_rules.so"),
       },
       {
         label: "System Extras",
         programId: "SysExL2WDyJi9aRZrXorrjHJut3JwHQ7R9bTyctbNNG",
-        deployPath: getExternalProgram("mpl_system_extras.so"),
+        deployPath: getProgram("mpl_system_extras.so"),
       },
       {
         label: "Token Extras",
         programId: "TokExjvjJmhKaRBShsBAsbSvEWMA1AgUNK7ps4SAc2p",
-        deployPath: getExternalProgram("mpl_token_extras.so"),
+        deployPath: getProgram("mpl_token_extras.so"),
       },
     ],
   },

package.json

@@ -1,12 +1,15 @@
 {
   "private": true,
   "scripts": {
+    "programs:build": "./configs/program-scripts/build.sh",
+    "programs:test": "./configs/program-scripts/test.sh",
+    "programs:clean": "rm -rf ./programs/.bin",
     "generate": "pnpm generate:idls && pnpm generate:clients",
     "generate:idls": "node ./configs/shank.cjs",
     "generate:clients": "node ./configs/kinobi.cjs",
     "validator": "CI=1 amman start --config ./configs/validator.cjs",
-    "validator:devnet": "CI=1 amman start --config ./configs/validator.devnet.cjs",
-    "validator:mainnet": "CI=1 amman start --config ./configs/validator.mainnet.cjs",
+    "validator:debug": "amman start --config ./configs/validator.cjs",
+    "validator:logs": "CI=1 amman logs",
     "validator:stop": "amman stop"
   },
   "devDependencies": {