Merge branch 'main' of https://github.com/eth-educators/eth-docker

besu.yml

@@ -24,6 +24,7 @@ services:
       - JWT_SECRET=${JWT_SECRET:-}
       - EL_EXTRAS=${EL_EXTRAS:-}
       - ARCHIVE_NODE=${ARCHIVE_NODE:-}
+      - NETWORK=${NETWORK}
     volumes:
       - besu-eth1-data:/var/lib/besu
       - /etc/localtime:/etc/localtime:ro
@@ -44,8 +45,6 @@ services:
       - --p2p-port
       - ${EL_P2P_PORT:-30303}
       - --rpc-http-enabled
-      - --rpc-http-api
-      - "WEB3,ETH,NET"
       - --rpc-http-host
       - 0.0.0.0
       - --rpc-http-port
@@ -68,8 +67,6 @@ services:
       - --engine-rpc-port=${EE_PORT:-8551}
       - --logging
       - ${LOG_LEVEL}
-      - --network
-      - ${NETWORK}
       - --metrics-enabled
       - --metrics-host
       - 0.0.0.0

besu/Dockerfile.binary

@@ -15,7 +15,7 @@ USER root
 RUN groupmod -g "${GID}" besu && usermod -u "${UID}" -g "${GID}" besu
 
 RUN set -eux; \
-        apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y gosu ca-certificates tzdata; \
+        apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y gosu ca-certificates tzdata git; \
         rm -rf /var/lib/apt/lists/*; \
 # verify that the binary works
         gosu nobody true

besu/Dockerfile.source

@@ -24,6 +24,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install
   tzdata \
   gosu \
   libjemalloc-dev \
+  git \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 

besu/docker-entrypoint.sh

@@ -26,6 +26,32 @@ if [[ -O "/var/lib/besu/ee-secret/jwtsecret" ]]; then
   chmod 666 /var/lib/besu/ee-secret/jwtsecret
 fi
 
+if [[ "${NETWORK}" =~ ^https?:// ]]; then
+  echo "Custom testnet at ${NETWORK}"
+  repo=$(awk -F'/tree/' '{print $1}' <<< "${NETWORK}")
+  branch=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f1)
+  config_dir=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f2-)
+  echo "This appears to be the ${repo} repo, branch ${branch} and config directory ${config_dir}."
+  # For want of something more amazing, let's just fail if git fails to pull this
+  set -e
+  if [ ! -d "/var/lib/besu/testnet/${config_dir}" ]; then
+    mkdir -p /var/lib/besu/testnet
+    cd /var/lib/besu/testnet
+    git init --initial-branch="${branch}"
+    git remote add origin "${repo}"
+    git config core.sparseCheckout true
+    echo "${config_dir}" > .git/info/sparse-checkout
+    git pull origin "${branch}"
+  fi
+  bootnodes="$(paste -s -d, "/var/lib/besu/testnet/${config_dir}/bootnode.txt")"
+  set +e
+  __network="--genesis-file=/var/lib/besu/testnet/${config_dir}/besu.json --bootnodes=${bootnodes} \
+--kzg-trusted-setup=/var/lib/besu/testnet/${config_dir}/trusted_setup.txt --Xfilter-on-enr-fork-id=true \
+--rpc-http-api=ADMIN,CLIQUE,MINER,ETH,NET,DEBUG,TXPOOL,ENGINE,TRACE,WEB3"
+else
+  __network="--network ${NETWORK} --rpc-http-api WEB3,ETH,NET"
+fi
+
 if [ "${ARCHIVE_NODE}" = "true" ]; then
   echo "Besu archive node without pruning"
   __prune="--data-storage-format=FOREST --sync-mode=FULL"
@@ -35,4 +61,4 @@ fi
 
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-exec "$@" ${__prune} ${EL_EXTRAS}
+exec "$@" ${__network} ${__prune} ${EL_EXTRAS}

erigon.yml

@@ -23,6 +23,7 @@ services:
       - JWT_SECRET=${JWT_SECRET}
       - EL_EXTRAS=${EL_EXTRAS:-}
       - ARCHIVE_NODE=${ARCHIVE_NODE:-}
+      - NETWORK=${NETWORK}
     volumes:
       - erigon-el-data:/var/lib/erigon
       - /etc/localtime:/etc/localtime:ro
@@ -57,8 +58,6 @@ services:
       - ${ERIGON_TORRENT_PORT:-42069}
       - --nat
       - any
-      - --chain
-      - ${NETWORK}
       - --log.console.verbosity
       - ${LOG_LEVEL}
       - --metrics
@@ -71,8 +70,6 @@ services:
       - ${EL_RPC_PORT}
       - --http.vhosts=*
       - --http.corsdomain=*
-      - --http.api
-      - web3,eth,net,engine
       - --ws
       # Allow RocketPool >=1.9 watchtower queries
       - --rpc.returndata.limit

erigon/Dockerfile.binary

@@ -12,7 +12,7 @@ ARG GID=10002
 
 USER root
 
-RUN apk --no-cache add shadow bash su-exec && groupmod -g "${GID}" ${USER} && usermod -u "${UID}" -g "${GID}" ${USER}
+RUN apk --no-cache add shadow bash su-exec git jq && groupmod -g "${GID}" ${USER} && usermod -u "${UID}" -g "${GID}" ${USER}
 
 RUN mkdir -p /var/lib/erigon/ee-secret && chown -R ${USER}:${USER} /var/lib/erigon && chmod -R 700 /var/lib/erigon && chmod 777 /var/lib/erigon/ee-secret
 

erigon/Dockerfile.source

@@ -20,7 +20,7 @@ ARG UID=10001
 # GID 10002 is deliberate so it can exchange secret with CL
 ARG GID=10002
 
-RUN apk add --no-cache libgcc libstdc++ bash ca-certificates tzdata su-exec
+RUN apk add --no-cache libgcc libstdc++ bash ca-certificates tzdata su-exec git jq
 
 RUN addgroup \
     --gid "${GID}" \

erigon/docker-entrypoint.sh

@@ -26,6 +26,34 @@ if [[ -O "/var/lib/erigon/ee-secret/jwtsecret" ]]; then
   chmod 666 /var/lib/erigon/ee-secret/jwtsecret
 fi
 
+if [[ "${NETWORK}" =~ ^https?:// ]]; then
+  echo "Custom testnet at ${NETWORK}"
+  repo=$(awk -F'/tree/' '{print $1}' <<< "${NETWORK}")
+  branch=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f1)
+  config_dir=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f2-)
+  echo "This appears to be the ${repo} repo, branch ${branch} and config directory ${config_dir}."
+  # For want of something more amazing, let's just fail if git fails to pull this
+  set -e
+  if [ ! -d "/var/lib/erigon/testnet/${config_dir}" ]; then
+    mkdir -p /var/lib/erigon/testnet
+    cd /var/lib/erigon/testnet
+    git init --initial-branch="${branch}"
+    git remote add origin "${repo}"
+    git config core.sparseCheckout true
+    echo "${config_dir}" > .git/info/sparse-checkout
+    git pull origin "${branch}"
+  fi
+  bootnodes="$(paste -s -d, "/var/lib/erigon/testnet/${config_dir}/bootnode.txt")"
+  networkid="$(jq -r '.config.chainId' "/var/lib/erigon/testnet/${config_dir}/genesis.json")"
+  set +e
+  __network="--bootnodes=${bootnodes} --networkid=${networkid} --http.api=eth,erigon,engine,web3,net,debug,trace,txpool,admin"
+  if [ ! -f /var/lib/erigon/setupdone ]; then
+    erigon init --datadir /var/lib/erigon "/var/lib/erigon/testnet/${config_dir}/genesis.json"
+    touch /var/lib/erigon/setupdone
+  fi
+else
+  __network="--chain ${NETWORK} --http.api web3,eth,net,engine"
+fi
 # Check for network, and set prune accordingly
 
 if [ "${ARCHIVE_NODE}" = "true" ]; then
@@ -52,4 +80,4 @@ fi
 
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-exec "$@" ${__prune} ${EL_EXTRAS}
+exec "$@" ${__network} ${__prune} ${EL_EXTRAS}

lighthouse-cl-only.yml

@@ -37,6 +37,7 @@ services:
       - ARCHIVE_NODE=${ARCHIVE_NODE:-false}
       - IPV6=${IPV6:-false}
       - CL_P2P_PORT=${CL_P2P_PORT:-9000}
+      - NETWORK=${NETWORK}
     ports:
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp
@@ -57,8 +58,6 @@ services:
       - 0.0.0.0
       - --http-port
       - ${CL_REST_PORT:-5052}
-      - --network
-      - ${NETWORK}
       - --port
       - ${CL_P2P_PORT:-9000}
       - --target-peers

lighthouse-vc-only.yml

@@ -29,6 +29,7 @@ services:
       - VC_EXTRAS=${VC_EXTRAS:-}
       - GRAFFITI=${GRAFFITI:-}
       - DEFAULT_GRAFFITI=${DEFAULT_GRAFFITI:-false}
+      - NETWORK=${NETWORK}
     volumes:
       - lhvalidator-data:/var/lib/lighthouse
       - /etc/localtime:/etc/localtime:ro
@@ -45,8 +46,6 @@ services:
       - /var/lib/lighthouse
       - --beacon-nodes
       - ${CL_NODE:-http://consensus:5052}
-      - --network
-      - ${NETWORK}
       - --debug-level=${LOG_LEVEL}
       - --metrics
       - --metrics-address

lighthouse.yml

@@ -38,6 +38,7 @@ services:
       - ARCHIVE_NODE=${ARCHIVE_NODE:-false}
       - IPV6=${IPV6:-false}
       - CL_P2P_PORT=${CL_P2P_PORT:-9000}
+      - NETWORK=${NETWORK}
     ports:
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp
@@ -58,8 +59,6 @@ services:
       - 0.0.0.0
       - --http-port
       - ${CL_REST_PORT:-5052}
-      - --network
-      - ${NETWORK}
       - --listen-address
       - 0.0.0.0
       - --port
@@ -94,6 +93,7 @@ services:
       - VC_EXTRAS=${VC_EXTRAS:-}
       - GRAFFITI=${GRAFFITI:-}
       - DEFAULT_GRAFFITI=${DEFAULT_GRAFFITI:-false}
+      - NETWORK=${NETWORK}
     volumes:
       - lhvalidator-data:/var/lib/lighthouse
       - /etc/localtime:/etc/localtime:ro
@@ -110,8 +110,6 @@ services:
       - /var/lib/lighthouse
       - --beacon-nodes
       - ${CL_NODE:-http://consensus:5052}
-      - --network
-      - ${NETWORK}
       - --debug-level=${LOG_LEVEL}
       - --metrics
       - --metrics-address

lighthouse/Dockerfile.binary

@@ -43,7 +43,7 @@ RUN mkdir -p /var/lib/lighthouse/validators && chown ${USER}:${USER} /var/lib/li
 
 RUN set -eux; \
         apt-get update; \
-        DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y gosu ca-certificates tzdata; \
+        DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y gosu ca-certificates tzdata git; \
         rm -rf /var/lib/apt/lists/*; \
 # verify that the binary works
         gosu nobody true

lighthouse/Dockerfile.source

@@ -36,6 +36,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install
   ca-certificates \
   wget \
   tzdata \
+  git \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 

lighthouse/docker-entrypoint-vc.sh

@@ -6,6 +6,29 @@ if [ "$(id -u)" = '0' ]; then
   exec gosu lhvalidator docker-entrypoint.sh "$@"
 fi
 
+if [[ "${NETWORK}" =~ ^https?:// ]]; then
+  echo "Custom testnet at ${NETWORK}"
+  repo=$(awk -F'/tree/' '{print $1}' <<< "${NETWORK}")
+  branch=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f1)
+  config_dir=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f2-)
+  echo "This appears to be the ${repo} repo, branch ${branch} and config directory ${config_dir}."
+  # For want of something more amazing, let's just fail if git fails to pull this
+  set -e
+  if [ ! -d "/var/lib/lighthouse/validators/testnet/${config_dir}" ]; then
+    mkdir -p /var/lib/lighthouse/validators/testnet
+    cd /var/lib/lighthouse/validators/testnet
+    git init --initial-branch="${branch}"
+    git remote add origin "${repo}"
+    git config core.sparseCheckout true
+    echo "${config_dir}" > .git/info/sparse-checkout
+    git pull origin "${branch}"
+  fi
+  set +e
+  __network="--testnet-dir=/var/lib/lighthouse/validators/testnet/${config_dir}"
+else
+  __network="--network=${NETWORK}"
+fi
+
 # Check whether we should use MEV Boost
 if [ "${MEV_BOOST}" = "true" ]; then
   __mev_boost="--builder-proposals"
@@ -32,9 +55,9 @@ fi
 if [ "${DEFAULT_GRAFFITI}" = "true" ]; then
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-  exec "$@" ${__mev_boost} ${__beacon_stats} ${__doppel} ${VC_EXTRAS}
+  exec "$@" ${__network} ${__mev_boost} ${__beacon_stats} ${__doppel} ${VC_EXTRAS}
 else
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-  exec "$@" "--graffiti" "${GRAFFITI}" ${__mev_boost} ${__beacon_stats} ${__doppel} ${VC_EXTRAS}
+  exec "$@" ${__network} "--graffiti" "${GRAFFITI}" ${__mev_boost} ${__beacon_stats} ${__doppel} ${VC_EXTRAS}
 fi

lighthouse/docker-entrypoint.sh

@@ -19,6 +19,30 @@ if [[ -O "/var/lib/lighthouse/ee-secret/jwtsecret" ]]; then
   chmod 666 /var/lib/lighthouse/beacon/ee-secret/jwtsecret
 fi
 
+if [[ "${NETWORK}" =~ ^https?:// ]]; then
+  echo "Custom testnet at ${NETWORK}"
+  repo=$(awk -F'/tree/' '{print $1}' <<< "${NETWORK}")
+  branch=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f1)
+  config_dir=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f2-)
+  echo "This appears to be the ${repo} repo, branch ${branch} and config directory ${config_dir}."
+  # For want of something more amazing, let's just fail if git fails to pull this
+  set -e
+  if [ ! -d "/var/lib/lighthouse/beacon/testnet/${config_dir}" ]; then
+    mkdir -p /var/lib/lighthouse/beacon/testnet
+    cd /var/lib/lighthouse/beacon/testnet
+    git init --initial-branch="${branch}"
+    git remote add origin "${repo}"
+    git config core.sparseCheckout true
+    echo "${config_dir}" > .git/info/sparse-checkout
+    git pull origin "${branch}"
+  fi
+  bootnodes="$(paste -s -d, "/var/lib/lighthouse/beacon/testnet/${config_dir}/bootstrap_nodes.txt")"
+  set +e
+  __network="--testnet-dir=/var/lib/lighthouse/beacon/testnet/${config_dir} --boot-nodes=${bootnodes}"
+else
+  __network="--network=${NETWORK}"
+fi
+
 # Check whether we should rapid sync
 if [ -n "${RAPID_SYNC_URL}" ]; then
   __rapid_sync="--checkpoint-sync-url=${RAPID_SYNC_URL}"
@@ -58,4 +82,4 @@ fi
 
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-exec "$@" ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__ipv6} ${CL_EXTRAS}
+exec "$@" ${__network} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__ipv6} ${CL_EXTRAS}

lodestar.yml

@@ -87,6 +87,7 @@ services:
       - GRAFFITI=${GRAFFITI:-}
       - DEFAULT_GRAFFITI=${DEFAULT_GRAFFITI:-false}
       - WEB3SIGNER=${WEB3SIGNER}
+      - NETWORK=${NETWORK}
     volumes:
       - lsvalidator-data:/var/lib/lodestar/validators
       - /etc/localtime:/etc/localtime:ro
@@ -120,8 +121,6 @@ services:
       - 0.0.0.0
       - --logLevel
       - ${LOG_LEVEL}
-      - --network
-      - ${NETWORK}
       - --suggestedFeeRecipient
       - ${FEE_RECIPIENT}
       - --force

lodestar/docker-entrypoint-vc.sh

@@ -6,6 +6,29 @@ if [ "$(id -u)" = '0' ]; then
   exec su-exec lsvalidator docker-entrypoint.sh "$@"
 fi
 
+if [[ "${NETWORK}" =~ ^https?:// ]]; then
+  echo "Custom testnet at ${NETWORK}"
+  repo=$(awk -F'/tree/' '{print $1}' <<< "${NETWORK}")
+  branch=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f1)
+  config_dir=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f2-)
+  echo "This appears to be the ${repo} repo, branch ${branch} and config directory ${config_dir}."
+  # For want of something more amazing, let's just fail if git fails to pull this
+  set -e
+  if [ ! -d "/var/lib/lodestar/validators/testnet/${config_dir}" ]; then
+    mkdir -p /var/lib/lodestar/validators/testnet
+    cd /var/lib/lodestar/validators/testnet
+    git init --initial-branch="${branch}"
+    git remote add origin "${repo}"
+    git config core.sparseCheckout true
+    echo "${config_dir}" > .git/info/sparse-checkout
+    git pull origin "${branch}"
+  fi
+  set +e
+  __network="--paramsFile=/var/lib/lodestar/validators/testnet/${config_dir}/config.yaml"
+else
+  __network="--network ${NETWORK}"
+fi
+
 # Check whether we should use MEV Boost
 if [ "${MEV_BOOST}" = "true" ]; then
   __mev_boost="--builder"
@@ -40,9 +63,9 @@ fi
 if [ "${DEFAULT_GRAFFITI}" = "true" ]; then
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-  exec "$@" ${__mev_boost} ${__beacon_stats} ${__doppel} ${__w3s_url} ${VC_EXTRAS}
+  exec "$@" ${__network} ${__mev_boost} ${__beacon_stats} ${__doppel} ${__w3s_url} ${VC_EXTRAS}
 else
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-  exec "$@" "--graffiti" "${GRAFFITI}" ${__mev_boost} ${__beacon_stats} ${__doppel} ${__w3s_url} ${VC_EXTRAS}
+  exec "$@" ${__network} "--graffiti" "${GRAFFITI}" ${__mev_boost} ${__beacon_stats} ${__doppel} ${__w3s_url} ${VC_EXTRAS}
 fi

nimbus-cl-only.yml

@@ -58,7 +58,6 @@ services:
       - --tcp-port=${CL_P2P_PORT:-9000}
       - --udp-port=${CL_P2P_PORT:-9000}
       - --max-peers=${CL_MAX_PEER_COUNT:-160}
-      - --network=${NETWORK}
       - --el=${EL_NODE}
       - --jwt-secret=/var/lib/nimbus/ee-secret/jwtsecret
       - --rest