Simplify build and install for CCF

Several changes to improve the performance (and remove dependencies) for CCF startup. Many of the changes are centered in the ccf Makefile to separate the creation of the ccf clients (like the ping test, ledger authority retrieval and policy scripts that might reasonably be executed from anywhere) from the environment needed for a ccf service node (the script to start a ccf network). The big win from the separation is that we can remove dependency on the sandbox script & pre-install all of the python package dependencies independently from the execution of the ccf nodes. The result is that the ledger containers start in a couple seconds instead of several minutes. Note that a future PR will package the ccf client scripts separately and install them with other PDO packages so they are available on all PDO installations. Signed-off-by: Mic Bowman <[email protected]>
hyperledger-labs · Feb 14, 2024 · c9cf579 · c9cf579
1 parent 6f74038
commit c9cf579
Show file tree

Hide file tree

Showing 6 changed files with 173 additions and 87 deletions.
diff --git a/docker/tools/build_ccf.sh b/docker/tools/build_ccf.sh
@@ -19,5 +19,8 @@ source /project/pdo/tools/environment.sh
 export PDO_HOSTNAME=
 export PDO_LEDGER_URL=
 
+# -----------------------------------------------------------------
+# set up the basic structure in ${PDO_INSTALL_ROOT}
+# -----------------------------------------------------------------
 make -C ${PDO_SOURCE_ROOT}/ledgers/ccf environment
 make -C ${PDO_SOURCE_ROOT}/ledgers/ccf install
diff --git a/docker/tools/environment.sh b/docker/tools/environment.sh
@@ -42,7 +42,6 @@ else
     export PDO_DEFAULT_SIGCURVE=SECP256K1
 fi
 
-export CCF_BASE=/opt/ccf_virtual
 export XFER_DIR=${XFER_DIR:-/project/pdo/xfer}
 
 # if the container is running HW mode, then we will grab the
@@ -62,3 +61,11 @@ export PDO_ENCLAVE_CODE_SIGN_PEM=${PDO_SGX_KEY_ROOT}/enclave_code_sign.pem
 # they are not used at build or run time
 export PDO_SPID="$(cat ${PDO_SGX_KEY_ROOT}/sgx_spid.txt)"
 export PDO_SPID_API_KEY="$(cat ${PDO_SGX_KEY_ROOT}/sgx_spid_api_key.txt)"
+
+# set up the ccf directories, ccf_base is where the ccf
+# core is installed, ccf_pdo_dir is where the pdo tp
+# components will be installed, and ccf_ledger_dir is
+# where the ccf python virtual environment will be built
+export CCF_BASE=/opt/ccf_virtual
+export CCF_PDO_DIR=${PDO_INSTALL_ROOT}
+export CCF_LEDGER_DIR=${PDO_HOME}/ccf
diff --git a/docker/tools/run_ccf_tests.sh b/docker/tools/run_ccf_tests.sh
@@ -34,9 +34,10 @@ mkdir -p ${PDO_LEDGER_KEY_ROOT}
 # -----------------------------------------------------------------
 yell start the ccf service
 # -----------------------------------------------------------------
-. ${PDO_HOME}/ccf/bin/activate
-yell ${PDO_HOME}/ccf/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}
-try ${PDO_HOME}/ccf/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}
+yell ${CCF_LEDGER_DIR}/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}  \
+    --pdo-dir ${CCF_PDO_DIR} --ledger-dir ${CCF_LEDGER_DIR}
+try ${CCF_LEDGER_DIR}/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}  \
+    --pdo-dir ${CCF_PDO_DIR} --ledger-dir ${CCF_LEDGER_DIR}
 
 # -----------------------------------------------------------------
 yell copy the ledger keys

diff --git a/docker/tools/start_ccf.sh b/docker/tools/start_ccf.sh
@@ -79,9 +79,11 @@ mkdir -p ${PDO_LEDGER_KEY_ROOT}
 # -----------------------------------------------------------------
 say start the ccf network
 # -----------------------------------------------------------------
-. ${PDO_HOME}/ccf/bin/activate
 if [ ${F_NETWORK_MODE} == "start" ] ; then
-    try ${PDO_HOME}/ccf/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}
+    yell ${CCF_LEDGER_DIR}/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}  \
+        --pdo-dir ${CCF_PDO_DIR} --ledger-dir ${CCF_LEDGER_DIR}
+    try ${CCF_LEDGER_DIR}/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}  \
+        --pdo-dir ${CCF_PDO_DIR} --ledger-dir ${CCF_LEDGER_DIR}
 elif [ ${F_NETWORK_MODE} == "join" ] ; then
     die "joining a network is not yet supported"
 else

diff --git a/ledgers/ccf/Makefile b/ledgers/ccf/Makefile
@@ -28,72 +28,99 @@ ifndef PDO_SOURCE_ROOT
 $(error Incomplete configuration, PDO_SOURCE_ROOT is not defined)
 endif
 
+CCF_COMPILE_TARGET ?= virtual
+CCF_VERSION ?= 4.0.1
+CCF_LEDGER_DIR ?= $(PDO_INSTALL_ROOT)/opt/pdo/ccf
+
 NINJA ?= ninja $(NINJA_OPTIONS)
 
 SCRIPTDIR ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-CCFDSTDIR ?= $(PDO_INSTALL_ROOT)/opt/pdo/ccf
-SRCDIR ?= $(PDO_SOURCE_ROOT)
-WORKSPACEDIR := $(CCFDSTDIR)/workspace
-BINDIR := $(CCFDSTDIR)/bin/
 BLDDIR := $(SCRIPTDIR)/build
 
-PYTHON_DIR=$(CCFDSTDIR)/lib/python3.8
-
-COMPILE_TARGET=virtual
-
 all: environment install
 
-build: build-pdo-tp
+# -----------------------------------------------------------------
+# There are two environments that need to be created: the PDO
+# environment and the ledger environment. This is necessitated by
+# the incompatibilities between the client requirements (which requires
+# ccf 1.0.19 which is the last python package with the ccf client modules)
+# and the ledger requirements (which installs the python modules from
+# our current verson of CCF). The client environment may be useful on
+# any pdo installation (e.g. ledger ping test makes sense on any client)
+# while the ledger environment is only useful where ccf nodes are run
+# -----------------------------------------------------------------
+environment: pdo-environment ledger-environment
+
+pdo-environment : $(PDO_INSTALL_ROOT)
+
+ledger-environment : $(CCF_LEDGER_DIR)/lib/python3.8
+
+# This directory indicates whether the pdo python virtual
+# environment has been created
+$(PDO_INSTALL_ROOT) :
+	make -C $(PDO_SOURCE_ROOT)/build environment
+	make -C $(PDO_SOURCE_ROOT)/bin install
+
+# This directory indicates whether the ccf ledger python
+# virtual environment has been created
+$(CCF_LEDGER_DIR)/lib/python3.8 :
+	mkdir -p $(CCF_LEDGER_DIR)
+	mkdir -p $(CCF_LEDGER_DIR)/workspace
+	virtualenv -p python3.8 --no-download $(PDO_HOME)/ccf
+	$(CCF_LEDGER_DIR)/bin/pip install --upgrade pip
+	$(CCF_LEDGER_DIR)/bin/pip install --upgrade -r $(CCF_BASE)/bin/requirements.txt
+	$(CCF_LEDGER_DIR)/bin/pip install ccf==$(CCF_VERSION)
+
+# -----------------------------------------------------------------
+# build it
+# -----------------------------------------------------------------
+build : build-pdo-tp
 
 build-pdo-tp : $(BLDDIR)
-	cd $(BLDDIR) && cmake .. -GNinja \
-		-DCCF_DIR=$(CCF_BASE) \
-		-DCOMPILE_TARGET=$(COMPILE_TARGET) \
-		-DCMAKE_INSTALL_PREFIX=$(CCFDSTDIR)
 	cd $(BLDDIR) && $(NINJA)
 
-clean : clean-build clean-install
+$(BLDDIR) :
+	cmake -S . -B $(BLDDIR) -GNinja \
+		-DCCF_DIR=$(CCF_BASE) \
+		-DCOMPILE_TARGET=$(CCF_COMPILE_TARGET) \
+		-DCMAKE_INSTALL_PREFIX=$(CCF_LEDGER_DIR)
+
+# -----------------------------------------------------------------
+# clean up
+# -----------------------------------------------------------------
+clean : clean-build clean-ledger
 
 clean-build:
 	rm -rf $(BLDDIR)
 
-clean-install :
-	rm -rf $(CCFDSTDIR)
+clean-ledger :
+	rm -rf $(CCF_LEDGER_DIR)
 
-environment : $(CCFDSTDIR) $(PYTHON_DIR)
+# -----------------------------------------------------------------
+# install the pdo tp library and scripts in the appropriate
+# directories; the library and bash scripts are only necessary
+# for running the ledger and go in the ledger directory; the
+# python scripts may be useful on any client so they are installed
+# in the pdo install root directory where the rest of the pdo
+# scripts are installed. future work to move the python scripts
+# to an installable wheel file
+# -----------------------------------------------------------------
 
-$(BLDDIR) :
-	@echo CREATE BUILD DIRECTORY $(BLDDIR)
-	mkdir -p $(BLDDIR)
-
-# build out the entire pdo directory tree since
-# we may use it for placement of the ledger keys
-$(CCFDSTDIR) :
-	@echo CREATE INSTALLATION DIRECTORY $(CCFDSTDIR)
-	@make -C $(PDO_SOURCE_ROOT)/build $(PDO_INSTALL_ROOT)
-	@mkdir -p $(CCFDSTDIR)
-	@mkdir -p $(WORKSPACEDIR)
-
-$(PYTHON_DIR) :
-	echo ${PYTHON_DIR}
-	virtualenv -p python3.8 --no-download $(CCFDSTDIR)
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade pip
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade setuptools
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade -r $(CCF_BASE)/bin/requirements.txt
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade ccf==1.0.19 toml
-
-install : install-pdo-tp
-
-PDO_BASH_SCRIPTS=start_ccf_network.sh stop_cchost.sh
-PDO_PYTHON_SCRIPTS=configure_ccf_network.py generate_ledger_authority.py fetch_ledger_authority.py register_enclave_attestation_verification_policy.py
+PDO_BASH_SCRIPTS = $(wildcard scripts/*.sh)
+PDO_PYTHON_SCRIPTS = $(wildcard scripts/*.py)
+
+install : install-pdo-tp install-pdo-scripts
 
 install-pdo-tp : build-pdo-tp
-	@ cd $(BLDDIR) && $(NINJA) install
-	@ cp $(addprefix scripts/,$(PDO_BASH_SCRIPTS)) $(BINDIR)
-	@ cp $(addprefix scripts/,$(PDO_PYTHON_SCRIPTS)) $(BINDIR)
-	@ make -C ${PDO_SOURCE_ROOT}/bin install
-
-.PHONY : all build build-pdo-tp
-.PHONY : clean clean-build clean-install
-.PHONY : environment
-.PHONY : install install-pdo-tp
+	cd $(BLDDIR) && $(NINJA) install
+	cp $(PDO_BASH_SCRIPTS) $(CCF_LEDGER_DIR)/bin
+
+install-pdo-scripts :
+	cp $(PDO_PYTHON_SCRIPTS) $(PDO_INSTALL_ROOT)/bin
+
+# -----------------------------------------------------------------
+.PHONY : all
+.PHONY : environment pdo-environment ledger-environment
+.PHONY : build build-pdo-tp
+.PHONY : clean clean-build clean-ledger
+.PHONY : install install-pdo-tp install-pdo-scripts
diff --git a/ledgers/ccf/scripts/start_ccf_network.sh b/ledgers/ccf/scripts/start_ccf_network.sh
@@ -17,19 +17,22 @@
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------
 source ${PDO_HOME}/bin/lib/common.sh
+
 check_pdo_runtime_env
 check_python_version
 
 # -----------------------------------------------------------------
 # Process command line arguments
 # -----------------------------------------------------------------
+F_CCF_PDO_DIR=${CCF_PDO_DIR:-${PDO_INSTALL_ROOT}}
+F_CCF_LEDGER_DIR=${CCF_LEDGER_DIR:-${PDO_HOME}/ccf}
 F_INTERFACE=${PDO_HOSTNAME:-${HOSTNAME}}
 F_PORT=6600
 
 SCRIPT_NAME=$(basename ${BASH_SOURCE[-1]} )
-USAGE='-i|--interface [hostname] -p|--port [port]'
+USAGE='-i|--interface [hostname] -p|--port [port] --pdo-dir [path] --ledger-dir [path]'
 SHORT_OPTS='i:p:'
-LONG_OPTS='interface:,port:'
+LONG_OPTS='interface:,port:,pdo-dir:,ledger-dir:'
 
 TEMP=$(getopt -o ${SHORT_OPTS} --long ${LONG_OPTS} -n "${SCRIPT_NAME}" -- "$@")
 if [ $? != 0 ] ; then echo "Usage: ${SCRIPT_NAME} ${USAGE}" >&2 ; exit 1 ; fi
@@ -39,20 +42,30 @@ while true ; do
     case "$1" in
         -i|--interface) F_INTERFACE="$2" ; shift 2 ;;
         -p|--port) F_PORT="$2" ; shift 2 ;;
+        --pdo-dir) F_CCF_PDO_DIR="$2" ; shift 2 ;;
+        --ledger-dir) F_CCF_LEDGER_DIR="$2" ; shift 2 ;;
         --help) echo "Usage: ${SCRIPT_NAME} ${USAGE}"; exit 0 ;;
     	--) shift ; break ;;
     	*) echo "Internal error!" ; exit 1 ;;
     esac
 done
 
-# the CCF sandbox script creates the python virtual environment
-# in the directory from which it is executed so we need to make
-# sure we run it in the install directory... this is generally bad
-# practice and we should probably move away from sandbox.
-cd ${PDO_HOME}/ccf
+# -----------------------------------------------------------------
+# make sure that we have a completely prepared environment
+# -----------------------------------------------------------------
+if [ ! -f ${F_CCF_PDO_DIR}/bin/activate ]; then
+    die incomplete configuration, unable to locate CCF_PDO_DIR virtual environment
+fi
+
+if [ ! -f ${F_CCF_LEDGER_DIR}/bin/activate ]; then
+    die incomplete configuration, unable to locate CCF_LEDGER_DIR virtual environment
+fi
 
-if [ -f ${PDO_HOME}/ccf/workspace/sandbox_0/node.pid ]; then
-    if ps -p $(cat ${PDO_HOME}/ccf/workspace/sandbox_0/node.pid) > /dev/null
+# -----------------------------------------------------------------
+# check to see if there is an instance already running
+# -----------------------------------------------------------------
+if [ -f ${PDO_LEDGER_DIR}/workspace/pdo_tp_0/node.pid ]; then
+    if ps -p $(<"${PDO_LEDGER_DIR}/workspace/pdo_tp_0/node.pid")
     then
         yell cchost appears to be running already
         exit -1
@@ -65,41 +78,71 @@ then
     exit -1
 fi
 
+# -----------------------------------------------------------------
+# clean up any old ledger keys
+# -----------------------------------------------------------------
 rm -f ${PDO_LEDGER_KEY_ROOT}/networkcert.pem
 rm -f ${PDO_LEDGER_KEY_ROOT}/ledger_authority_pub.pem
 
 # ensure that we use an IP address for the interface
 F_INTERFACE_ADDRESS=$(force_to_ip ${F_INTERFACE})
-
 yell start ccf network with "local://${F_INTERFACE_ADDRESS}:${F_PORT}"
 
+# -----------------------------------------------------------------
+# run the CCF script to start the network. this script must operate
+# in the python virtual environment created for it in ${F_CCF_LEDGER_DIR}
+# note that simply executing python out of the ccf ledger virtual env
+# is not sufficient since ccf code has expectations for the PATH which
+# is only changed when you activate.
+# -----------------------------------------------------------------
 say attempt to start ccf node
-${CCF_BASE}/bin/sandbox.sh \
-    --verbose \
-    --host-log-level info \
-    -p ${PDO_HOME}/ccf/lib/libpdoenc \
-    -n "local://${F_INTERFACE_ADDRESS}:${F_PORT}" \
-    -e virtual -t virtual \
-    --workspace ${PDO_HOME}/ccf/workspace \
-    --initial-node-cert-validity-days 365 \
-    --initial-service-cert-validity-days 365 &
-
-sleep 5
-
-while [ ! -f ${PDO_HOME}/ccf/workspace/sandbox_common/service_cert.pem ]; do
+
+source ${F_CCF_LEDGER_DIR}/bin/activate
+CURL_CLIENT=ON INITIAL_MEMBER_COUNT=1 \
+    ${F_CCF_LEDGER_DIR}/bin/python ${CCF_BASE}/bin/start_network.py \
+        --binary-dir ${CCF_BASE}/bin \
+        --enclave-type virtual \
+        --enclave-platform virtual \
+        --constitution ${CCF_BASE}/bin/actions.js \
+        --constitution ${CCF_BASE}/bin/validate.js \
+        --constitution ${CCF_BASE}/bin/resolve.js \
+        --constitution ${CCF_BASE}/bin/apply.js \
+        --ledger-chunk-bytes 5000000 \
+        --snapshot-tx-interval 10000 \
+        --initial-node-cert-validity-days 365 \
+        --initial-service-cert-validity-days 365 \
+        --label pdo_tp \
+        --verbose \
+        --host-log-level info \
+        --workspace ${F_CCF_LEDGER_DIR}/workspace \
+        -p ${F_CCF_LEDGER_DIR}/lib/libpdoenc \
+        -n "local://${F_INTERFACE_ADDRESS}:${F_PORT}" &
+deactivate
+
+# the ledger keys are the indicator that the service has started successfully
+while [ ! -f ${F_CCF_LEDGER_DIR}/workspace/pdo_tp_common/service_cert.pem ]; do
     say "wait for cchost to write the ledger keys"
     sleep 5
 done
 
+# -----------------------------------------------------------------
 say ledger keys written, copy the keys to PDO_LEDGER_KEY_ROOT
-cp ${PDO_HOME}/ccf/workspace/sandbox_common/service_cert.pem ${PDO_LEDGER_KEY_ROOT}/networkcert.pem
-cp ${PDO_HOME}/ccf/workspace/sandbox_common/member0_cert.pem ${PDO_LEDGER_KEY_ROOT}/memberccf_cert.pem
-cp ${PDO_HOME}/ccf/workspace/sandbox_common/member0_privk.pem ${PDO_LEDGER_KEY_ROOT}/memberccf_privk.pem
+# -----------------------------------------------------------------
+cp ${F_CCF_LEDGER_DIR}/workspace/pdo_tp_common/service_cert.pem ${PDO_LEDGER_KEY_ROOT}/networkcert.pem
+cp ${F_CCF_LEDGER_DIR}/workspace/pdo_tp_common/member0_cert.pem ${PDO_LEDGER_KEY_ROOT}/memberccf_cert.pem
+cp ${F_CCF_LEDGER_DIR}/workspace/pdo_tp_common/member0_privk.pem ${PDO_LEDGER_KEY_ROOT}/memberccf_privk.pem
 
+# -----------------------------------------------------------------
 # Generate authority needs to check to make sure that the ledger
-# is open and delay briefly if it is not open
+# is open and delay briefly if it is not open; this operation and
+# all of the following operations are ccf *client* operations and
+# require the client python virtual environment that was created
+# in ${F_CCF_PDO_DIR}
+# -----------------------------------------------------------------
+source ${F_CCF_PDO_DIR}/bin/activate
+
 say generate the ledger authority
-try ${PDO_HOME}/ccf/bin/generate_ledger_authority.py \
+try ${F_CCF_PDO_DIR}/bin/generate_ledger_authority.py \
     --logfile __screen__ --loglevel WARNING \
     --interface ${F_INTERFACE_ADDRESS} --port ${F_PORT}
 
@@ -109,15 +152,18 @@ try ${PDO_HOME}/ccf/bin/generate_ledger_authority.py \
 
 if [ "${SGX_MODE}" == "SIM" ]; then
     say set check_attestation to false in SGX SIM mode
-    try ${PDO_HOME}/ccf/bin/register_enclave_attestation_verification_policy.py \
+    try ${F_CCF_PDO_DIR}/bin/register_enclave_attestation_verification_policy.py \
         --logfile __screen__ --loglevel WARNING \
         --interface ${F_INTERFACE_ADDRESS} --port ${F_PORT}
-
 fi
 
 say save the ledger authority key
-try ${PDO_HOME}/ccf/bin/fetch_ledger_authority.py \
+try ${F_CCF_PDO_DIR}/bin/fetch_ledger_authority.py \
     --logfile __screen__ --loglevel WARNING \
     --interface ${F_INTERFACE_ADDRESS} --port ${F_PORT}
 
+deactivate
+
+# -----------------------------------------------------------------
 yell ledger URL is http://${F_INTERFACE_ADDRESS}:${F_PORT}
+# -----------------------------------------------------------------