Several updates to fix and simplify support for SGX

A new docker build target is added to build the services to run in SGX. The resulting image is clearly typed/tagged as pdo_service_sgx to distinguish it from non-SGX builds. Also, base images are now SGX-MODE agnostic. 3 new scripts are added for handling SGX keys in docker builds. First, IAS certificates are retrieved by the host (not by the docker container) and transferred to the container through the "repository" before the build starts. Second, the enclave signing key is similarly checked for on (or directly passed by) the host, and transferred to the container through the "repository" before the build starts. Third, at run time, the necessary SGX keys are transferred from the host to the container through the xfer folder. The testing support is updated to pass the required SGX volumes and devices. Also, current small hacks to work behind a proxy are extended to support also the SGX-based image behind a proxy. The environment variables related to SPID, SPID_API_KEY and proxy are removed. All the material is passed to the build or at runtime through files, and proxy configuration happens through docker config settings (as usual) -- without forcing specific configurations, except for the small hack above. As a result the eservice configuration also gets simplified by reducing and merging the enclave.toml with the eservice.toml. Additional clean up will follow in later icommits. The documentation is updated to provide details on using and testing with SGX-based builds. Signed-off-by: Bruno Vavala <[email protected]>
hyperledger-labs · Apr 9, 2024 · 22f4ec9 · 22f4ec9
1 parent b3bbd5f
commit 22f4ec9
Show file tree

Hide file tree

Showing 36 changed files with 528 additions and 154 deletions.
diff --git a/bin/lib/common_service.sh b/bin/lib/common_service.sh
@@ -125,7 +125,7 @@ service_start() {
 			     ${F_LOGLEVEL} 2> $EFILE > $OFILE &
             echo $! > ${F_LOGDIR}/${IDENTITY}.pid
 	else
-            ${F_SERVICE_CMD} --identity ${IDENTITY} --config ${IDENTITY}.toml enclave.toml --config-dir ${F_CONFDIR} \
+            ${F_SERVICE_CMD} --identity ${IDENTITY} --config ${IDENTITY}.toml --config-dir ${F_CONFDIR} \
                              ${F_LEDGERURL} ${F_LOGLEVEL} 2> $EFILE > $OFILE &
             echo $! > ${F_LOGDIR}/${IDENTITY}.pid
 	fi

diff --git a/build/Makefile b/build/Makefile
@@ -21,8 +21,8 @@ ifndef PDO_INSTALL_ROOT
 $(error Incomplete configuration, PDO_INSTALL_ROOT is not defined)
 endif
 
-ifndef PDO_ENCLAVE_CODE_SIGN_PEM
-$(error Incomplete configuration, PDO_ENCLAVE_CODE_SIGN_PEM is not defined)
+ifndef PDO_SGX_KEY_ROOT
+$(error Incomplete configuration, PDO_SGX_KEY_ROOT is not defined)
 endif
 
 ifndef SGX_MODE
@@ -43,7 +43,6 @@ SRCDIR ?= $(abspath $(SCRIPTDIR)/..)
 
 BUILD = $(abspath $(SCRIPTDIR)/__tools__/build.sh)
 VERIFY_PRE_BUILD = $(abspath $(SCRIPTDIR)/__tools__/verify-pre-build.sh)
-VERIFY_PRE_CONF = $(abspath $(SCRIPTDIR)/__tools__/verify-pre-conf.sh)
 CLEAN = $(abspath $(SCRIPTDIR)/__tools__/clean.sh)
 TESTSCRIPT = $(abspath $(SCRIPTDIR)/__tools__/run-tests.sh)
 BENCHMARKSCRIPT = $(abspath $(SCRIPTDIR)/__tools__/run-benchmarks.sh)
@@ -95,9 +94,6 @@ $(DSTDIR) :
 verify-pre-build :
 	$(VERIFY_PRE_BUILD)
 
-verify-pre-config :
-	$(VERIFY_PRE_CONF)
-
 build : $(PYTHON_DIR)
 	. $(abspath $(DSTDIR)/bin/activate) && $(BUILD)
 
@@ -106,16 +102,16 @@ verified-build : verify-pre-build
 
 rebuild : clean-build build $(CONDITIONAL_REGISTER_TARGET)
 
-system-keys : ${PDO_ENCLAVE_CODE_SIGN_PEM}
+system-keys : ${PDO_SGX_KEY_ROOT}/enclave_code_sign.pem
 
-${PDO_ENCLAVE_CODE_SIGN_PEM} :
-	openssl genrsa -3 -out ${PDO_ENCLAVE_CODE_SIGN_PEM} 3072
+${PDO_SGX_KEY_ROOT}/enclave_code_sign.pem :
+	openssl genrsa -3 -out ${PDO_SGX_KEY_ROOT}/enclave_code_sign.pem 3072
 
 # SERVICES_COUNT is the number of services of each type to create
 # First value is the number of eservices, then pservices, then
 # sservices, 5 of each is the default
 SERVICES_COUNT ?= 5 5 5
-SERVICES_CONF_TEMPLATES = $(addprefix $(SCRIPTDIR)/template/, eservice.toml pservice.toml enclave.toml)
+SERVICES_CONF_TEMPLATES = $(addprefix $(SCRIPTDIR)/template/, eservice.toml pservice.toml)
 SERVICES_CONF_TARGET = $(DSTDIR)/opt/pdo/.services_configured
 
 $(SERVICES_CONF_TARGET) : $(PYTHON_DIR) $(SERVICE_CONF_TEMPLATES)
@@ -125,18 +121,18 @@ $(SERVICES_CONF_TARGET) : $(PYTHON_DIR) $(SERVICE_CONF_TEMPLATES)
 
 config-services : $(SERVICES_CONF_TARGET)
 
-verified-config : verify-pre-config
+verified-config :
 	${MAKE} config
 
 config : config-services config-client
 
-force-config : verify-pre-config
+force-config :
 	- rm -f $(SERVICES_CONF_TARGET) $(CLIENT_CONF_TARGET)
 	${MAKE} config
 
 ifeq ($(SGX_MODE),HW)
 register : $(PYTHON_DIR)
-	@ echo registering enclave and IAS public key on the ledger
+	@ echo Register the enclave registration policy on the ledger
 	. $(abspath $(DSTDIR)/bin/activate) && $(SRCDIR)/eservice/bin/register-with-ledger.sh
 
 else
@@ -171,6 +167,6 @@ benchmark : $(PYTHON_DIR)
 .PHONY : all environment register system-keys
 .PHONY : build rebuild verified-build verify-pre-build
 .PHONY : clean clean-build clean-install
-.PHONY : config config-services force-config verified-config verify-pre-config
+.PHONY : config config-services force-config verified-config
 .PHONY : client build-client config-client
 .PHONY : benchmark test
diff --git a/build/__tools__/verify-pre-build.sh b/build/__tools__/verify-pre-build.sh
@@ -41,7 +41,6 @@ yell --------------- CONFIG AND ENVIRONMENT PRE-BUILD CHECK ---------------
 
 : "${PDO_INSTALL_ROOT:-$(warn Missing environment variable PDO_INSTALL_ROOT)}"
 : "${PDO_HOME:-$(warn Missing environment variable PDO_HOME)}"
-: "${PDO_ENCLAVE_CODE_SIGN_PEM:-$(warn Missing environment variable PDO_ENCLAVE_CODE_SIGN_PEM)}"
 ([ ! -z "${SGX_SSL}" ] && [ -f ${SGX_SSL}/include/openssl/err.h ] ) || warn "Missing or invalid environment variable SGX_SSL"
 ([ ! -z "${SGX_SDK}" ] && [ -f ${SGX_SDK}/include/sgx.h ] ) || warn "Missing or invalid environment variable SGX_SDK"
 : "${SGX_MODE:-$(warn Missing environment variable SGX_MODE, set it to HW or SIM)}"
@@ -59,8 +58,4 @@ if [ ! -d "${PDO_INSTALL_ROOT}" ]; then
     warn "PDO_INSTALL_ROOT directory does not exist"
 fi
 
-if [ ! -f "${PDO_ENCLAVE_CODE_SIGN_PEM}" ]; then
-    warn "PDO_ENCLAVE_CODE_SIGN_PEM file does not exist"
-fi
-
 exit $F_VERIFIED
diff --git a/build/cmake/SGX.cmake b/build/cmake/SGX.cmake
@@ -16,10 +16,10 @@
 # Environment Variables
 ################################################################################
 
-IF (NOT DEFINED ENV{PDO_ENCLAVE_CODE_SIGN_PEM})
-  MESSAGE(FATAL_ERROR "PDO_ENCLAVE_CODE_SIGN_PEM not defined")
+IF (NOT DEFINED ENV{PDO_SGX_KEY_ROOT})
+  MESSAGE(FATAL_ERROR "PDO_SGX_KEY_ROOT not defined")
 ENDIF()
-SET(PDO_ENCLAVE_CODE_SIGN_PEM "$ENV{PDO_ENCLAVE_CODE_SIGN_PEM}")
+SET(PDO_SGX_KEY_ROOT "$ENV{PDO_SGX_KEY_ROOT}")
 
 IF (NOT DEFINED ENV{SGX_MODE})
   MESSAGE(FATAL_ERROR "SGX_MODE not defined")

diff --git a/build/template/eservice.toml b/build/template/eservice.toml
@@ -72,3 +72,20 @@ DataPath = "${data}"
 # BaseName is the root of the name used to store data
 # about the enclave. A 'enc' extension will be added
 BaseName = "${identity}"
+
+# --------------------------------------------------
+# EnclaveModule -- configuration of the SGX contract enclave
+# --------------------------------------------------
+[EnclaveModule]
+
+# Number of available enclave workers to service requests
+NumberOfEnclaves = '7'
+
+# ias_url is the URL of the Intel Attestation Service (IAS) server.  The
+# example server is for debug enclaves only,
+# the production url is without the trailing '/dev'
+ias_url = 'https://api.trustedservices.intel.com/sgx/dev'
+
+# sgx key root folder configuration
+sgx_key_root = "${sgx_key_root}"
+
diff --git a/build/template/pservice.toml b/build/template/pservice.toml
@@ -68,3 +68,20 @@ DataPath = "${data}"
 # BaseName is the root of the name used to store data
 # about the enclave. A 'enc' extension will be added
 BaseName = "${identity}"
+
+# --------------------------------------------------
+# EnclaveModule -- configuration of the SGX contract enclave
+# --------------------------------------------------
+[EnclaveModule]
+
+# Number of available enclave workers to service requests
+NumberOfEnclaves = '7'
+
+# ias_url is the URL of the Intel Attestation Service (IAS) server.  The
+# example server is for debug enclaves only,
+# the production url is without the trailing '/dev'
+ias_url = 'https://api.trustedservices.intel.com/sgx/dev'
+
+# sgx key root folder configuration
+sgx_key_root = "${sgx_key_root}"
+
diff --git a/common/tests/crypto/trusted/enclave/CMakeLists.txt b/common/tests/crypto/trusted/enclave/CMakeLists.txt
@@ -41,4 +41,4 @@ TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${COMMON_TRUSTED_LIBS})
 TARGET_LINK_LIBRARIES(${PROJECT_NAME} -Wl,--end-group)
 
 SGX_PREPARE_TRUSTED_LINK(${PROJECT_NAME})
-SGX_SIGN_ENCLAVE(${PROJECT_NAME} ${PDO_ENCLAVE_CODE_SIGN_PEM} ${ENCLAVE_CONFIG})
+SGX_SIGN_ENCLAVE(${PROJECT_NAME} ${PDO_SGX_KEY_ROOT}/enclave_code_sign.pem ${ENCLAVE_CONFIG})
diff --git a/docker/Makefile b/docker/Makefile
@@ -50,7 +50,9 @@ DOCKER_BUILDARGS += --build-arg UID=$(PDO_USER_UID)
 DOCKER_BUILDARGS += --build-arg GID=$(PDO_GROUP_UID)
 DOCKER_ARGS = $(DOCKER_BUILDARGS)
 
-IMAGES=base client services_base services ccf_base ccf
+IAS_CERTIFICATES=$(DOCKER_DIR)/repository/common/crypto/verify_ias_report/ias-certificates.txt
+
+IMAGES=base client services_base services services_sgx ccf_base ccf
 
 # for the most part this is just used to force rebuild when the
 # PDO repository has changed
@@ -71,6 +73,13 @@ build_% : repository
 		--tag pdo_$*:$(PDO_VERSION) \
 		--file '$(DOCKER_DIR)'/pdo_$*.dockerfile .
 
+sgx_build_services : $(IAS_CERTIFICATES) repository build_services_base
+	docker build $(DOCKER_ARGS) \
+		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--build-arg SGX_MODE=HW \
+		--tag pdo_services_sgx:$(PDO_VERSION) \
+		--file $(DOCKER_DIR)/pdo_services.dockerfile .
+
 # docker build dependencies
 build_client: build_base
 build_services: build_services_base
@@ -104,6 +113,11 @@ stop_services :
 stop_client :
 	- docker rm -f client_container
 
+$(IAS_CERTIFICATES) : repository
+	# the script prepares the certificates from the source repo
+	# and moves only the necessary artifacts to the destination repo (absolute path required)
+	$(DOCKER_DIR)/tools/prepare_ias_certificates.sh "$(PDO_SOURCE_ROOT)" $(DOCKER_DIR)/$<
+
 # -----------------------------------------------------------------
 # We need a repository with the source for the branch we are going
 # to build. In theory this could just be a copy of the local source
@@ -112,8 +126,14 @@ stop_client :
 # performance requirements are relatively low.
 # -----------------------------------------------------------------
 repository :
+	# clone the repo
 	git clone --single-branch --branch $(PDO_BRANCH) --recurse-submodules '$(PDO_REPO)' repository
 
+	# Prepare enclave signing key (if any, this goes in the repo itself).
+	# This is effective only in HW mode builds.
+	# PDO_SGX_KEY_ROOT is an optional parameter (either set in environment or empty)
+	$(DOCKER_DIR)/tools/copy_enclave_signing_key.sh "$(PDO_SOURCE_ROOT)" repository "${PDO_SGX_KEY_ROOT}"
+
 clean_repository :
 	rm -rf repository
 
@@ -130,12 +150,35 @@ TEST_FILES += -f services_base.yaml
 TEST_FILES += -f ccf_base.yaml
 TEST_FILES += -f test.yaml
 
+TEST_SGX_FILES = ${TEST_FILES}
+TEST_SGX_FILES += -f test-sgx.yaml
+
+SGX_DEVICE_PATH=$(shell if [ -e "/dev/isgx" ]; \
+				then echo "/dev/isgx"; \
+				elif [ -e "/dev/sgx/enclave" ]; \
+					then echo "/dev/sgx/enclave"; \
+				else echo "ERROR: NO SGX DEVICE FOUND"; \
+				fi)
+
+DOCKER_COMPOSE_SGX := env SGX_DEVICE_PATH=${SGX_DEVICE_PATH} docker-compose
+
 build_test : repository build_services build_ccf build_client
 
 test : clean_config clean_repository build_test stop_all
 	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) up --abort-on-container-exit
 	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) down
 
+sgx_build_test : repository sgx_build_services build_ccf build_client
+
+sgx_keys :
+	# Prepare sgx keys.
+	# PDO_SGX_KEY_ROOT is an optional parameter (either set in environment or empty)
+	$(DOCKER_DIR)/tools/copy_sgx_keys.sh "$(PDO_SOURCE_ROOT)" "$(DOCKER_DIR)" "${PDO_SGX_KEY_ROOT}"
+
+sgx_test : clean_config clean_repository sgx_build_test stop_all sgx_keys
+	PDO_VERSION=$(PDO_VERSION) $(DOCKER_COMPOSE_SGX) $(TEST_SGX_FILES) up --abort-on-container-exit
+	PDO_VERSION=$(PDO_VERSION) $(DOCKER_COMPOSE_SGX) $(TEST_SGX_FILES) down
+
 # -----------------------------------------------------------------
 # Cleaning is a bit interesting because the containers don't go away
 # unless they are told to very nicely. Until they go away they hold onto
@@ -151,12 +194,16 @@ clean_images : $(addprefix clean_,$(IMAGES))
 clean_config :
 	rm -f '$(DOCKER_DIR)'/xfer/ccf/keys/*.pem '$(DOCKER_DIR)'/xfer/ccf/etc/*.toml
 	rm -f '$(DOCKER_DIR)'/xfer/services/keys/*.pem '$(DOCKER_DIR)'/xfer/services/etc/*.toml
+	rm -f '$(DOCKER_DIR)'/xfer/services/keys/sgx/*.pem '$(DOCKER_DIR)'/xfer/services/keys/sgx/*.txt
 	rm -f '$(DOCKER_DIR)'/xfer/services/etc/site.psh
+	# clean the artifacts of the prepare_ias_certificates target
+	rm -f $(IAS_CERTIFICATES)
 
 clean : clean_images clean_config clean_repository
 
 .PHONY: clean clean_images clean_config clean_repository
 .PHONY: build_test test
+.PHONY: sgx_build_test sgx_test sgx_keys
 .PHONY: run_ccf run_client run_services
 
 # -----------------------------------------------------------------

diff --git a/docker/README.md b/docker/README.md
@@ -79,6 +79,17 @@ as services in detached mode. The last for the client will run an
 interactive shell in the client container. See below for information
 on how to use the client container.
 
+### Build for SGX ###
+
+For the contract enclave to run in SGX hardware mode, the services
+image must be built using the following target:
+```bash
+    make sgx_build_services
+```
+This will create the `pdo_service_sgx` image. Inside the image,
+the `SGX_MODE=HW` environment variable further indicates that the
+service were built to run in SGX.
+
 <!--- xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --->
 ## Pattern: Local Development in a Container ##
 
@@ -246,6 +257,30 @@ with the PDO tool `pdo-configure-services`.
         --name ${USER}_services_container pdo_services --mode copy
 ```
 
+#### PDO Services Deployment Using SGX ####
+
+There are a few _additional_ considerations when using the services with SGX.
+
+Before starting the container, make sure that the SGX collateral is available
+as described [here](../docs/install).
+
+Also, recall that the attestation policy on the ledger has to be set once by the
+first eservice of a ledger consortium member. Hence, the first service container
+that is deputed to perform such registration must be instructed to do so.
+```bash
+    docker run -v $(SCRIPT_DIR)/xfer/:/project/pdo/xfer --network host \
+        -v <host aesmd socket>:/var/run/aesmd --device=<host SGX device>:/dev/sgx/enclave \
+        --name ${USER}_services_container pdo_services_sgx --register
+```
+This updated command allows to trigger the registration step right before
+starting the services. The policy registration must happen before enclaves are
+registered (or any enclave registration will fail).
+
+Finally, the _same_ SGX collateral must be made available to all service containers.
+At enclave registration time, this will allow the eservice to generate the right
+quote (and attestation verification report) that meets the attestation policy
+originally registered with the PDO Transaction Processor.
+
 ### PDO Client Deployment ###
 
 The client image creates an interactive environment for connecting

diff --git a/docker/pdo_services.dockerfile b/docker/pdo_services.dockerfile
@@ -27,6 +27,9 @@ FROM pdo_services_base:${PDO_VERSION}
 # -----------------------------------------------------------------
 ARG REBUILD=0
 
+ARG SGX_MODE=SIM
+ENV SGX_MODE $SGX_MODE
+
 ARG PDO_DEBUG_BUILD=0
 ENV PDO_DEBUG_BUILD=${PDO_DEBUG_BUILD}
 

diff --git a/docker/pdo_services_base.dockerfile b/docker/pdo_services_base.dockerfile
@@ -24,9 +24,6 @@ ARG SGX=2.22
 ARG OPENSSL=3.0.12
 ARG SGXSSL=3.0_Rev1
 
-ARG SGX_MODE=SIM
-ENV SGX_MODE $SGX_MODE
-
 RUN echo "deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu ${UBUNTU_NAME} main" >> /etc/apt/sources.list \
  && wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add - \
  && apt-get update \
@@ -71,17 +68,16 @@ ENV PATH="/opt/intel/sgxsdk.extras/external/toolset/ubuntu${UBUNTU_VERSION}:${PA
 
 # -----------------------------------------------------------------
 # SGXSSL
-# Note that we build sgxssl with SIM mode; the SGX_MODE only changes
-# the mode for running tests and we do not want the tests run in HW
-# mode
+# Note that the SGX_MODE variable only determines the mode for
+# running tests. We do not want the tests to run in HW mode here.
+# This allows us to keep this image mode-agnostic.
 # -----------------------------------------------------------------
 WORKDIR /tmp
 RUN . /opt/intel/sgxsdk/environment \
     && git clone --depth 1 --branch ${SGXSSL} 'https://github.com/intel/intel-sgx-ssl.git' \
     && wget -q -P /tmp/intel-sgx-ssl/openssl_source https://www.openssl.org/source/openssl-${OPENSSL}.tar.gz \
     && cd /tmp/intel-sgx-ssl/Linux \
-    && if [ $SGX_MODE = SIM ] ; then SKIP_INTELCPU_CHECK=TRUE ; else SKIP_INTELCPU_CHECK=FALSE ; fi \
-    && bash -c "make SKIP_INTELCPU_CHECK=$SKIP_INTELCPU_CHECK SGX_MODE=$SGX_MODE NO_THREADS=1 DESTDIR=/opt/intel/sgxssl VERBOSE=0 all &> /dev/null" \
+    && bash -c "make SKIP_INTELCPU_CHECK=TRUE SGX_MODE=SIM NO_THREADS=1 DESTDIR=/opt/intel/sgxssl VERBOSE=0 all &> /dev/null" \
     && make install \
     && make clean \
     && rm -rf /tmp/intel-sgx-ssl

diff --git a/docker/test-sgx.yaml b/docker/test-sgx.yaml
@@ -0,0 +1,33 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ------------------------------------------------------------------------------
+version: "3.4"
+
+services:
+  ccf_container:
+    environment:
+      - SGX_MODE=HW
+
+  services_container:
+    environment:
+      # the PDO_FORCE_IAS_PROXY is a small hack that is used to force IAS connections
+      # through a proxy when one such proxy must be used.
+      # If the variable is defined in the host environment, it is propagated to the containers.
+      - PDO_FORCE_IAS_PROXY=${PDO_FORCE_IAS_PROXY:-false}
+    image: pdo_services_sgx:${PDO_VERSION:-latest}
+    volumes:
+      - /var/run/aesmd:/var/run/aesmd
+    devices:
+      - ${SGX_DEVICE_PATH:-/dev/isgx}:${SGX_DEVICE_PATH:-/dev/isgx}
+