From 01dd1a5dcf7dfa1e0e31f7ab27cf3d4d1457c239 Mon Sep 17 00:00:00 2001 From: Mic Bowman Date: Thu, 11 Jan 2024 11:47:24 -0800 Subject: [PATCH 1/2] Make the ccf ping_test.py more useful for connection debugging Several changes to ping test to make it a little more useful: * Removed the dependencies on PDO configuration including configuration files and environment variables; everything gets specified as command line parameters: ping_test.py --url ${PDO_LEDGER_URL} --cert ${PDO_LEDGER_KEY_ROOT}/networkcert.pem * Allow for "quiet" mode operation so that the result code can be used in other scripts. * Change behavior on failure to print the root cause message for an exception that occurs; CCF was hiding the useful messages Signed-off-by: Mic Bowman --- ledgers/ccf/Makefile | 2 +- ledgers/ccf/README.md | 2 +- ledgers/ccf/scripts/ping_test.py | 55 +++++++++++++++++--------------- ledgers/ccf/scripts/utils.py | 38 ---------------------- 4 files changed, 31 insertions(+), 66 deletions(-) delete mode 100644 ledgers/ccf/scripts/utils.py diff --git a/ledgers/ccf/Makefile b/ledgers/ccf/Makefile index 8159cfa4..58d24203 100644 --- a/ledgers/ccf/Makefile +++ b/ledgers/ccf/Makefile @@ -85,7 +85,7 @@ $(PYTHON_DIR) : install : install-pdo-tp PDO_BASH_SCRIPTS=start_ccf_network.sh stop_cchost.sh -PDO_PYTHON_SCRIPTS=configure_ccf_network.py generate_ledger_authority.py fetch_ledger_authority.py register_enclave_attestation_verification_policy.py utils.py +PDO_PYTHON_SCRIPTS=configure_ccf_network.py generate_ledger_authority.py fetch_ledger_authority.py register_enclave_attestation_verification_policy.py install-pdo-tp : build-pdo-tp @ cd $(BLDDIR) && $(NINJA) install diff --git a/ledgers/ccf/README.md b/ledgers/ccf/README.md index 6fc107af..1bd95af5 100644 --- a/ledgers/ccf/README.md +++ b/ledgers/ccf/README.md @@ -131,7 +131,7 @@ at the end of the test. ```bash source $PDO_HOME/ccf/bin/activate -${PDO_SOURCE_ROOT}/ledgers/ccf/scripts/ping_test.py +${PDO_SOURCE_ROOT}/ledgers/ccf/scripts/ping_test.py --url ${PDO_LEDGER_URL} --cert ${PDO_LEDGER_KEY_ROOT}/networkcert.pem ``` While invoking the test from a remote machine, be sure to 1) copy the diff --git a/ledgers/ccf/scripts/ping_test.py b/ledgers/ccf/scripts/ping_test.py index 21ccdd08..7c4cbac8 100755 --- a/ledgers/ccf/scripts/ping_test.py +++ b/ledgers/ccf/scripts/ping_test.py @@ -19,20 +19,11 @@ import os import sys import time -import toml from urllib.parse import urlparse from ccf.clients import CCFClient - -from utils import parse_ledger_url -# pick up the logger used by the rest of CCF from loguru import logger as LOG -## ----------------------------------------------------------------- -ContractHome = os.environ.get("PDO_HOME") or os.path.realpath("/opt/pdo") -CCF_Etc = os.path.join(ContractHome, "ccf", "etc") -CCF_Keys = os.environ.get("PDO_LEDGER_KEY_ROOT") or os.path.join(ContractHome, "ccf", "keys") - # ----------------------------------------------------------------- def ping_test(client, options): num_pings = options.num_pings @@ -47,17 +38,20 @@ def ping_test(client, options): total_time = end_time - start_time txn_throuput = num_pings/total_time - LOG.info("Performed {0} pings. Average txn_throuput is {1} pings per second".format(num_pings, txn_throuput)) + if options.verbose : + LOG.warning("Performed {0} pings. Average throughput is {1} pings per second".format(num_pings, txn_throuput)) # ----------------------------------------------------------------- def Main() : - parser = argparse.ArgumentParser(description='Script to enable the CCF network') - - parser.add_argument('--logfile', help='Name of the log file, __screen__ for standard output', type=str) - parser.add_argument('--loglevel', help='Logging level', default='INFO', type=str) - parser.add_argument('--user-name', help="Name of the user being added", default = "userccf", type=str) - parser.add_argument("--num-pings", help="Number of ping operations to do", default = 100, type=int) - + parser = argparse.ArgumentParser(description='Test the connection to a CCF server') + + parser.add_argument('--loglevel', help='Logging level', default='WARNING', type=str) + parser.add_argument("--num-pings", help="Number of ping operations to do", default = 1, type=int) + parser.add_argument('--url', type=str, required=True) + parser.add_argument('--cert', type=str, required=True) + group = parser.add_mutually_exclusive_group() + group.add_argument('--verbose', action='store_true', default=True) + group.add_argument('--quiet', action='store_false', dest='verbose') options = parser.parse_args() # ----------------------------------------------------------------- @@ -65,20 +59,29 @@ def Main() : LOG.add(sys.stderr, level=options.loglevel) # ----------------------------------------------------------------- - network_cert = os.path.join(CCF_Keys, "networkcert.pem") - - host, port = parse_ledger_url() + try : + (host, port) = urlparse(options.url).netloc.split(':') + except Exception as e: + if options.verbose : + LOG.error('failed to parse ledger URL: {}'.format(str(e))) + sys.exit(-1) try : - user_client = CCFClient( - host, - port, - network_cert) + user_client = CCFClient(host, port, options.cert) except Exception as e: - LOG.error('failed to connect to CCF service: {}'.format(str(e))) + if options.verbose : + LOG.error('failed to connect to CCF service: {}'.format(str(e))) sys.exit(-1) - ping_test(user_client, options) + try : + ping_test(user_client, options) + except Exception as e: + # this just lets the script get back to the original error + # that caused the execption + if options.verbose : + while e.__context__ : e = e.__context__ + LOG.error('ping test failed: {}', str(e)) + sys.exit(-1) sys.exit(0) diff --git a/ledgers/ccf/scripts/utils.py b/ledgers/ccf/scripts/utils.py deleted file mode 100644 index c411bd38..00000000 --- a/ledgers/ccf/scripts/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -utils.py -- common utility routines useful for deploying CCF based pdo ledger -""" - -import os -from urllib.parse import urlparse - -__all__ = [ - 'parse_ledger_url', -] - -def parse_ledger_url(config = None): - """Parse Ledger URL into host & port""" - - if config: - (host, port) = config["rpc-address"].split(':') - return host, port - - if os.environ.get("PDO_LEDGER_URL") is not None: - url = os.environ.get("PDO_LEDGER_URL") - (host, port) = urlparse(url).netloc.split(':') - return host, port - - raise Exception("Insufficient info to parse ledger url") From 23acb8899b0b059615fdb69e6401be477c6c638c Mon Sep 17 00:00:00 2001 From: Mic Bowman Date: Thu, 11 Jan 2024 11:53:49 -0800 Subject: [PATCH 2/2] Small fixes in the docker files * add lsof to the packages to install for ccf; the test for duplicate services was failing (non-fatally); this should fix it * add explicit setting of noproxy in start_services to mirror handling in other tools scripts. this fixes a problem when running persistent test services on localhost. Signed-off-by: Mic Bowman --- docker/pdo_ccf_base.dockerfile | 1 + docker/tools/start_services.sh | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/pdo_ccf_base.dockerfile b/docker/pdo_ccf_base.dockerfile index 2ad6469b..5eabdc31 100644 --- a/docker/pdo_ccf_base.dockerfile +++ b/docker/pdo_ccf_base.dockerfile @@ -32,6 +32,7 @@ ARG ADD_APT_PKGS= ENV DEBIAN_FRONTEND "noninteractive" RUN apt-get update \ && apt-get install -y -q \ + lsof \ python \ python3-dev \ python3-venv \ diff --git a/docker/tools/start_services.sh b/docker/tools/start_services.sh index 0c333d8c..e815fd68 100755 --- a/docker/tools/start_services.sh +++ b/docker/tools/start_services.sh @@ -26,6 +26,7 @@ SCRIPT_NAME=$(basename ${BASH_SOURCE[-1]} ) # ----------------------------------------------------------------- # Process command line arguments # ----------------------------------------------------------------- +F_COUNT=5 F_LOGLEVEL= F_MODE=build F_REGISTER=no @@ -36,8 +37,8 @@ F_LEDGER_URL= F_USAGE='-c|--count -i|--interface [hostname] -1|--ledger [url] " F_USAGE+="--loglevel [debug|info|warn] -m|--mode [build|copy|skip] -r|--register' -SHORT_OPTS='i:l:m:r' -LONG_OPTS='interface:,ledger:,loglevel:,mode:,register' +SHORT_OPTS='c:i:l:m:r' +LONG_OPTS='count:,interface:,ledger:,loglevel:,mode:,register' TEMP=$(getopt -o ${SHORT_OPTS} --long ${LONG_OPTS} -n "${SCRIPT_NAME}" -- "$@") if [ $? != 0 ] ; then echo "Usage: ${SCRIPT_NAME} ${F_USAGE}" >&2 ; exit 1 ; fi @@ -73,10 +74,11 @@ export PDO_LEDGER_ADDRESS=$(force_to_ip ${PDO_HOSTNAME}) export PDO_LEDGER_URL=${PDO_LEDGER_URL:-http://${PDO_LEDGER_ADDRESS}:6600} if [ ! -z "${F_LEDGER_URL}" ] ; then export PDO_LEDGER_URL=${F_LEDGER_URL} + export PDO_LEDGER_ADDRESS=$( echo $PDO_LEDGER_URL | awk -F[/:] '{print $4}' ) fi -export no_proxy=$PDO_HOSTNAME,$no_proxy -export NO_PROXY=$PDO_HOSTNAME,$NO_PROXY +export no_proxy=$PDO_HOSTNAME,$PDO_LEDGER_ADDRESS,$no_proxy +export NO_PROXY=$PDO_HOSTNAME,$PDO_LEDGER_ADDRESS,$NO_PROXY # ----------------------------------------------------------------- # Handle the configuration of the services @@ -84,7 +86,7 @@ export NO_PROXY=$PDO_HOSTNAME,$NO_PROXY if [ "${F_MODE,,}" == "build" ]; then yell configure services for host $PDO_HOSTNAME and ledger $PDO_LEDGER_URL try ${PDO_INSTALL_ROOT}/bin/pdo-configure-services -t ${PDO_SOURCE_ROOT}/build/template -o ${PDO_HOME}\ - --count 5 5 5 + --count ${F_COUNT} ${F_COUNT} ${F_COUNT} elif [ "${F_MODE,,}" == "copy" ]; then yell copy the configuration from xfer/services/etc and xfer/services/keys try mkdir -p ${PDO_HOME}/etc ${PDO_HOME}/keys