From c9cf57926d4befd544e4f4cdd6f20d1c36a9f44b Mon Sep 17 00:00:00 2001
From: Mic Bowman <mic.bowman@intel.com>
Date: Wed, 14 Feb 2024 09:55:00 -0800
Subject: [PATCH 1/3] Simplify build and install for CCF

Several changes to improve the performance (and remove dependencies)
for CCF startup. Many of the changes are centered in the ccf Makefile
to separate the creation of the ccf clients (like the ping test,
ledger authority retrieval and policy scripts that might reasonably be
executed from anywhere) from the environment needed for a ccf service
node (the script to start a ccf network).

The big win from the separation is that we can remove dependency on
the sandbox script & pre-install all of the python package
dependencies independently from the execution of the ccf nodes. The
result is that the ledger containers start in a couple seconds instead
of several minutes.

Note that a future PR will package the ccf client scripts separately
and install them with other PDO packages so they are available on all
PDO installations.

Signed-off-by: Mic Bowman <mic.bowman@intel.com>
---
 docker/tools/build_ccf.sh                |   3 +
 docker/tools/environment.sh              |   9 +-
 docker/tools/run_ccf_tests.sh            |   7 +-
 docker/tools/start_ccf.sh                |   6 +-
 ledgers/ccf/Makefile                     | 127 ++++++++++++++---------
 ledgers/ccf/scripts/start_ccf_network.sh | 108 +++++++++++++------
 6 files changed, 173 insertions(+), 87 deletions(-)

diff --git a/docker/tools/build_ccf.sh b/docker/tools/build_ccf.sh
index c298eef6..5b4e61af 100755
--- a/docker/tools/build_ccf.sh
+++ b/docker/tools/build_ccf.sh
@@ -19,5 +19,8 @@ source /project/pdo/tools/environment.sh
 export PDO_HOSTNAME=
 export PDO_LEDGER_URL=
 
+# -----------------------------------------------------------------
+# set up the basic structure in ${PDO_INSTALL_ROOT}
+# -----------------------------------------------------------------
 make -C ${PDO_SOURCE_ROOT}/ledgers/ccf environment
 make -C ${PDO_SOURCE_ROOT}/ledgers/ccf install
diff --git a/docker/tools/environment.sh b/docker/tools/environment.sh
index 41313b96..c0850b12 100755
--- a/docker/tools/environment.sh
+++ b/docker/tools/environment.sh
@@ -42,7 +42,6 @@ else
     export PDO_DEFAULT_SIGCURVE=SECP256K1
 fi
 
-export CCF_BASE=/opt/ccf_virtual
 export XFER_DIR=${XFER_DIR:-/project/pdo/xfer}
 
 # if the container is running HW mode, then we will grab the
@@ -62,3 +61,11 @@ export PDO_ENCLAVE_CODE_SIGN_PEM=${PDO_SGX_KEY_ROOT}/enclave_code_sign.pem
 # they are not used at build or run time
 export PDO_SPID="$(cat ${PDO_SGX_KEY_ROOT}/sgx_spid.txt)"
 export PDO_SPID_API_KEY="$(cat ${PDO_SGX_KEY_ROOT}/sgx_spid_api_key.txt)"
+
+# set up the ccf directories, ccf_base is where the ccf
+# core is installed, ccf_pdo_dir is where the pdo tp
+# components will be installed, and ccf_ledger_dir is
+# where the ccf python virtual environment will be built
+export CCF_BASE=/opt/ccf_virtual
+export CCF_PDO_DIR=${PDO_INSTALL_ROOT}
+export CCF_LEDGER_DIR=${PDO_HOME}/ccf
diff --git a/docker/tools/run_ccf_tests.sh b/docker/tools/run_ccf_tests.sh
index b8d36f33..ff537d90 100755
--- a/docker/tools/run_ccf_tests.sh
+++ b/docker/tools/run_ccf_tests.sh
@@ -34,9 +34,10 @@ mkdir -p ${PDO_LEDGER_KEY_ROOT}
 # -----------------------------------------------------------------
 yell start the ccf service
 # -----------------------------------------------------------------
-. ${PDO_HOME}/ccf/bin/activate
-yell ${PDO_HOME}/ccf/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}
-try ${PDO_HOME}/ccf/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}
+yell ${CCF_LEDGER_DIR}/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}  \
+    --pdo-dir ${CCF_PDO_DIR} --ledger-dir ${CCF_LEDGER_DIR}
+try ${CCF_LEDGER_DIR}/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}  \
+    --pdo-dir ${CCF_PDO_DIR} --ledger-dir ${CCF_LEDGER_DIR}
 
 # -----------------------------------------------------------------
 yell copy the ledger keys
diff --git a/docker/tools/start_ccf.sh b/docker/tools/start_ccf.sh
index dc15ff08..a2589d01 100755
--- a/docker/tools/start_ccf.sh
+++ b/docker/tools/start_ccf.sh
@@ -79,9 +79,11 @@ mkdir -p ${PDO_LEDGER_KEY_ROOT}
 # -----------------------------------------------------------------
 say start the ccf network
 # -----------------------------------------------------------------
-. ${PDO_HOME}/ccf/bin/activate
 if [ ${F_NETWORK_MODE} == "start" ] ; then
-    try ${PDO_HOME}/ccf/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}
+    yell ${CCF_LEDGER_DIR}/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}  \
+        --pdo-dir ${CCF_PDO_DIR} --ledger-dir ${CCF_LEDGER_DIR}
+    try ${CCF_LEDGER_DIR}/bin/start_ccf_network.sh -i ${PDO_LEDGER_ADDRESS}  \
+        --pdo-dir ${CCF_PDO_DIR} --ledger-dir ${CCF_LEDGER_DIR}
 elif [ ${F_NETWORK_MODE} == "join" ] ; then
     die "joining a network is not yet supported"
 else
diff --git a/ledgers/ccf/Makefile b/ledgers/ccf/Makefile
index 58d24203..9c970403 100644
--- a/ledgers/ccf/Makefile
+++ b/ledgers/ccf/Makefile
@@ -28,72 +28,99 @@ ifndef PDO_SOURCE_ROOT
 $(error Incomplete configuration, PDO_SOURCE_ROOT is not defined)
 endif
 
+CCF_COMPILE_TARGET ?= virtual
+CCF_VERSION ?= 4.0.1
+CCF_LEDGER_DIR ?= $(PDO_INSTALL_ROOT)/opt/pdo/ccf
+
 NINJA ?= ninja $(NINJA_OPTIONS)
 
 SCRIPTDIR ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-CCFDSTDIR ?= $(PDO_INSTALL_ROOT)/opt/pdo/ccf
-SRCDIR ?= $(PDO_SOURCE_ROOT)
-WORKSPACEDIR := $(CCFDSTDIR)/workspace
-BINDIR := $(CCFDSTDIR)/bin/
 BLDDIR := $(SCRIPTDIR)/build
 
-PYTHON_DIR=$(CCFDSTDIR)/lib/python3.8
-
-COMPILE_TARGET=virtual
-
 all: environment install
 
-build: build-pdo-tp
+# -----------------------------------------------------------------
+# There are two environments that need to be created: the PDO
+# environment and the ledger environment. This is necessitated by
+# the incompatibilities between the client requirements (which requires
+# ccf 1.0.19 which is the last python package with the ccf client modules)
+# and the ledger requirements (which installs the python modules from
+# our current verson of CCF). The client environment may be useful on
+# any pdo installation (e.g. ledger ping test makes sense on any client)
+# while the ledger environment is only useful where ccf nodes are run
+# -----------------------------------------------------------------
+environment: pdo-environment ledger-environment
+
+pdo-environment : $(PDO_INSTALL_ROOT)
+
+ledger-environment : $(CCF_LEDGER_DIR)/lib/python3.8
+
+# This directory indicates whether the pdo python virtual
+# environment has been created
+$(PDO_INSTALL_ROOT) :
+	make -C $(PDO_SOURCE_ROOT)/build environment
+	make -C $(PDO_SOURCE_ROOT)/bin install
+
+# This directory indicates whether the ccf ledger python
+# virtual environment has been created
+$(CCF_LEDGER_DIR)/lib/python3.8 :
+	mkdir -p $(CCF_LEDGER_DIR)
+	mkdir -p $(CCF_LEDGER_DIR)/workspace
+	virtualenv -p python3.8 --no-download $(PDO_HOME)/ccf
+	$(CCF_LEDGER_DIR)/bin/pip install --upgrade pip
+	$(CCF_LEDGER_DIR)/bin/pip install --upgrade -r $(CCF_BASE)/bin/requirements.txt
+	$(CCF_LEDGER_DIR)/bin/pip install ccf==$(CCF_VERSION)
+
+# -----------------------------------------------------------------
+# build it
+# -----------------------------------------------------------------
+build : build-pdo-tp
 
 build-pdo-tp : $(BLDDIR)
-	cd $(BLDDIR) && cmake .. -GNinja \
-		-DCCF_DIR=$(CCF_BASE) \
-		-DCOMPILE_TARGET=$(COMPILE_TARGET) \
-		-DCMAKE_INSTALL_PREFIX=$(CCFDSTDIR)
 	cd $(BLDDIR) && $(NINJA)
 
-clean : clean-build clean-install
+$(BLDDIR) :
+	cmake -S . -B $(BLDDIR) -GNinja \
+		-DCCF_DIR=$(CCF_BASE) \
+		-DCOMPILE_TARGET=$(CCF_COMPILE_TARGET) \
+		-DCMAKE_INSTALL_PREFIX=$(CCF_LEDGER_DIR)
+
+# -----------------------------------------------------------------
+# clean up
+# -----------------------------------------------------------------
+clean : clean-build clean-ledger
 
 clean-build:
 	rm -rf $(BLDDIR)
 
-clean-install :
-	rm -rf $(CCFDSTDIR)
+clean-ledger :
+	rm -rf $(CCF_LEDGER_DIR)
 
-environment : $(CCFDSTDIR) $(PYTHON_DIR)
+# -----------------------------------------------------------------
+# install the pdo tp library and scripts in the appropriate
+# directories; the library and bash scripts are only necessary
+# for running the ledger and go in the ledger directory; the
+# python scripts may be useful on any client so they are installed
+# in the pdo install root directory where the rest of the pdo
+# scripts are installed. future work to move the python scripts
+# to an installable wheel file
+# -----------------------------------------------------------------
 
-$(BLDDIR) :
-	@echo CREATE BUILD DIRECTORY $(BLDDIR)
-	mkdir -p $(BLDDIR)
-
-# build out the entire pdo directory tree since
-# we may use it for placement of the ledger keys
-$(CCFDSTDIR) :
-	@echo CREATE INSTALLATION DIRECTORY $(CCFDSTDIR)
-	@make -C $(PDO_SOURCE_ROOT)/build $(PDO_INSTALL_ROOT)
-	@mkdir -p $(CCFDSTDIR)
-	@mkdir -p $(WORKSPACEDIR)
-
-$(PYTHON_DIR) :
-	echo ${PYTHON_DIR}
-	virtualenv -p python3.8 --no-download $(CCFDSTDIR)
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade pip
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade setuptools
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade -r $(CCF_BASE)/bin/requirements.txt
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade ccf==1.0.19 toml
-
-install : install-pdo-tp
-
-PDO_BASH_SCRIPTS=start_ccf_network.sh stop_cchost.sh
-PDO_PYTHON_SCRIPTS=configure_ccf_network.py generate_ledger_authority.py fetch_ledger_authority.py register_enclave_attestation_verification_policy.py
+PDO_BASH_SCRIPTS = $(wildcard scripts/*.sh)
+PDO_PYTHON_SCRIPTS = $(wildcard scripts/*.py)
+
+install : install-pdo-tp install-pdo-scripts
 
 install-pdo-tp : build-pdo-tp
-	@ cd $(BLDDIR) && $(NINJA) install
-	@ cp $(addprefix scripts/,$(PDO_BASH_SCRIPTS)) $(BINDIR)
-	@ cp $(addprefix scripts/,$(PDO_PYTHON_SCRIPTS)) $(BINDIR)
-	@ make -C ${PDO_SOURCE_ROOT}/bin install
-
-.PHONY : all build build-pdo-tp
-.PHONY : clean clean-build clean-install
-.PHONY : environment
-.PHONY : install install-pdo-tp
+	cd $(BLDDIR) && $(NINJA) install
+	cp $(PDO_BASH_SCRIPTS) $(CCF_LEDGER_DIR)/bin
+
+install-pdo-scripts :
+	cp $(PDO_PYTHON_SCRIPTS) $(PDO_INSTALL_ROOT)/bin
+
+# -----------------------------------------------------------------
+.PHONY : all
+.PHONY : environment pdo-environment ledger-environment
+.PHONY : build build-pdo-tp
+.PHONY : clean clean-build clean-ledger
+.PHONY : install install-pdo-tp install-pdo-scripts
diff --git a/ledgers/ccf/scripts/start_ccf_network.sh b/ledgers/ccf/scripts/start_ccf_network.sh
index e9352fca..a156b742 100755
--- a/ledgers/ccf/scripts/start_ccf_network.sh
+++ b/ledgers/ccf/scripts/start_ccf_network.sh
@@ -17,19 +17,22 @@
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------
 source ${PDO_HOME}/bin/lib/common.sh
+
 check_pdo_runtime_env
 check_python_version
 
 # -----------------------------------------------------------------
 # Process command line arguments
 # -----------------------------------------------------------------
+F_CCF_PDO_DIR=${CCF_PDO_DIR:-${PDO_INSTALL_ROOT}}
+F_CCF_LEDGER_DIR=${CCF_LEDGER_DIR:-${PDO_HOME}/ccf}
 F_INTERFACE=${PDO_HOSTNAME:-${HOSTNAME}}
 F_PORT=6600
 
 SCRIPT_NAME=$(basename ${BASH_SOURCE[-1]} )
-USAGE='-i|--interface [hostname] -p|--port [port]'
+USAGE='-i|--interface [hostname] -p|--port [port] --pdo-dir [path] --ledger-dir [path]'
 SHORT_OPTS='i:p:'
-LONG_OPTS='interface:,port:'
+LONG_OPTS='interface:,port:,pdo-dir:,ledger-dir:'
 
 TEMP=$(getopt -o ${SHORT_OPTS} --long ${LONG_OPTS} -n "${SCRIPT_NAME}" -- "$@")
 if [ $? != 0 ] ; then echo "Usage: ${SCRIPT_NAME} ${USAGE}" >&2 ; exit 1 ; fi
@@ -39,20 +42,30 @@ while true ; do
     case "$1" in
         -i|--interface) F_INTERFACE="$2" ; shift 2 ;;
         -p|--port) F_PORT="$2" ; shift 2 ;;
+        --pdo-dir) F_CCF_PDO_DIR="$2" ; shift 2 ;;
+        --ledger-dir) F_CCF_LEDGER_DIR="$2" ; shift 2 ;;
         --help) echo "Usage: ${SCRIPT_NAME} ${USAGE}"; exit 0 ;;
     	--) shift ; break ;;
     	*) echo "Internal error!" ; exit 1 ;;
     esac
 done
 
-# the CCF sandbox script creates the python virtual environment
-# in the directory from which it is executed so we need to make
-# sure we run it in the install directory... this is generally bad
-# practice and we should probably move away from sandbox.
-cd ${PDO_HOME}/ccf
+# -----------------------------------------------------------------
+# make sure that we have a completely prepared environment
+# -----------------------------------------------------------------
+if [ ! -f ${F_CCF_PDO_DIR}/bin/activate ]; then
+    die incomplete configuration, unable to locate CCF_PDO_DIR virtual environment
+fi
+
+if [ ! -f ${F_CCF_LEDGER_DIR}/bin/activate ]; then
+    die incomplete configuration, unable to locate CCF_LEDGER_DIR virtual environment
+fi
 
-if [ -f ${PDO_HOME}/ccf/workspace/sandbox_0/node.pid ]; then
-    if ps -p $(cat ${PDO_HOME}/ccf/workspace/sandbox_0/node.pid) > /dev/null
+# -----------------------------------------------------------------
+# check to see if there is an instance already running
+# -----------------------------------------------------------------
+if [ -f ${PDO_LEDGER_DIR}/workspace/pdo_tp_0/node.pid ]; then
+    if ps -p $(<"${PDO_LEDGER_DIR}/workspace/pdo_tp_0/node.pid")
     then
         yell cchost appears to be running already
         exit -1
@@ -65,41 +78,71 @@ then
     exit -1
 fi
 
+# -----------------------------------------------------------------
+# clean up any old ledger keys
+# -----------------------------------------------------------------
 rm -f ${PDO_LEDGER_KEY_ROOT}/networkcert.pem
 rm -f ${PDO_LEDGER_KEY_ROOT}/ledger_authority_pub.pem
 
 # ensure that we use an IP address for the interface
 F_INTERFACE_ADDRESS=$(force_to_ip ${F_INTERFACE})
-
 yell start ccf network with "local://${F_INTERFACE_ADDRESS}:${F_PORT}"
 
+# -----------------------------------------------------------------
+# run the CCF script to start the network. this script must operate
+# in the python virtual environment created for it in ${F_CCF_LEDGER_DIR}
+# note that simply executing python out of the ccf ledger virtual env
+# is not sufficient since ccf code has expectations for the PATH which
+# is only changed when you activate.
+# -----------------------------------------------------------------
 say attempt to start ccf node
-${CCF_BASE}/bin/sandbox.sh \
-    --verbose \
-    --host-log-level info \
-    -p ${PDO_HOME}/ccf/lib/libpdoenc \
-    -n "local://${F_INTERFACE_ADDRESS}:${F_PORT}" \
-    -e virtual -t virtual \
-    --workspace ${PDO_HOME}/ccf/workspace \
-    --initial-node-cert-validity-days 365 \
-    --initial-service-cert-validity-days 365 &
-
-sleep 5
-
-while [ ! -f ${PDO_HOME}/ccf/workspace/sandbox_common/service_cert.pem ]; do
+
+source ${F_CCF_LEDGER_DIR}/bin/activate
+CURL_CLIENT=ON INITIAL_MEMBER_COUNT=1 \
+    ${F_CCF_LEDGER_DIR}/bin/python ${CCF_BASE}/bin/start_network.py \
+        --binary-dir ${CCF_BASE}/bin \
+        --enclave-type virtual \
+        --enclave-platform virtual \
+        --constitution ${CCF_BASE}/bin/actions.js \
+        --constitution ${CCF_BASE}/bin/validate.js \
+        --constitution ${CCF_BASE}/bin/resolve.js \
+        --constitution ${CCF_BASE}/bin/apply.js \
+        --ledger-chunk-bytes 5000000 \
+        --snapshot-tx-interval 10000 \
+        --initial-node-cert-validity-days 365 \
+        --initial-service-cert-validity-days 365 \
+        --label pdo_tp \
+        --verbose \
+        --host-log-level info \
+        --workspace ${F_CCF_LEDGER_DIR}/workspace \
+        -p ${F_CCF_LEDGER_DIR}/lib/libpdoenc \
+        -n "local://${F_INTERFACE_ADDRESS}:${F_PORT}" &
+deactivate
+
+# the ledger keys are the indicator that the service has started successfully
+while [ ! -f ${F_CCF_LEDGER_DIR}/workspace/pdo_tp_common/service_cert.pem ]; do
     say "wait for cchost to write the ledger keys"
     sleep 5
 done
 
+# -----------------------------------------------------------------
 say ledger keys written, copy the keys to PDO_LEDGER_KEY_ROOT
-cp ${PDO_HOME}/ccf/workspace/sandbox_common/service_cert.pem ${PDO_LEDGER_KEY_ROOT}/networkcert.pem
-cp ${PDO_HOME}/ccf/workspace/sandbox_common/member0_cert.pem ${PDO_LEDGER_KEY_ROOT}/memberccf_cert.pem
-cp ${PDO_HOME}/ccf/workspace/sandbox_common/member0_privk.pem ${PDO_LEDGER_KEY_ROOT}/memberccf_privk.pem
+# -----------------------------------------------------------------
+cp ${F_CCF_LEDGER_DIR}/workspace/pdo_tp_common/service_cert.pem ${PDO_LEDGER_KEY_ROOT}/networkcert.pem
+cp ${F_CCF_LEDGER_DIR}/workspace/pdo_tp_common/member0_cert.pem ${PDO_LEDGER_KEY_ROOT}/memberccf_cert.pem
+cp ${F_CCF_LEDGER_DIR}/workspace/pdo_tp_common/member0_privk.pem ${PDO_LEDGER_KEY_ROOT}/memberccf_privk.pem
 
+# -----------------------------------------------------------------
 # Generate authority needs to check to make sure that the ledger
-# is open and delay briefly if it is not open
+# is open and delay briefly if it is not open; this operation and
+# all of the following operations are ccf *client* operations and
+# require the client python virtual environment that was created
+# in ${F_CCF_PDO_DIR}
+# -----------------------------------------------------------------
+source ${F_CCF_PDO_DIR}/bin/activate
+
 say generate the ledger authority
-try ${PDO_HOME}/ccf/bin/generate_ledger_authority.py \
+try ${F_CCF_PDO_DIR}/bin/generate_ledger_authority.py \
     --logfile __screen__ --loglevel WARNING \
     --interface ${F_INTERFACE_ADDRESS} --port ${F_PORT}
 
@@ -109,15 +152,18 @@ try ${PDO_HOME}/ccf/bin/generate_ledger_authority.py \
 
 if [ "${SGX_MODE}" == "SIM" ]; then
     say set check_attestation to false in SGX SIM mode
-    try ${PDO_HOME}/ccf/bin/register_enclave_attestation_verification_policy.py \
+    try ${F_CCF_PDO_DIR}/bin/register_enclave_attestation_verification_policy.py \
         --logfile __screen__ --loglevel WARNING \
         --interface ${F_INTERFACE_ADDRESS} --port ${F_PORT}
-
 fi
 
 say save the ledger authority key
-try ${PDO_HOME}/ccf/bin/fetch_ledger_authority.py \
+try ${F_CCF_PDO_DIR}/bin/fetch_ledger_authority.py \
     --logfile __screen__ --loglevel WARNING \
     --interface ${F_INTERFACE_ADDRESS} --port ${F_PORT}
 
+deactivate
+
+# -----------------------------------------------------------------
 yell ledger URL is http://${F_INTERFACE_ADDRESS}:${F_PORT}
+# -----------------------------------------------------------------

From 6b1d6cde3404b1774bd33502249f4181ba28fbf0 Mon Sep 17 00:00:00 2001
From: Mic Bowman <mic.bowman@intel.com>
Date: Wed, 14 Feb 2024 15:16:11 -0800
Subject: [PATCH 2/3] Copy the member keys from the xfer directory

When registration is requested, we need the CCF member keys in order
to perform the registration Check for the keys & copy them to
${PDO_LEDGER_KEY_ROOT}.

Signed-off-by: Mic Bowman <mic.bowman@intel.com>
---
 docker/tools/run_services_tests.sh | 7 +++++++
 docker/tools/start_services.sh     | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/docker/tools/run_services_tests.sh b/docker/tools/run_services_tests.sh
index 5f35b148..773b9eaa 100755
--- a/docker/tools/run_services_tests.sh
+++ b/docker/tools/run_services_tests.sh
@@ -56,6 +56,13 @@ yell check for registration
 # -----------------------------------------------------------------
 # this probably requires additional CCF keys, need to test this
 if [ "$SGX_MODE" == "HW" ]; then
+    if [ ! -f ${XFER}/ccf/keys/memberccf_privk.pem ] ; then
+        die unable to locate CCF policies keys
+    fi
+
+    try cp ${XFER_DIR}/ccf/keys/memberccf_cert.pem ${PDO_LEDGER_KEY_ROOT}/
+    try cp ${XFER_DIR}/ccf/keys/memberccf_privk.pem ${PDO_LEDGER_KEY_ROOT}/
+
     try make -C ${PDO_SOURCE_ROOT}/build register
 fi
 
diff --git a/docker/tools/start_services.sh b/docker/tools/start_services.sh
index b5b85277..11f21926 100755
--- a/docker/tools/start_services.sh
+++ b/docker/tools/start_services.sh
@@ -115,6 +115,13 @@ try cp ${XFER_DIR}/ccf/keys/networkcert.pem ${PDO_LEDGER_KEY_ROOT}/
 yell register the enclave if necessary
 # -----------------------------------------------------------------
 if [ "${F_REGISTER,,}" == 'yes' ]; then
+    if [ ! -f ${XFER}/ccf/keys/memberccf_privk.pem ] ; then
+        die unable to locate CCF policies keys
+    fi
+
+    try cp ${XFER_DIR}/ccf/keys/memberccf_cert.pem ${PDO_LEDGER_KEY_ROOT}/
+    try cp ${XFER_DIR}/ccf/keys/memberccf_privk.pem ${PDO_LEDGER_KEY_ROOT}/
+
     try make -C ${PDO_SOURCE_ROOT}/build register
 fi
 

From 27ea18bf16801e0136b4d9c32a13e8b833e0981b Mon Sep 17 00:00:00 2001
From: Mic Bowman <mic.bowman@intel.com>
Date: Wed, 14 Feb 2024 16:05:45 -0800
Subject: [PATCH 3/3] Small fixes to address PR feedback

* add dependency checking to remove parallel make issues
* add secp256k1 to PDO CCF docker
* more use of cmake, add clean_config to test_no_reset
* wrap makefiles with check for failure

Signed-off-by: Mic Bowman <mic.bowman@intel.com>
---
 bin/Makefile                             | 14 +++++++-------
 build/Makefile                           |  2 +-
 docker/make.dev                          |  4 ++--
 docker/pdo_ccf_base.dockerfile           |  1 +
 docker/tools/build_ccf.sh                | 13 +++++++------
 docker/tools/build_client.sh             |  8 +++++---
 docker/tools/build_services.sh           | 14 ++++++++++----
 ledgers/ccf/Makefile                     | 11 ++++++-----
 ledgers/ccf/scripts/start_ccf_network.sh |  1 +
 9 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/bin/Makefile b/bin/Makefile
index fc07f2c7..ae771ed8 100644
--- a/bin/Makefile
+++ b/bin/Makefile
@@ -12,20 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PDO_HOME ?= /opt/pdo
-BIN_DIR=${PDO_HOME}/bin
-BIN_LIB_DIR=${PDO_HOME}/bin/lib
-
+ifndef PDO_INSTALL_ROOT
+$(error Incomplete configuration, PDO_INSTALL_ROOT is not defined)
+endif
 
+DSTDIR ?= $(PDO_INSTALL_ROOT)/opt/pdo
+BIN_LIB_DIR=$(DSTDIR)/bin/lib
 
 all : install
 
 build :
 
 install : build
-	mkdir -p ${BIN_DIR} ${BIN_LIB_DIR}
-	cp lib/*.sh ${BIN_LIB_DIR}
-	
+	mkdir -p $(BIN_LIB_DIR)
+	cp lib/*.sh $(BIN_LIB_DIR)
 
 clean:
 
diff --git a/build/Makefile b/build/Makefile
index 0d279121..2f354557 100644
--- a/build/Makefile
+++ b/build/Makefile
@@ -74,7 +74,7 @@ clean-build : $(PYTHON_DIR)
 
 environment : $(DSTDIR) $(PYTHON_DIR)
 
-$(PYTHON_DIR) :
+$(PYTHON_DIR) : $(DSTDIR)
 	virtualenv -p python3 --no-download $(DSTDIR)
 	. $(abspath $(DSTDIR)/bin/activate) && pip install --upgrade pip
 	. $(abspath $(DSTDIR)/bin/activate) && pip install --upgrade setuptools
diff --git a/docker/make.dev b/docker/make.dev
index e7c65d23..411b9d79 100644
--- a/docker/make.dev
+++ b/docker/make.dev
@@ -34,7 +34,7 @@ PIP_CACHE_DIR   ?= $(DOCKER_DIR)/cache/pip
 DOCKER_DEV_ARGS += -v $(PIP_CACHE_DIR):/project/pdo/.cache/pip
 
 # Docker runtime args specific to client
-DOCKER_CLIENT_DEV_ARGS ?= 
+DOCKER_CLIENT_DEV_ARGS ?=
 # - if you want pdo-contracts mounted in client dev image, define (e.g., in make.loc)
 #   variable 'DOCKER_CLIENT_WITH_PDO_CONTRACTS'
 ifdef DOCKER_CLIENT_WITH_PDO_CONTRACTS
@@ -64,7 +64,7 @@ stop_services_dev :
 stop_client_dev :
 	- docker rm -f client_container_dev
 
-test_no_reset : build_test stop_all
+test_no_reset : clean_config build_test stop_all
 	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) up --abort-on-container-exit
 	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) down
 
diff --git a/docker/pdo_ccf_base.dockerfile b/docker/pdo_ccf_base.dockerfile
index 46a80203..e163d9bb 100644
--- a/docker/pdo_ccf_base.dockerfile
+++ b/docker/pdo_ccf_base.dockerfile
@@ -32,6 +32,7 @@ ARG ADD_APT_PKGS=
 ENV DEBIAN_FRONTEND "noninteractive"
 RUN apt-get update \
     && apt-get install -y -q \
+        libsecp256k1-dev \
         lsof \
         python \
         python3-dev \
diff --git a/docker/tools/build_ccf.sh b/docker/tools/build_ccf.sh
index 5b4e61af..af6f2d8d 100755
--- a/docker/tools/build_ccf.sh
+++ b/docker/tools/build_ccf.sh
@@ -13,14 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source /project/pdo/tools/environment.sh
-
-# these variables should be unused during build
 export PDO_HOSTNAME=
 export PDO_LEDGER_URL=
 
+source /project/pdo/tools/environment.sh
+source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+check_pdo_build_env
+
 # -----------------------------------------------------------------
-# set up the basic structure in ${PDO_INSTALL_ROOT}
+yell Build and install CCF into ${PDO_INSTALL_ROOT}
 # -----------------------------------------------------------------
-make -C ${PDO_SOURCE_ROOT}/ledgers/ccf environment
-make -C ${PDO_SOURCE_ROOT}/ledgers/ccf install
+try make -C ${PDO_SOURCE_ROOT}/ledgers/ccf environment
+try make -C ${PDO_SOURCE_ROOT}/ledgers/ccf install
diff --git a/docker/tools/build_client.sh b/docker/tools/build_client.sh
index a52266d3..8fd9117f 100755
--- a/docker/tools/build_client.sh
+++ b/docker/tools/build_client.sh
@@ -14,11 +14,13 @@
 # limitations under the License.
 
 source /project/pdo/tools/environment.sh
+source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+check_pdo_build_env
 
 # to get build without (ignored) errors
 export PDO_HOSTNAME=localhost
 export PDO_LEDGER_URL=https://127.0.0.1:6600
 
-make -C ${PDO_SOURCE_ROOT}/build environment
-make -C ${PDO_SOURCE_ROOT}/build template
-make -C ${PDO_SOURCE_ROOT}/build build-client
+try make -C ${PDO_SOURCE_ROOT}/build environment
+try make -C ${PDO_SOURCE_ROOT}/build template
+try make -C ${PDO_SOURCE_ROOT}/build build-client
diff --git a/docker/tools/build_services.sh b/docker/tools/build_services.sh
index 96f5d859..fbc46511 100755
--- a/docker/tools/build_services.sh
+++ b/docker/tools/build_services.sh
@@ -20,7 +20,13 @@ export PDO_LEDGER_URL=https://127.0.0.1:6600
 source /opt/intel/sgxsdk/environment
 source /project/pdo/tools/environment.sh
 
-make -C ${PDO_SOURCE_ROOT}/build environment
-make -C ${PDO_SOURCE_ROOT}/build template
-make -C ${PDO_SOURCE_ROOT}/build system-keys
-make -C ${PDO_SOURCE_ROOT}/build verified-build
+source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+check_pdo_build_env
+
+# -----------------------------------------------------------------
+yell Build and install services into ${PDO_INSTALL_ROOT}
+# -----------------------------------------------------------------
+try make -C ${PDO_SOURCE_ROOT}/build environment
+try make -C ${PDO_SOURCE_ROOT}/build template
+try make -C ${PDO_SOURCE_ROOT}/build system-keys
+try make -C ${PDO_SOURCE_ROOT}/build verified-build
diff --git a/ledgers/ccf/Makefile b/ledgers/ccf/Makefile
index 9c970403..1a96e38f 100644
--- a/ledgers/ccf/Makefile
+++ b/ledgers/ccf/Makefile
@@ -63,7 +63,7 @@ $(PDO_INSTALL_ROOT) :
 
 # This directory indicates whether the ccf ledger python
 # virtual environment has been created
-$(CCF_LEDGER_DIR)/lib/python3.8 :
+$(CCF_LEDGER_DIR)/lib/python3.8 : pdo-environment
 	mkdir -p $(CCF_LEDGER_DIR)
 	mkdir -p $(CCF_LEDGER_DIR)/workspace
 	virtualenv -p python3.8 --no-download $(PDO_HOME)/ccf
@@ -77,7 +77,7 @@ $(CCF_LEDGER_DIR)/lib/python3.8 :
 build : build-pdo-tp
 
 build-pdo-tp : $(BLDDIR)
-	cd $(BLDDIR) && $(NINJA)
+	cmake --build $(BLDDIR)
 
 $(BLDDIR) :
 	cmake -S . -B $(BLDDIR) -GNinja \
@@ -111,11 +111,12 @@ PDO_PYTHON_SCRIPTS = $(wildcard scripts/*.py)
 
 install : install-pdo-tp install-pdo-scripts
 
-install-pdo-tp : build-pdo-tp
-	cd $(BLDDIR) && $(NINJA) install
+# cd $(BLDDIR) && $(NINJA) install
+install-pdo-tp : ledger-environment build-pdo-tp
+	cmake --install $(BLDDIR)
 	cp $(PDO_BASH_SCRIPTS) $(CCF_LEDGER_DIR)/bin
 
-install-pdo-scripts :
+install-pdo-scripts : pdo-environment
 	cp $(PDO_PYTHON_SCRIPTS) $(PDO_INSTALL_ROOT)/bin
 
 # -----------------------------------------------------------------
diff --git a/ledgers/ccf/scripts/start_ccf_network.sh b/ledgers/ccf/scripts/start_ccf_network.sh
index a156b742..17eb6e8b 100755
--- a/ledgers/ccf/scripts/start_ccf_network.sh
+++ b/ledgers/ccf/scripts/start_ccf_network.sh
@@ -98,6 +98,7 @@ yell start ccf network with "local://${F_INTERFACE_ADDRESS}:${F_PORT}"
 say attempt to start ccf node
 
 source ${F_CCF_LEDGER_DIR}/bin/activate
+# Note below is adopted from (the end of ) CCF's tests/sandbox/sandbox.sh ..
 CURL_CLIENT=ON INITIAL_MEMBER_COUNT=1 \
     ${F_CCF_LEDGER_DIR}/bin/python ${CCF_BASE}/bin/start_network.py \
         --binary-dir ${CCF_BASE}/bin \