From 064914c794537875258dbd06fb970b1d80ae4718 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 4 Oct 2023 14:20:55 -0700
Subject: [PATCH 01/12] mvp profiling

---
 Makefile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Makefile b/Makefile
index 6ced6275..4b7c9d42 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,13 @@ endif
 TARGET := aws
 include $(PWD)/src/toolchain/Makefile
 
+mkc := $(shell mkdir -p $(CACHE_DIR))
+rmp := $(shell rm $(CACHE_DIR)/profile.txt||:)
+.PHONY: FORCE
+FORCE:
+% : FORCE
+	@echo "$(shell date --rfc-3339=ns) $@" >> $(CACHE_DIR)/profile.txt
+
 KEYS := \
 	449E6BFA40E1119328688F981929C2481BEAC51B \
 	6B61ECD76088748C70590D55E90A401336C8AAA9 \

From e84340fa373ca2cc1734e3050a5f07c4b8e4c7ed Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 5 Oct 2023 14:58:26 -0700
Subject: [PATCH 02/12] non working scaffold

---
 Makefile | 55 +++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 49 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 4b7c9d42..4384b317 100644
--- a/Makefile
+++ b/Makefile
@@ -5,12 +5,49 @@ endif
 TARGET := aws
 include $(PWD)/src/toolchain/Makefile
 
-mkc := $(shell mkdir -p $(CACHE_DIR))
-rmp := $(shell rm $(CACHE_DIR)/profile.txt||:)
-.PHONY: FORCE
-FORCE:
-% : FORCE
-	@echo "$(shell date --rfc-3339=ns) $@" >> $(CACHE_DIR)/profile.txt
+ifneq ($(TOOLCHAIN_PROFILE),false)
+mkc := $(shell mkdir -p $(CACHE_DIR_ROOT))
+ifndef TOOLCHAIN_PROFILE_RUNNING
+rmp := $(shell rm -f $(CACHE_DIR_ROOT)/toolchain-profile.csv)
+TOOLCHAIN_PROFILE_START := 0
+TOOLCHAIN_PROFILE_TIME := 0
+TOOLCHAIN_PROFILE_RUNNING := true
+export TOOLCHAIN_PROFILE_RUNNING TOOLCHAIN_PROFILE_START TOOLCHAIN_PROFILE_TIME
+endif
+endif
+
+define toolchain_profile_start
+	$(eval TOOLCHAIN_PROFILE_START=$(shell date +%s))
+	echo START=$(TOOLCHAIN_PROFILE_START)
+	@printf "%s," "$@" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+endef
+
+define toolchain_profile_end
+	echo START=$(TOOLCHAIN_PROFILE_START)
+	echo STOP=$(shell date +%s)
+	$(eval TOOLCHAIN_PROFILE_TIME=$(shell printf $$(($(shell date +%s)-$(TOOLCHAIN_PROFILE_START)))))
+	$(eval TOOLCHAIN_PROFILE_TIME=$(shell printf $$(($(shell date +%s)-$(TOOLCHAIN_PROFILE_START)))))
+	@echo TIME=$(TOOLCHAIN_PROFILE_TIME)
+	@printf "%s\n" "$(shell date -d@$(TOOLCHAIN_PROFILE_TIME) -u +%H:%M:%S)" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+endef
+
+.PHONY: toolchain-profile
+toolchain-profile:
+	@echo Target build times:
+	@column -s, -t < $(CACHE_DIR_ROOT)/toolchain-profile.csv
+
+.PHONY: dummy
+dummy:
+	$(call toolchain_profile_start)
+	sleep 2
+	$(call toolchain_profile_end)
+
+.PHONY: dummy2
+dummy2:
+	$(call toolchain_profile_start)
+	sleep 3
+	$(call toolchain_profile_end)
+
 
 KEYS := \
 	449E6BFA40E1119328688F981929C2481BEAC51B \
@@ -93,6 +130,7 @@ linux-config:
 	make TARGET=$(TARGET) $(CONFIG_DIR)/$(TARGET)/linux.config
 
 define oci-build
+	$(call toolchain_profile_start)
 	$(call toolchain," \
 		mkdir -p $(CACHE_DIR)/$(notdir $(word 2,$^)) \
 		&& cp $(word 1,$^) $(word 2,$^) \
@@ -118,9 +156,11 @@ define oci-build
 				-cf /home/build/$@ \
 				. \
 	")
+	$(call toolchain_profile_end)
 endef
 
 define tar-build
+	$(call toolchain_profile_start)
 	$(call toolchain," \
 		mkdir -p $(CACHE_DIR)/$(notdir $(word 2,$^))-tar \
 		&& cp $(word 2,$^) $(CACHE_DIR)/$(notdir $(word 2,$^))-tar \
@@ -134,6 +174,7 @@ define tar-build
 				-cf /home/build/$@ \
 				. \
 	")
+	$(call toolchain_profile_end)
 endef
 
 $(KEY_DIR)/%.asc:
@@ -144,6 +185,7 @@ $(OUT_DIR)/$(TARGET)-$(ARCH).eif $(OUT_DIR)/$(TARGET)-$(ARCH).pcrs: \
 	$(MAKE) $(CACHE_DIR)/rootfs.cpio
 	$(MAKE) $(CACHE_DIR)/bzImage
 	$(MAKE) $(BIN_DIR)/eif_build
+	$(call toolchain_profile_start)
 	mkdir -p $(CACHE_DIR)/eif
 	$(call toolchain," \
 		export \
@@ -161,6 +203,7 @@ $(OUT_DIR)/$(TARGET)-$(ARCH).eif $(OUT_DIR)/$(TARGET)-$(ARCH).pcrs: \
 			--pcrs_output $(OUT_DIR)/$(TARGET)-$(ARCH).pcrs \
 			--output $(OUT_DIR)/$(TARGET)-$(ARCH).eif; \
 	")
+	$(call toolchain_profile_end)
 
 $(OUT_DIR)/qos_host.$(PLATFORM)-$(ARCH): \
 	$(shell git ls-files src/qos_host src/qos_core config)

From ca55ecbc7c7be4cc903e980cb47ddd34d2fa7250 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 9 Oct 2023 14:56:04 -0700
Subject: [PATCH 03/12] bump toolchain

---
 src/toolchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/toolchain b/src/toolchain
index 832b9640..f0d39c33 160000
--- a/src/toolchain
+++ b/src/toolchain
@@ -1 +1 @@
-Subproject commit 832b9640c541bc5de7964dfb8428cab76324eb24
+Subproject commit f0d39c33bcc9b5dc7546eb35c315f1fb1b7646d4

From 298bbac3e5ab71e6733e12ea47c4d2e3b0ef88fb Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 9 Oct 2023 14:57:10 -0700
Subject: [PATCH 04/12] drop native profile logic favor logic in toolchain

---
 Makefile | 44 --------------------------------------------
 1 file changed, 44 deletions(-)

diff --git a/Makefile b/Makefile
index 4384b317..39ccf54b 100644
--- a/Makefile
+++ b/Makefile
@@ -5,50 +5,6 @@ endif
 TARGET := aws
 include $(PWD)/src/toolchain/Makefile
 
-ifneq ($(TOOLCHAIN_PROFILE),false)
-mkc := $(shell mkdir -p $(CACHE_DIR_ROOT))
-ifndef TOOLCHAIN_PROFILE_RUNNING
-rmp := $(shell rm -f $(CACHE_DIR_ROOT)/toolchain-profile.csv)
-TOOLCHAIN_PROFILE_START := 0
-TOOLCHAIN_PROFILE_TIME := 0
-TOOLCHAIN_PROFILE_RUNNING := true
-export TOOLCHAIN_PROFILE_RUNNING TOOLCHAIN_PROFILE_START TOOLCHAIN_PROFILE_TIME
-endif
-endif
-
-define toolchain_profile_start
-	$(eval TOOLCHAIN_PROFILE_START=$(shell date +%s))
-	echo START=$(TOOLCHAIN_PROFILE_START)
-	@printf "%s," "$@" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
-endef
-
-define toolchain_profile_end
-	echo START=$(TOOLCHAIN_PROFILE_START)
-	echo STOP=$(shell date +%s)
-	$(eval TOOLCHAIN_PROFILE_TIME=$(shell printf $$(($(shell date +%s)-$(TOOLCHAIN_PROFILE_START)))))
-	$(eval TOOLCHAIN_PROFILE_TIME=$(shell printf $$(($(shell date +%s)-$(TOOLCHAIN_PROFILE_START)))))
-	@echo TIME=$(TOOLCHAIN_PROFILE_TIME)
-	@printf "%s\n" "$(shell date -d@$(TOOLCHAIN_PROFILE_TIME) -u +%H:%M:%S)" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
-endef
-
-.PHONY: toolchain-profile
-toolchain-profile:
-	@echo Target build times:
-	@column -s, -t < $(CACHE_DIR_ROOT)/toolchain-profile.csv
-
-.PHONY: dummy
-dummy:
-	$(call toolchain_profile_start)
-	sleep 2
-	$(call toolchain_profile_end)
-
-.PHONY: dummy2
-dummy2:
-	$(call toolchain_profile_start)
-	sleep 3
-	$(call toolchain_profile_end)
-
-
 KEYS := \
 	449E6BFA40E1119328688F981929C2481BEAC51B \
 	6B61ECD76088748C70590D55E90A401336C8AAA9 \

From 5a9551081028e78d9cfb0c874a9bebe172f4527c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 10 Oct 2023 14:59:46 -0700
Subject: [PATCH 05/12] update toolchain

---
 src/toolchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/toolchain b/src/toolchain
index f0d39c33..9a968c87 160000
--- a/src/toolchain
+++ b/src/toolchain
@@ -1 +1 @@
-Subproject commit f0d39c33bcc9b5dc7546eb35c315f1fb1b7646d4
+Subproject commit 9a968c87c874a796d17706d5afd2c0386ee5abb2

From b48d551e963c2f7245fd1b99f7d980847383389f Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 10 Oct 2023 15:00:48 -0700
Subject: [PATCH 06/12] various profiling fixes

---
 Makefile | 69 ++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 57 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index 57f1e7f2..3d3f7e2f 100644
--- a/Makefile
+++ b/Makefile
@@ -32,7 +32,8 @@ default: \
 	$(OUT_DIR)/qos_host.linux-x86_64 \
 	$(OUT_DIR)/qos_enclave.linux-x86_64 \
 	images \
-	$(OUT_DIR)/release.env
+	$(OUT_DIR)/release.env \
+	toolchain-profile
 
 .PHONY: images
 images: \
@@ -86,7 +87,7 @@ linux-config:
 	make TARGET=$(TARGET) $(CONFIG_DIR)/$(TARGET)/linux.config
 
 define oci-build
-	$(call toolchain_profile_start)
+	$(call toolchain-profile-start);
 	$(call toolchain," \
 		mkdir -p $(CACHE_DIR)/$(notdir $(word 2,$^)) \
 		&& cp $(word 1,$^) $(word 2,$^) \
@@ -111,12 +112,12 @@ define oci-build
 				--numeric-owner \
 				-cf /home/build/$@ \
 				. \
-	")
-	$(call toolchain_profile_end)
+	");
+	$(call toolchain-profile-stop)
 endef
 
 define tar-build
-	$(call toolchain_profile_start)
+	$(call toolchain-profile-start);
 	$(call toolchain," \
 		mkdir -p $(CACHE_DIR)/$(notdir $(word 2,$^))-tar \
 		&& cp $(word 2,$^) $(CACHE_DIR)/$(notdir $(word 2,$^))-tar \
@@ -129,8 +130,8 @@ define tar-build
 				--numeric-owner \
 				-cf /home/build/$@ \
 				. \
-	")
-	$(call toolchain_profile_end)
+	");
+	$(call toolchain-profile-stop)
 endef
 
 $(KEY_DIR)/%.asc:
@@ -141,7 +142,7 @@ $(OUT_DIR)/$(TARGET)-$(ARCH).eif $(OUT_DIR)/$(TARGET)-$(ARCH).pcrs: \
 	$(MAKE) $(CACHE_DIR)/rootfs.cpio
 	$(MAKE) $(CACHE_DIR)/bzImage
 	$(MAKE) $(BIN_DIR)/eif_build
-	$(call toolchain_profile_start)
+	$(call toolchain-profile-start)
 	mkdir -p $(CACHE_DIR)/eif
 	$(call toolchain," \
 		export \
@@ -159,11 +160,12 @@ $(OUT_DIR)/$(TARGET)-$(ARCH).eif $(OUT_DIR)/$(TARGET)-$(ARCH).pcrs: \
 			--pcrs_output $(OUT_DIR)/$(TARGET)-$(ARCH).pcrs \
 			--output $(OUT_DIR)/$(TARGET)-$(ARCH).eif; \
 	")
-	$(call toolchain_profile_end)
+	$(call toolchain-profile-stop)
 
 $(OUT_DIR)/qos_host.$(PLATFORM)-$(ARCH): \
 	$(shell git ls-files src/qos_host src/qos_core config)
 	$(MAKE) $(CACHE_DIR)/lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/libc.a
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		export \
 			RUSTFLAGS=' \
@@ -180,6 +182,7 @@ $(OUT_DIR)/qos_host.$(PLATFORM)-$(ARCH): \
 			../target/x86_64-unknown-linux-musl/release/qos_host \
 			/home/build/$@; \
 	")
+	$(call toolchain-profile-stop)
 
 $(OUT_DIR)/qos_host.oci.$(ARCH).tar: \
 	$(SRC_DIR)/images/host/Dockerfile \
@@ -195,6 +198,7 @@ $(OUT_DIR)/qos_enclave.$(PLATFORM)-$(ARCH): \
 	$(shell git ls-files src/qos_enclave config)
 	$(MAKE) $(CACHE_DIR)/lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/libc.a
 	$(MAKE) $(CACHE_DIR)/lib64/libssl.a
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		cd $(SRC_DIR)/qos_enclave \
 		&& export \
@@ -214,14 +218,15 @@ $(OUT_DIR)/qos_enclave.$(PLATFORM)-$(ARCH): \
 			target/x86_64-unknown-linux-musl/release/qos_enclave \
 			/home/build/$@; \
 	")
+	$(call toolchain-profile-stop)
 
 $(OUT_DIR)/qos_enclave.oci.$(ARCH).tar: \
 	$(SRC_DIR)/images/enclave/Dockerfile \
 	$(OUT_DIR)/qos_enclave.$(PLATFORM)-$(ARCH) \
 	$(OUT_DIR)/aws-x86_64.eif
-	mkdir -p $(CACHE_DIR)/$(notdir $(word 2,$^)) \
-	&& cp $(word 3,$^) $(CACHE_DIR)/$(notdir $(word 2,$^)) \
-	&& $(call oci-build)
+	mkdir -p $(CACHE_DIR)/$(notdir $(word 2,$^))
+	cp $(word 3,$^) $(CACHE_DIR)/$(notdir $(word 2,$^))
+	$(call oci-build)
 
 $(OUT_DIR)/qos_enclave.$(ARCH).tar: \
 	$(SRC_DIR)/images/enclave/Dockerfile \
@@ -241,6 +246,7 @@ $(OUT_DIR)/qos_client.$(PLATFORM)-$(ARCH): \
 	)
 	$(MAKE) $(CACHE_DIR)/lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/libc.a
 	$(MAKE) $(CACHE_DIR)/lib/libpcsclite.a
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		cd $(SRC_DIR)/qos_client \
 		&& export \
@@ -258,6 +264,7 @@ $(OUT_DIR)/qos_client.$(PLATFORM)-$(ARCH): \
 			../target/x86_64-unknown-linux-musl/release/qos_client \
 			/home/build/$@; \
 	")
+	$(call toolchain-profile-stop)
 
 $(OUT_DIR)/qos_client.oci.$(ARCH).tar: \
 	$(SRC_DIR)/images/client/Dockerfile \
@@ -270,35 +277,50 @@ $(OUT_DIR)/qos_client.$(ARCH).tar: \
 	$(call tar-build)
 
 $(CONFIG_DIR)/$(TARGET)/linux.config:
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		unset FAKETIME \
 		&& cd /cache/linux-$(LINUX_VERSION) \
 		&& make menuconfig \
 		&& cp .config /config/$(TARGET)/linux.config; \
 	")
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/src/aws-nitro-enclaves-sdk-bootstrap:
+	$(call toolchain-profile-start)
 	$(call git_clone,$@,$(AWS_NITRO_DRIVER_REPO),$(AWS_NITRO_DRIVER_REF))
+	$(call toolchain-profile-stop)
 
 $(FETCH_DIR)/linux-$(LINUX_VERSION).tar.sign:
+	$(call toolchain-profile-start)
 	curl --url $(LINUX_SERVER)/linux-$(LINUX_VERSION).tar.sign --output $@
+	$(call toolchain-profile-stop)
 
 $(FETCH_DIR)/linux-$(LINUX_VERSION).tar.xz:
+	$(call toolchain-profile-start)
 	curl --url $(LINUX_SERVER)/linux-$(LINUX_VERSION).tar.xz --output $@
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/src/pcsc:
+	$(call toolchain-profile-start)
 	$(call git_clone,$@,$(PCSC_REPO),$(PCSC_REF))
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/src/openssl:
+	$(call toolchain-profile-start)
 	$(call git_clone,$@,$(OPENSSL_REPO),$(OPENSSL_REF))
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/libc.a: \
 	$(CACHE_DIR)/rust-libstd-musl.tgz
+	$(call toolchain-profile-start)
 	mkdir -p $(CACHE_DIR)/lib/rustlib
 	tar -xzf $(CACHE_DIR)/rust-libstd-musl.tgz -C $(CACHE_DIR)/lib/rustlib
 	find $(CACHE_DIR)/lib/rustlib -type f -exec touch {} +
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/rust-libstd-musl.tgz:
+	$(call toolchain-profile-start)
 	$(call git_clone,$(CACHE_DIR)/src/rust,$(RUST_REPO),$(RUST_REF))
 	$(call toolchain," \
 		cd $(CACHE_DIR)/src/rust \
@@ -322,9 +344,11 @@ $(CACHE_DIR)/rust-libstd-musl.tgz:
 			-czvf /home/build/$@ \
 			. \
 	")
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/lib/libpcsclite.a:
 	$(MAKE) $(CACHE_DIR)/src/pcsc
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		cd $(CACHE_DIR)/src/pcsc \
 		&& export \
@@ -344,9 +368,11 @@ $(CACHE_DIR)/lib/libpcsclite.a:
 		&& mkdir -p /home/build/$(CACHE_DIR)/lib \
 		&& cp src/.libs/libpcsclite.a /home/build/$@ \
 	")
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/libssl-static.tgz:
 	$(MAKE) $(CACHE_DIR)/src/openssl
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		cd $(CACHE_DIR)/src/openssl \
 		&& export CC='musl-gcc -fPIE -pie -static' \
@@ -372,11 +398,14 @@ $(CACHE_DIR)/libssl-static.tgz:
 			-czvf /home/build/$@ \
 			. \
 	")
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/lib64/libssl.a: \
 	$(CACHE_DIR)/libssl-static.tgz
+	$(call toolchain-profile-start)
 	tar -xzf $(CACHE_DIR)/libssl-static.tgz -C $(CACHE_DIR)/
 	touch $(CACHE_DIR)/lib64/libssl.a
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/init: \
 	$(shell git ls-files \
@@ -390,6 +419,7 @@ $(CACHE_DIR)/init: \
 	) \
 	| $(CACHE_DIR)/lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/libc.a
 
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		export \
 			RUSTFLAGS=' \
@@ -402,22 +432,27 @@ $(CACHE_DIR)/init: \
 		&& cargo build $(CARGO_FLAGS) \
 		&& cp target/x86_64-unknown-linux-musl/release/init /home/build/$@ \
 	")
+	$(call toolchain-profile-stop)
 
 $(BIN_DIR)/gen_init_cpio:
 	$(MAKE) $(CACHE_DIR)/src/linux-$(LINUX_VERSION)/Makefile
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		cd $(CACHE_DIR)/src/linux-$(LINUX_VERSION) && \
 		gcc usr/gen_init_cpio.c -o /home/build/$@ \
 	")
+	$(call toolchain-profile-stop)
 
 $(BIN_DIR)/gen_initramfs.sh: \
 	$(BIN_DIR)/gen_init_cpio \
 	$(CACHE_DIR)/src/linux-$(LINUX_VERSION)/Makefile \
 	$(CACHE_DIR)/src/linux-$(LINUX_VERSION)/usr/gen_initramfs.sh
+	$(call toolchain-profile-start)
 	cat $(CACHE_DIR)/src/linux-$(LINUX_VERSION)/usr/gen_initramfs.sh \
 	| sed 's:usr/gen_init_cpio:gen_init_cpio:g' \
 	> $@
 	chmod +x $@
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/rootfs.cpio: \
 	$(CONFIG_DIR)/$(TARGET)/rootfs.list \
@@ -425,6 +460,7 @@ $(CACHE_DIR)/rootfs.cpio: \
 	$(CACHE_DIR)/nsm.ko \
 	$(BIN_DIR)/gen_init_cpio \
 	$(BIN_DIR)/gen_initramfs.sh
+	$(call toolchain-profile-start)
 	mkdir -p $(CACHE_DIR)/rootfs
 	cp $(CACHE_DIR)/nsm.ko $(CACHE_DIR)/rootfs/
 	cp $(CACHE_DIR)/init $(CACHE_DIR)/rootfs/
@@ -436,12 +472,14 @@ $(CACHE_DIR)/rootfs.cpio: \
 		cpio -itv < $@ && \
 		sha256sum $@; \
 	")
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/src/linux-$(LINUX_VERSION)/Makefile: \
 	  $(KEY_DIR)/$(LINUX_KEY).asc \
 	  $(CONFIG_DIR)/$(TARGET)/linux.config \
 	  | $(FETCH_DIR)/linux-$(LINUX_VERSION).tar.xz \
 	    $(FETCH_DIR)/linux-$(LINUX_VERSION).tar.sign
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		mkdir -p $(CACHE_DIR)/src \
 		&& xz -d --stdout $(FETCH_DIR)/linux-$(LINUX_VERSION).tar.xz > $(CACHE_DIR)/linux-$(LINUX_VERSION).tar \
@@ -454,11 +492,13 @@ $(CACHE_DIR)/src/linux-$(LINUX_VERSION)/Makefile: \
 			-mxf /home/build/$(CACHE_DIR)/linux-$(LINUX_VERSION).tar \
 		&& rm $(CACHE_DIR)/linux-$(LINUX_VERSION).tar \
 	")
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/bzImage: \
 	$(CONFIG_DIR)/$(TARGET)/linux.config \
 	| $(CACHE_DIR)/src/linux-$(LINUX_VERSION)/Makefile \
 	  $(CACHE_DIR)/rootfs.cpio
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		cd $(CACHE_DIR)/src/linux-$(LINUX_VERSION) && \
 		cp /home/build/$(CONFIG_DIR)/$(TARGET)/linux.config .config && \
@@ -467,8 +507,10 @@ $(CACHE_DIR)/bzImage: \
 		cp arch/$(ARCH)/boot/bzImage /home/build/$@ && \
 		sha256sum /home/build/$@; \
 	")
+	$(call toolchain-profile-stop)
 
 $(BIN_DIR)/eif_build:
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		cd $(SRC_DIR)/eif_build && \
 		cargo build \
@@ -476,11 +518,13 @@ $(BIN_DIR)/eif_build:
 			--target x86_64-unknown-linux-gnu && \
 		cp target/x86_64-unknown-linux-gnu/debug/eif_build /home/build/$@; \
 	")
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR)/nsm.ko: \
 	$(CONFIG_DIR)/$(TARGET)/linux.config \
 	| $(CACHE_DIR)/src/linux-$(LINUX_VERSION)/Makefile \
 	  $(CACHE_DIR)/src/aws-nitro-enclaves-sdk-bootstrap
+	$(call toolchain-profile-start)
 	$(call toolchain," \
 		cd $(CACHE_DIR)/src/linux-$(LINUX_VERSION) && \
 		cp /home/build/$(CONFIG_DIR)/$(TARGET)/linux.config .config && \
@@ -493,3 +537,4 @@ $(CACHE_DIR)/nsm.ko: \
 		    M=/home/build/$(CACHE_DIR)/src/aws-nitro-enclaves-sdk-bootstrap/nsm-driver && \
 		cp nsm-driver/nsm.ko /home/build/$@ \
 	")
+	$(call toolchain-profile-stop)

From e8db4d378dedea677124986f813770d7072074f5 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 10 Oct 2023 15:19:27 -0700
Subject: [PATCH 07/12] update toolchain

---
 src/toolchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/toolchain b/src/toolchain
index 9a968c87..b33b2a98 160000
--- a/src/toolchain
+++ b/src/toolchain
@@ -1 +1 @@
-Subproject commit 9a968c87c874a796d17706d5afd2c0386ee5abb2
+Subproject commit b33b2a98aca87f1bdd53e4cf703c1f9ae533ea02

From 26aa59116e5c6985df5635f74be86cd195609056 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 10 Oct 2023 15:21:22 -0700
Subject: [PATCH 08/12] explicitly add .toolchain to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 6b987254..55fa553e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,7 @@ target/
 *.secret
 *.configuration
 *.share
+.toolchain
 
 # Mock data
 !src/integration/mock/boot-e2e/all-personal-dir/user1-dir/*

From 85694b506693d170814a35d54ca3104e53313ace Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 13 Oct 2023 02:39:37 -0700
Subject: [PATCH 09/12] bump toolchain

---
 src/toolchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/toolchain b/src/toolchain
index b33b2a98..10767638 160000
--- a/src/toolchain
+++ b/src/toolchain
@@ -1 +1 @@
-Subproject commit b33b2a98aca87f1bdd53e4cf703c1f9ae533ea02
+Subproject commit 10767638ca795513ef5a63c98cedba060958687a

From 11e7ae2f686c1a70f49ec3d710c945df70ea6146 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 13 Oct 2023 05:06:21 -0700
Subject: [PATCH 10/12] Update profiling logic

---
 Makefile      | 12 ++++++++----
 src/toolchain |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 3d3f7e2f..0938ec18 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
-ifeq ("$(wildcard ./src/toolchain/Makefile)","")
-	gsu := $(shell git submodule update --init --recursive)
-endif
+#ifeq ("$(wildcard ./src/toolchain/Makefile)","")
+#	gsu := $(shell git submodule update --init --recursive)
+#endif
 
 TARGET := aws
 include $(PWD)/src/toolchain/Makefile
@@ -26,6 +26,7 @@ CACHE_FILENAMES := \
 default: \
 	cache \
 	dist-cache \
+	toolchain \
 	$(patsubst %,$(KEY_DIR)/%.asc,$(KEYS)) \
 	$(OUT_DIR)/aws-x86_64.eif \
 	$(OUT_DIR)/qos_client.linux-x86_64 \
@@ -53,7 +54,7 @@ clean: toolchain-clean
 dist: toolchain-dist
 
 .PHONY: reproduce
-reproduce: toolchain-reproduce
+reproduce: toolchain-reproduce toolchain-profile
 
 .PHONY: cache-filenames
 cache-filenames:
@@ -70,8 +71,10 @@ endif
 
 .PHONY: dist-cache
 dist-cache:
+ifneq ($(TOOLCHAIN_REPRODUCE),true)
 	git lfs pull --include=$(DIST_DIR)
 	$(MAKE) toolchain-dist-cache toolchain-restore-mtime
+endif
 
 .PHONY: run
 run: $(CACHE_DIR)/bzImage
@@ -165,6 +168,7 @@ $(OUT_DIR)/$(TARGET)-$(ARCH).eif $(OUT_DIR)/$(TARGET)-$(ARCH).pcrs: \
 $(OUT_DIR)/qos_host.$(PLATFORM)-$(ARCH): \
 	$(shell git ls-files src/qos_host src/qos_core config)
 	$(MAKE) $(CACHE_DIR)/lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/libc.a
+	echo "I am getting called"
 	$(call toolchain-profile-start)
 	$(call toolchain," \
 		export \
diff --git a/src/toolchain b/src/toolchain
index 10767638..00ce00c2 160000
--- a/src/toolchain
+++ b/src/toolchain
@@ -1 +1 @@
-Subproject commit 10767638ca795513ef5a63c98cedba060958687a
+Subproject commit 00ce00c246766b8a2ce09b96cf23b812a04d03a4

From d36238f5f0e46ff3ecc5352d1dc0b66f073e8d8d Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 15 Oct 2023 23:40:24 -0700
Subject: [PATCH 11/12] remove debug changes

---
 Makefile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 0938ec18..0d2e5036 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
-#ifeq ("$(wildcard ./src/toolchain/Makefile)","")
-#	gsu := $(shell git submodule update --init --recursive)
-#endif
+ifeq ("$(wildcard ./src/toolchain/Makefile)","")
+	gsu := $(shell git submodule update --init --recursive)
+endif
 
 TARGET := aws
 include $(PWD)/src/toolchain/Makefile
@@ -168,7 +168,6 @@ $(OUT_DIR)/$(TARGET)-$(ARCH).eif $(OUT_DIR)/$(TARGET)-$(ARCH).pcrs: \
 $(OUT_DIR)/qos_host.$(PLATFORM)-$(ARCH): \
 	$(shell git ls-files src/qos_host src/qos_core config)
 	$(MAKE) $(CACHE_DIR)/lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/libc.a
-	echo "I am getting called"
 	$(call toolchain-profile-start)
 	$(call toolchain," \
 		export \

From d5c3792d662cfe56a73632cc05e978eda879724e Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 16 Oct 2023 01:39:15 -0700
Subject: [PATCH 12/12] explicitly call toolchain for one-off targets

---
 .github/workflows/artifacts.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml
index 3cba4672..2c42de1b 100644
--- a/.github/workflows/artifacts.yml
+++ b/.github/workflows/artifacts.yml
@@ -77,7 +77,7 @@ jobs:
         shell: 'script -q -e -c "bash {0}"'
         run: |
           touch cache/toolchain.tgz
-          make -d toolchain-restore-mtime out/${{ matrix.target }}
+          make -d toolchain-restore-mtime toolchain out/${{ matrix.target }}
 
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with: