From dfd4d9502c8b9e0f445ad5f60093160da5d8cd6e Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Mon, 18 Nov 2024 09:13:25 -0500 Subject: [PATCH 1/8] outputs: Removed missed comment from Makefile --- outputs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/outputs/Makefile b/outputs/Makefile index 0788a78ff..747cd98ed 100644 --- a/outputs/Makefile +++ b/outputs/Makefile @@ -25,7 +25,7 @@ sections := environments run_targets := $(addprefix run-,$(sections)) # Entrypoint -update-outputs: #run +update-outputs: run echo "Filtering raw outputs though col" for raw in raw/*/*.out; do \ out=$$(echo $$raw | sed 's.raw/..'); \ From ee04e57bd1f06d387ea1c67cf0a8ee6a1daf3583 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Mon, 18 Nov 2024 09:16:50 -0500 Subject: [PATCH 2/8] Add outputs/Makefile to outputs rebuild trigger --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3ce6ed6e..e614cbd72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: - '.github/**' - 'container/**' - 'outputs/**.sh' + - 'outputs/Makefile' container: if: ${{ needs.changes.outputs.container == 'true' || startsWith(github.ref, 'refs/tags/') }} From 29097b43b4085118f59c74552ab48efe15baac2d Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Mon, 18 Nov 2024 10:01:55 -0500 Subject: [PATCH 3/8] Switch docker container to run make local and keep state between sections --- outputs/Makefile | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/outputs/Makefile b/outputs/Makefile index 747cd98ed..80cb7d87c 100644 --- a/outputs/Makefile +++ b/outputs/Makefile @@ -14,7 +14,7 @@ DOCKER_RUN_OPTS := # Name of the container we'll generate the tutorial outputs with -container := ghcr.io/spack/tutorial:latest@sha256:417900cce4592159456050a91d844fc9e81fcb01125a55555092fa450ce59a67 +container := ghcr.io/spack/tutorial:latest # Sections to build sections := environments @@ -32,34 +32,25 @@ update-outputs: run cat $$raw | perl -pe 's/\x1b]0;.+?\x07//g' | perl -pe 's/\x1b\[\d+F\x1b\[J//g' | perl -pe 's/\033\[([01];)?\d+m//g' | col -bp | sed '/^==> Waiting for/d' > $$out; \ done -run: run-scripting - -run-scripting: run-cache scripting.sh -run-cache: run-dev cache.sh -run-dev: run-packaging dev.sh -run-packaging: run-stacks packaging.sh -run-stacks: run-environments stacks.sh -run-environments: run-basics environments.sh -run-basics: basics.sh init_spack.sh defs.sh +run: + $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -t \ + --mount type=bind,source=$(CURDIR),target=/project \ + ${container} \ + make -f /project/Makefile local local: local-scripting -local-scripting: local-cache scripting.sh -local-cache: local-dev cache.sh +local-scripting: local-dev scripting.sh +#local-cache: local-dev cache.sh local-dev: local-packaging dev.sh local-packaging: local-stacks packaging.sh local-stacks: local-environments stacks.sh +# local-config: local-environments config.sh # coming soon^TM local-environments: local-basics environments.sh local-basics: basics.sh init_spack.sh defs.sh local-%: %.sh init_spack.sh defs.sh $(CURDIR)/$(@:local-%=%).sh -run-%: %.sh init_spack.sh defs.sh - $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -t \ - --mount type=bind,source=$(CURDIR),target=/project \ - ${container} \ - /project/$(@:run-%=%).sh && touch $@ - interactive: $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -it \ --mount type=bind,source=$(CURDIR),target=/project \ From dca2ac3ea405e68319b9152eaa740d68642bf91b Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Mon, 18 Nov 2024 10:34:22 -0500 Subject: [PATCH 4/8] Fix container to allow executing binary files as args to run --- container/Dockerfile | 2 +- outputs/Makefile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/container/Dockerfile b/container/Dockerfile index ce8cb7433..213b93c0c 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -32,4 +32,4 @@ USER spack WORKDIR /home/spack -ENTRYPOINT [ "bash" ] +ENTRYPOINT [ "/bin/bash", "-l", "-c" ] diff --git a/outputs/Makefile b/outputs/Makefile index 80cb7d87c..296ff081f 100644 --- a/outputs/Makefile +++ b/outputs/Makefile @@ -35,8 +35,7 @@ update-outputs: run run: $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -t \ --mount type=bind,source=$(CURDIR),target=/project \ - ${container} \ - make -f /project/Makefile local + ${container} make -f /project/Makefile local local: local-scripting local-scripting: local-dev scripting.sh From f96f9c3caa7d153bf982ec759694d64d3efae1a5 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Mon, 18 Nov 2024 10:42:51 -0500 Subject: [PATCH 5/8] Switch to setting a work directory and just running make --- outputs/Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/outputs/Makefile b/outputs/Makefile index 296ff081f..79ba5a0ee 100644 --- a/outputs/Makefile +++ b/outputs/Makefile @@ -32,11 +32,6 @@ update-outputs: run cat $$raw | perl -pe 's/\x1b]0;.+?\x07//g' | perl -pe 's/\x1b\[\d+F\x1b\[J//g' | perl -pe 's/\033\[([01];)?\d+m//g' | col -bp | sed '/^==> Waiting for/d' > $$out; \ done -run: - $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -t \ - --mount type=bind,source=$(CURDIR),target=/project \ - ${container} make -f /project/Makefile local - local: local-scripting local-scripting: local-dev scripting.sh #local-cache: local-dev cache.sh @@ -50,9 +45,14 @@ local-basics: basics.sh init_spack.sh defs.sh local-%: %.sh init_spack.sh defs.sh $(CURDIR)/$(@:local-%=%).sh +run: + $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -t \ + --mount type=bind,source=$(CURDIR),target=/project -w /project \ + ${container} make local + interactive: $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -it \ - --mount type=bind,source=$(CURDIR),target=/project \ + --mount type=bind,source=$(CURDIR),target=/project -w /project \ ${container} $(addprefix clean-,$(sections)): From f3d2210637895af88cda14fa5bcd10caa8a65fc3 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Mon, 18 Nov 2024 10:58:26 -0500 Subject: [PATCH 6/8] Fix use container if rebuilt on PR --- .github/workflows/ci.yml | 2 ++ .github/workflows/outputs.yml | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e614cbd72..a7936409d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,3 +57,5 @@ jobs: || startsWith(github.ref, 'refs/tags/')) needs: [ changes, container ] uses: ./.github/workflows/outputs.yml + with: + container: ${{ needs.changes.outputs.container == 'true' }} diff --git a/.github/workflows/outputs.yml b/.github/workflows/outputs.yml index 3552c5d32..8b7fe7779 100644 --- a/.github/workflows/outputs.yml +++ b/.github/workflows/outputs.yml @@ -1,6 +1,10 @@ name: Generate outputs on: workflow_call: + inputs: + container: + required: true + type: boolean jobs: re-generate: @@ -39,15 +43,15 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - - name: Download container from artifact if PR - if: github.event_name == 'pull_request' && needs.container.result == 'success' + - name: Download container if was previously created + if: github.event_name == 'pull_request' && inputs.container uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 with: name: tutorial-container path: /tmp - name: Import container from PR artifact - if: github.event_name == 'pull_request' && needs.container.result == 'success' + if: github.event_name == 'pull_request' && inputs.container run: | docker load --input /tmp/tutorial-container.tar docker image ls -a From 169fe798604ff111d4d7d3419868457c07b40dd9 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Mon, 18 Nov 2024 11:19:00 -0500 Subject: [PATCH 7/8] Fix comand passed to make inside of the container --- outputs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/outputs/Makefile b/outputs/Makefile index 79ba5a0ee..2e2ad4420 100644 --- a/outputs/Makefile +++ b/outputs/Makefile @@ -48,7 +48,7 @@ local-%: %.sh init_spack.sh defs.sh run: $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -t \ --mount type=bind,source=$(CURDIR),target=/project -w /project \ - ${container} make local + ${container} "make local" interactive: $(DOCKER) run $(DOCKER_RUN_OPTS) --rm -it \ From 3519e6da122da51ec3b4e032206523b9bfa0d965 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Mon, 18 Nov 2024 13:46:02 -0500 Subject: [PATCH 8/8] Improve name of input option --- .github/workflows/ci.yml | 2 +- .github/workflows/outputs.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7936409d..e3211ff3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,4 +58,4 @@ jobs: needs: [ changes, container ] uses: ./.github/workflows/outputs.yml with: - container: ${{ needs.changes.outputs.container == 'true' }} + container_artifact: ${{ needs.changes.outputs.container == 'true' }} diff --git a/.github/workflows/outputs.yml b/.github/workflows/outputs.yml index 8b7fe7779..8eceed91f 100644 --- a/.github/workflows/outputs.yml +++ b/.github/workflows/outputs.yml @@ -2,7 +2,7 @@ name: Generate outputs on: workflow_call: inputs: - container: + container_artifact: required: true type: boolean @@ -44,14 +44,14 @@ jobs: repository: ${{ github.event.pull_request.head.repo.full_name }} - name: Download container if was previously created - if: github.event_name == 'pull_request' && inputs.container + if: github.event_name == 'pull_request' && inputs.container_artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 with: name: tutorial-container path: /tmp - name: Import container from PR artifact - if: github.event_name == 'pull_request' && inputs.container + if: github.event_name == 'pull_request' && inputs.container_artifact run: | docker load --input /tmp/tutorial-container.tar docker image ls -a