From 762e8781d264b09c5ad0ac08736c7130f6860c11 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Wed, 29 May 2024 08:07:16 -0600 Subject: [PATCH] bats tests - parallelize All bats tests run with custom root/runroot, so it should be possible to parallelize them. Signed-off-by: Ed Santiago --- tests/blobcache.bats | 6 ++-- tests/bud.bats | 28 ++++++++++------- tests/helpers.bash | 44 +++++++++++++++----------- tests/mkcw.bats | 12 ++++---- tests/sbom.bats | 73 +++++++++++++++++++++++++++++++------------- tests/source.bats | 2 -- tests/test_runner.sh | 2 +- 7 files changed, 106 insertions(+), 61 deletions(-) diff --git a/tests/blobcache.bats b/tests/blobcache.bats index 0f3f819d8cd..56233bad930 100644 --- a/tests/blobcache.bats +++ b/tests/blobcache.bats @@ -58,6 +58,8 @@ function _check_matches() { # Integration test for https://github.com/containers/image/pull/1645 @test "blobcache: blobs must be reused when pushing across registry" { start_registry + + imgname=blobimg$(random_string | tr A-Z a-z) run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT} outputdir=${TEST_SCRATCH_DIR}/outputdir mkdir -p ${outputdir} @@ -66,7 +68,7 @@ function _check_matches() { run_buildah pull dir:${outputdir} run_buildah images -a --format '{{.ID}}' cid=$output - run_buildah --log-level debug push --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth $cid docker://localhost:${REGISTRY_PORT}/test + run_buildah --log-level debug push --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth $cid docker://localhost:${REGISTRY_PORT}/$imgname # must not contain "Skipping blob" since push must happen assert "$output" !~ "Skipping blob" @@ -85,7 +87,7 @@ function _check_matches() { run_buildah pull dir:${outputdir} run_buildah images -a --format '{{.ID}}' cid=$output - run_buildah --log-level debug push --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth $cid docker://localhost:${REGISTRY_PORT}/test + run_buildah --log-level debug push --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth $cid docker://localhost:${REGISTRY_PORT}/$imgname expect_output --substring "Skipping blob" } diff --git a/tests/bud.bats b/tests/bud.bats index 9884f9e22ca..99f27822cad 100644 --- a/tests/bud.bats +++ b/tests/bud.bats @@ -5546,7 +5546,12 @@ _EOF } _EOF - run_buildah build --runtime=crun --runtime-flag=debug --security-opt seccomp=${TEST_SCRATCH_DIR}/seccomp.json \ + # crun caches seccomp profiles, so this test fails if run more than once. + # See https://github.com/containers/crun/issues/1475 + cruntmp=${TEST_SCRATCH_DIR}/crun + mkdir $cruntmp + run_buildah build --runtime=crun --runtime-flag=debug --runtime-flag=root=$cruntmp \ + --security-opt seccomp=${TEST_SCRATCH_DIR}/seccomp.json \ -q -t alpine-bud-crun $WITH_POLICY_JSON --file ${mytmpdir}/Containerfile . expect_output --substring "unknown seccomp syscall" fi @@ -6365,10 +6370,11 @@ _EOF _prefetch alpine local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount cp -R $BUDFILES/buildkit-mount $contextdir + # Use a private TMPDIR so type=cache tests can run in parallel # try writing something to persistent cache - run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewrite + TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewrite # try reading something from persistent cache in a different build - run_buildah build -t testbud2 $WITH_POLICY_JSON -f $contextdir/Dockerfilecacheread + TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud2 $WITH_POLICY_JSON -f $contextdir/Dockerfilecacheread expect_output --substring "hello" } @@ -6379,11 +6385,11 @@ _EOF local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount cp -R $BUDFILES/buildkit-mount $contextdir # try writing something to persistent cache - run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewrite + TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewrite # prune the mount cache - run_buildah prune + TMPDIR=${TEST_SCRATCH_DIR} run_buildah prune # try reading something from persistent cache in a different build - run_buildah 1 build -t testbud2 $WITH_POLICY_JSON -f $contextdir/Dockerfilecacheread + TMPDIR=${TEST_SCRATCH_DIR} run_buildah 1 build -t testbud2 $WITH_POLICY_JSON -f $contextdir/Dockerfilecacheread expect_output --substring "No such file or directory" } @@ -6409,7 +6415,7 @@ _EOF local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount cp -R $BUDFILES/buildkit-mount $contextdir # try writing something to persistent cache - run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewritesharing + TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewritesharing expect_output --substring "world" } @@ -6507,7 +6513,7 @@ _EOF local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount-from cp -R $BUDFILES/buildkit-mount-from $contextdir # try reading something from persistent cache in a different build - run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachefrom $contextdir/ + TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachefrom $contextdir/ expect_output --substring "hello" } @@ -6520,10 +6526,10 @@ _EOF cp -R $BUDFILES/buildkit-mount-from $contextdir # build base image which we will use as our `from` - run_buildah build -t buildkitbase $WITH_POLICY_JSON -f $contextdir/Dockerfilebuildkitbase $contextdir/ + TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t buildkitbase $WITH_POLICY_JSON -f $contextdir/Dockerfilebuildkitbase $contextdir/ # try reading something from persistent cache in a different build - run_buildah 125 build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachefromimage + TMPDIR=${TEST_SCRATCH_DIR} run_buildah 125 build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachefromimage expect_output --substring "no stage found with name buildkitbase" } @@ -6534,7 +6540,7 @@ _EOF local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount-from cp -R $BUDFILES/buildkit-mount-from $contextdir # try reading something from persistent cache in a different build - run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachemultiplefrom $contextdir/ + TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachemultiplefrom $contextdir/ expect_output --substring "hello" expect_output --substring "hello2" } diff --git a/tests/helpers.bash b/tests/helpers.bash index 1c5f699e7bd..c3939d0c812 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -142,9 +142,11 @@ function normalize_image_name() { function _prefetch() { if [ -z "${_BUILDAH_IMAGE_CACHEDIR}" ]; then - _pgid=$(sed -ne 's/^NSpgid:\s*//p' /proc/$$/status) - export _BUILDAH_IMAGE_CACHEDIR=${BATS_TMPDIR}/buildah-image-cache.$_pgid + export _BUILDAH_IMAGE_CACHEDIR=${BATS_SUITE_TMPDIR}/buildah-image-cache mkdir -p ${_BUILDAH_IMAGE_CACHEDIR} + + # It's 700 by default; this prevents 'unshare' from reading cached images + chmod 711 ${BATS_SUITE_TMPDIR} ${BATS_SUITE_TMPDIR}/.. fi local storage= @@ -156,24 +158,32 @@ function _prefetch() { img=$(normalize_image_name "$img") echo "# [checking for: $img]" >&2 fname=$(tr -c a-zA-Z0-9.- - <<< "$img") - if [ -d $_BUILDAH_IMAGE_CACHEDIR/$fname ]; then - echo "# [restoring from cache: $_BUILDAH_IMAGE_CACHEDIR / $img]" >&2 - copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$storage""$img" - else - rm -fr $_BUILDAH_IMAGE_CACHEDIR/$fname - echo "# [copy docker://$img dir:$_BUILDAH_IMAGE_CACHEDIR/$fname]" >&2 - for attempt in $(seq 3) ; do - if copy $COPY_REGISTRY_OPTS docker://"$img" dir:$_BUILDAH_IMAGE_CACHEDIR/$fname ; then - break - fi - sleep 5 - done - echo "# [copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:$storage$img]" >&2 - copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$storage""$img" - fi + ( flock --timeout 300 9 || die "Could not flock"; _prefetch_locksafe $img $fname ) 9> $_BUILDAH_IMAGE_CACHEDIR/$fname.lock done } +# DO NOT CALL THIS. EVER. This must only be called from _prefetch(). +function _prefetch_locksafe() { + local img="$1" + local fname="$2" + + if [ -d $_BUILDAH_IMAGE_CACHEDIR/$fname ]; then + echo "# [restoring from cache: $_BUILDAH_IMAGE_CACHEDIR / $img]" >&2 + copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$storage""$img" + else + rm -fr $_BUILDAH_IMAGE_CACHEDIR/$fname + echo "# [copy docker://$img dir:$_BUILDAH_IMAGE_CACHEDIR/$fname]" >&2 + for attempt in $(seq 3) ; do + if copy $COPY_REGISTRY_OPTS docker://"$img" dir:$_BUILDAH_IMAGE_CACHEDIR/$fname ; then + break + fi + sleep 5 + done + echo "# [copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:$storage$img]" >&2 + copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$storage""$img" + fi +} + function createrandom() { dd if=/dev/urandom bs=1 count=${2:-256} of=${1:-${BATS_TMPDIR}/randomfile} status=none } diff --git a/tests/mkcw.bats b/tests/mkcw.bats index 54e6a9e6633..b4910a48331 100644 --- a/tests/mkcw.bats +++ b/tests/mkcw.bats @@ -59,16 +59,16 @@ function mkcw_check_image() { fi _prefetch busybox _prefetch bash - createrandom randomfile1 - createrandom randomfile2 + createrandom ${TEST_SCRATCH_DIR}/randomfile1 + createrandom ${TEST_SCRATCH_DIR}/randomfile2 echo -n mkcw-convert > "$TEST_SCRATCH_DIR"/key # image has one layer, check with all-lower-case TEE type name - run_buildah mkcw --ignore-attestation-errors --type snp --passphrase=mkcw-convert --add-file randomfile1:/in-a-subdir/rnd1 busybox busybox-cw - mkcw_check_image busybox-cw "" randomfile1:in-a-subdir/rnd1 + run_buildah mkcw --ignore-attestation-errors --type snp --passphrase=mkcw-convert --add-file ${TEST_SCRATCH_DIR}/randomfile1:/in-a-subdir/rnd1 busybox busybox-cw + mkcw_check_image busybox-cw "" ${TEST_SCRATCH_DIR}/randomfile1:in-a-subdir/rnd1 # image has multiple layers, check with all-upper-case TEE type name - run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert --add-file randomfile2:rnd2 bash bash-cw - mkcw_check_image bash-cw "" randomfile2:/rnd2 + run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert --add-file ${TEST_SCRATCH_DIR}/randomfile2:rnd2 bash bash-cw + mkcw_check_image bash-cw "" ${TEST_SCRATCH_DIR}/randomfile2:/rnd2 } @test "mkcw-commit" { diff --git a/tests/sbom.bats b/tests/sbom.bats index 1d1b7ea2a7f..d8d019563ad 100644 --- a/tests/sbom.bats +++ b/tests/sbom.bats @@ -12,19 +12,24 @@ load helpers # clear out one file that we might need to overwrite, but leave the other to # ensure that we don't accidentally append content to files that are already # present - rm -f localpurl.json + rm -f ${TEST_SCRATCH_DIR}/localpurl.json # write to both the image and the local filesystem - run_buildah commit $WITH_POLICY_JSON --sbom ${sbomtype} --sbom-output=localsbom.json --sbom-purl-output=localpurl.json --sbom-image-output=/root/sbom.json --sbom-image-purl-output=/root/purl.json $squash $cid alpine-derived-image + run_buildah commit $WITH_POLICY_JSON --sbom ${sbomtype} \ + --sbom-output=${TEST_SCRATCH_DIR}/localsbom.json \ + --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.json \ + --sbom-image-output=/root/sbom.json \ + --sbom-image-purl-output=/root/purl.json \ + $squash $cid alpine-derived-image # both files should exist now, and neither should be empty - test -s localsbom.json - test -s localpurl.json + test -s ${TEST_SCRATCH_DIR}/localsbom.json + test -s ${TEST_SCRATCH_DIR}/localpurl.json # compare them to their equivalents in the image run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine-derived-image dcid=$output run_buildah mount $dcid mountpoint=$output - cmp $mountpoint/root/purl.json localpurl.json - cmp $mountpoint/root/sbom.json localsbom.json + cmp $mountpoint/root/purl.json ${TEST_SCRATCH_DIR}/localpurl.json + cmp $mountpoint/root/sbom.json ${TEST_SCRATCH_DIR}/localsbom.json done done } @@ -37,19 +42,24 @@ load helpers # clear out one file that we might need to overwrite, but leave the other to # ensure that we don't accidentally append content to files that are already # present - rm -f localpurl.json + rm -f ${TEST_SCRATCH_DIR}/localpurl.json # write to both the image and the local filesystem - run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} --sbom-output=localsbom.json --sbom-purl-output=localpurl.json --sbom-image-output=/root/sbom.json --sbom-image-purl-output=/root/purl.json $layers -t alpine-derived-image $BUDFILES/simple-multi-step + run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} \ + --sbom-output=${TEST_SCRATCH_DIR}/localsbom.json \ + --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.json \ + --sbom-image-output=/root/sbom.json \ + --sbom-image-purl-output=/root/purl.json \ + $layers -t alpine-derived-image $BUDFILES/simple-multi-step # both files should exist now, and neither should be empty - test -s localsbom.json - test -s localpurl.json + test -s ${TEST_SCRATCH_DIR}/localsbom.json + test -s ${TEST_SCRATCH_DIR}/localpurl.json # compare them to their equivalents in the image run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine-derived-image dcid=$output run_buildah mount $dcid mountpoint=$output - cmp $mountpoint/root/purl.json localpurl.json - cmp $mountpoint/root/sbom.json localsbom.json + cmp $mountpoint/root/purl.json ${TEST_SCRATCH_DIR}/localpurl.json + cmp $mountpoint/root/sbom.json ${TEST_SCRATCH_DIR}/localsbom.json done done } @@ -58,10 +68,15 @@ load helpers _prefetch alpine ghcr.io/anchore/syft ghcr.io/aquasecurity/trivy for sbomtype in syft syft-cyclonedx syft-spdx trivy trivy-cyclonedx trivy-spdx; do echo "[sbom type $sbomtype with $layers]" - run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} --sbom-output=localsbom.json --sbom-purl-output=localpurl.json --sbom-image-output=/root/sbom.json --sbom-image-purl-output=/root/purl.json -t busybox-derived-image $BUDFILES/pull + run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} \ + --sbom-output=${TEST_SCRATCH_DIR}/localsbom.json \ + --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.json \ + --sbom-image-output=/root/sbom.json \ + --sbom-image-purl-output=/root/purl.json \ + -t busybox-derived-image $BUDFILES/pull # both files should exist now, and neither should be empty - test -s localsbom.json - test -s localpurl.json + test -s ${TEST_SCRATCH_DIR}/localsbom.json + test -s ${TEST_SCRATCH_DIR}/localpurl.json done } @@ -73,19 +88,33 @@ load helpers # clear out one file that we might need to overwrite, but leave the other to # ensure that we don't accidentally append content to files that are already # present - rm -f localpurl.json - run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} --sbom-output=localsbom.json --sbom-purl-output=localpurl.json --sbom-image-output=/root/sbom.json --sbom-image-purl-output=/root/purl.json $layers -t alpine-derived-image -f $BUDFILES/env/Dockerfile.check-env $BUDFILES/env + rm -f ${TEST_SCRATCH_DIR}/localpurl.json + run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} \ + --sbom-output=${TEST_SCRATCH_DIR}/localsbom.json \ + --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.json \ + --sbom-image-output=/root/sbom.json \ + --sbom-image-purl-output=/root/purl.json \ + $layers -t alpine-derived-image -f $BUDFILES/env/Dockerfile.check-env $BUDFILES/env # both files should exist now, and neither should be empty - test -s localsbom.json - test -s localpurl.json + test -s ${TEST_SCRATCH_DIR}/localsbom.json + test -s ${TEST_SCRATCH_DIR}/localpurl.json done done } @test "bud-sbom-with-non-presets" { _prefetch alpine busybox - run_buildah build --debug $WITH_POLICY_JSON --sbom-output=localsbom.txt --sbom-purl-output=localpurl.txt --sbom-image-output=/root/sbom.txt --sbom-image-purl-output=/root/purl.txt --sbom-scanner-image=alpine --sbom-scanner-command='echo SCANNED ROOT {ROOTFS} > {OUTPUT}' --sbom-scanner-command='echo SCANNED BUILD CONTEXT {CONTEXT} > {OUTPUT}' --sbom-merge-strategy=cat -t busybox-derived-image $BUDFILES/pull + run_buildah build --debug $WITH_POLICY_JSON \ + --sbom-output=${TEST_SCRATCH_DIR}/localsbom.txt \ + --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.txt \ + --sbom-image-output=/root/sbom.txt \ + --sbom-image-purl-output=/root/purl.txt \ + --sbom-scanner-image=alpine \ + --sbom-scanner-command='echo SCANNED ROOT {ROOTFS} > {OUTPUT}' \ + --sbom-scanner-command='echo SCANNED BUILD CONTEXT {CONTEXT} > {OUTPUT}' \ + --sbom-merge-strategy=cat \ + -t busybox-derived-image $BUDFILES/pull # both files should exist now, and neither should be empty - test -s localsbom.json - test -s localpurl.json + test -s ${TEST_SCRATCH_DIR}/localsbom.txt + test -s ${TEST_SCRATCH_DIR}/localpurl.txt } diff --git a/tests/source.bats b/tests/source.bats index 20c669b7565..b8cfa7d415f 100644 --- a/tests/source.bats +++ b/tests/source.bats @@ -153,6 +153,4 @@ load helpers run diff -r $srcdir $pulldir # FIXME: if there's a nonzero chance of this failing, include actual diffs assert "$status" -eq 0 "status from diff of srcdir vs pulldir" - - stop_registry } diff --git a/tests/test_runner.sh b/tests/test_runner.sh index a93654b3721..2cde1830116 100755 --- a/tests/test_runner.sh +++ b/tests/test_runner.sh @@ -19,4 +19,4 @@ function execute() { TESTS=${@:-.} # Run the tests. -execute time bats --tap $TESTS +execute time bats -j 4 --tap $TESTS