bats tests - parallelize

All bats tests run with custom root/runroot, so it should be possible to parallelize them. Signed-off-by: Ed Santiago <[email protected]>
containers · Jun 11, 2024 · d05bc18 · d05bc18
1 parent 44ef3a8
commit d05bc18
Show file tree

Hide file tree

Showing 7 changed files with 106 additions and 61 deletions.
diff --git a/tests/blobcache.bats b/tests/blobcache.bats
@@ -58,6 +58,8 @@ function _check_matches() {
 # Integration test for https://github.com/containers/image/pull/1645
 @test "blobcache: blobs must be reused when pushing across registry" {
 	start_registry
+
+        imgname=blobimg$(random_string | tr A-Z a-z)
 	run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT}
 	outputdir=${TEST_SCRATCH_DIR}/outputdir
 	mkdir -p ${outputdir}
@@ -66,7 +68,7 @@ function _check_matches() {
 	run_buildah pull dir:${outputdir}
 	run_buildah images -a --format '{{.ID}}'
 	cid=$output
-	run_buildah --log-level debug push --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth $cid docker://localhost:${REGISTRY_PORT}/test
+	run_buildah --log-level debug push --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth $cid docker://localhost:${REGISTRY_PORT}/$imgname
 	# must not contain "Skipping blob" since push must happen
 	assert "$output" !~ "Skipping blob"
 
@@ -85,7 +87,7 @@ function _check_matches() {
 	run_buildah pull dir:${outputdir}
 	run_buildah images -a --format '{{.ID}}'
 	cid=$output
-	run_buildah --log-level debug push --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth $cid docker://localhost:${REGISTRY_PORT}/test
+	run_buildah --log-level debug push --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth $cid docker://localhost:${REGISTRY_PORT}/$imgname
 	expect_output --substring "Skipping blob"
 }
 

diff --git a/tests/bud.bats b/tests/bud.bats
@@ -5547,7 +5547,12 @@ _EOF
 }
 _EOF
 
-    run_buildah build --runtime=crun --runtime-flag=debug --security-opt seccomp=${TEST_SCRATCH_DIR}/seccomp.json \
+    # crun caches seccomp profiles, so this test fails if run more than once.
+    # See https://github.com/containers/crun/issues/1475
+    cruntmp=${TEST_SCRATCH_DIR}/crun
+    mkdir $cruntmp
+    run_buildah build --runtime=crun --runtime-flag=debug --runtime-flag=root=$cruntmp \
+                    --security-opt seccomp=${TEST_SCRATCH_DIR}/seccomp.json \
                     -q -t alpine-bud-crun $WITH_POLICY_JSON --file ${mytmpdir}/Containerfile .
     expect_output --substring "unknown seccomp syscall"
   fi
@@ -6366,10 +6371,11 @@ _EOF
   _prefetch alpine
   local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount
   cp -R $BUDFILES/buildkit-mount $contextdir
+  # Use a private TMPDIR so type=cache tests can run in parallel
   # try writing something to persistent cache
-  run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewrite
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewrite
   # try reading something from persistent cache in a different build
-  run_buildah build -t testbud2 $WITH_POLICY_JSON -f $contextdir/Dockerfilecacheread
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud2 $WITH_POLICY_JSON -f $contextdir/Dockerfilecacheread
   expect_output --substring "hello"
 }
 
@@ -6380,11 +6386,11 @@ _EOF
   local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount
   cp -R $BUDFILES/buildkit-mount $contextdir
   # try writing something to persistent cache
-  run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewrite
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewrite
   # prune the mount cache
-  run_buildah prune
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah prune
   # try reading something from persistent cache in a different build
-  run_buildah 1 build -t testbud2 $WITH_POLICY_JSON -f $contextdir/Dockerfilecacheread
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah 1 build -t testbud2 $WITH_POLICY_JSON -f $contextdir/Dockerfilecacheread
   expect_output --substring "No such file or directory"
 }
 
@@ -6410,7 +6416,7 @@ _EOF
   local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount
   cp -R $BUDFILES/buildkit-mount $contextdir
   # try writing something to persistent cache
-  run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewritesharing
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachewritesharing
   expect_output --substring "world"
 }
 
@@ -6508,7 +6514,7 @@ _EOF
   local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount-from
   cp -R $BUDFILES/buildkit-mount-from $contextdir
   # try reading something from persistent cache in a different build
-  run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachefrom $contextdir/
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachefrom $contextdir/
   expect_output --substring "hello"
 }
 
@@ -6521,10 +6527,10 @@ _EOF
   cp -R $BUDFILES/buildkit-mount-from $contextdir
 
   # build base image which we will use as our `from`
-  run_buildah build -t buildkitbase $WITH_POLICY_JSON -f $contextdir/Dockerfilebuildkitbase $contextdir/
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t buildkitbase $WITH_POLICY_JSON -f $contextdir/Dockerfilebuildkitbase $contextdir/
 
   # try reading something from persistent cache in a different build
-  run_buildah 125 build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachefromimage
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah 125 build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachefromimage
   expect_output --substring "no stage found with name buildkitbase"
 }
 
@@ -6535,7 +6541,7 @@ _EOF
   local contextdir=${TEST_SCRATCH_DIR}/buildkit-mount-from
   cp -R $BUDFILES/buildkit-mount-from $contextdir
   # try reading something from persistent cache in a different build
-  run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachemultiplefrom $contextdir/
+  TMPDIR=${TEST_SCRATCH_DIR} run_buildah build -t testbud $WITH_POLICY_JSON -f $contextdir/Dockerfilecachemultiplefrom $contextdir/
   expect_output --substring "hello"
   expect_output --substring "hello2"
 }

diff --git a/tests/helpers.bash b/tests/helpers.bash
@@ -142,9 +142,11 @@ function normalize_image_name() {
 
 function _prefetch() {
     if [ -z "${_BUILDAH_IMAGE_CACHEDIR}" ]; then
-        _pgid=$(sed -ne 's/^NSpgid:\s*//p' /proc/$$/status)
-        export _BUILDAH_IMAGE_CACHEDIR=${BATS_TMPDIR}/buildah-image-cache.$_pgid
+        export _BUILDAH_IMAGE_CACHEDIR=${BATS_SUITE_TMPDIR}/buildah-image-cache
         mkdir -p ${_BUILDAH_IMAGE_CACHEDIR}
+
+        # It's 700 by default; this prevents 'unshare' from reading cached images
+        chmod 711 ${BATS_SUITE_TMPDIR} ${BATS_SUITE_TMPDIR}/..
     fi
 
     local storage=
@@ -156,24 +158,32 @@ function _prefetch() {
         img=$(normalize_image_name "$img")
         echo "# [checking for: $img]" >&2
         fname=$(tr -c a-zA-Z0-9.- - <<< "$img")
-        if [ -d $_BUILDAH_IMAGE_CACHEDIR/$fname ]; then
-            echo "# [restoring from cache: $_BUILDAH_IMAGE_CACHEDIR / $img]" >&2
-            copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$storage""$img"
-        else
-            rm -fr $_BUILDAH_IMAGE_CACHEDIR/$fname
-            echo "# [copy docker://$img dir:$_BUILDAH_IMAGE_CACHEDIR/$fname]" >&2
-            for attempt in $(seq 3) ; do
-                if copy $COPY_REGISTRY_OPTS docker://"$img" dir:$_BUILDAH_IMAGE_CACHEDIR/$fname ; then
-                    break
-                fi
-                sleep 5
-            done
-            echo "# [copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:$storage$img]" >&2
-            copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$storage""$img"
-        fi
+        ( flock --timeout 300 9 || die "Could not flock"; _prefetch_locksafe $img $fname ) 9> $_BUILDAH_IMAGE_CACHEDIR/$fname.lock
     done
 }
 
+# DO NOT CALL THIS. EVER. This must only be called from _prefetch().
+function _prefetch_locksafe() {
+    local img="$1"
+    local fname="$2"
+
+    if [ -d $_BUILDAH_IMAGE_CACHEDIR/$fname ]; then
+        echo "# [restoring from cache: $_BUILDAH_IMAGE_CACHEDIR / $img]" >&2
+        copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$storage""$img"
+    else
+        rm -fr $_BUILDAH_IMAGE_CACHEDIR/$fname
+        echo "# [copy docker://$img dir:$_BUILDAH_IMAGE_CACHEDIR/$fname]" >&2
+        for attempt in $(seq 3) ; do
+            if copy $COPY_REGISTRY_OPTS docker://"$img" dir:$_BUILDAH_IMAGE_CACHEDIR/$fname ; then
+                break
+            fi
+            sleep 5
+        done
+        echo "# [copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:$storage$img]" >&2
+        copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$storage""$img"
+    fi
+}
+
 function createrandom() {
     dd if=/dev/urandom bs=1 count=${2:-256} of=${1:-${BATS_TMPDIR}/randomfile} status=none
 }

diff --git a/tests/mkcw.bats b/tests/mkcw.bats
@@ -59,16 +59,16 @@ function mkcw_check_image() {
   fi
   _prefetch busybox
   _prefetch bash
-  createrandom randomfile1
-  createrandom randomfile2
+  createrandom ${TEST_SCRATCH_DIR}/randomfile1
+  createrandom ${TEST_SCRATCH_DIR}/randomfile2
 
   echo -n mkcw-convert > "$TEST_SCRATCH_DIR"/key
   # image has one layer, check with all-lower-case TEE type name
-  run_buildah mkcw --ignore-attestation-errors --type snp --passphrase=mkcw-convert --add-file randomfile1:/in-a-subdir/rnd1 busybox busybox-cw
-  mkcw_check_image busybox-cw "" randomfile1:in-a-subdir/rnd1
+  run_buildah mkcw --ignore-attestation-errors --type snp --passphrase=mkcw-convert --add-file ${TEST_SCRATCH_DIR}/randomfile1:/in-a-subdir/rnd1 busybox busybox-cw
+  mkcw_check_image busybox-cw "" ${TEST_SCRATCH_DIR}/randomfile1:in-a-subdir/rnd1
   # image has multiple layers, check with all-upper-case TEE type name
-  run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert --add-file randomfile2:rnd2 bash bash-cw
-  mkcw_check_image bash-cw "" randomfile2:/rnd2
+  run_buildah mkcw --ignore-attestation-errors --type SNP --passphrase=mkcw-convert --add-file ${TEST_SCRATCH_DIR}/randomfile2:rnd2 bash bash-cw
+  mkcw_check_image bash-cw "" ${TEST_SCRATCH_DIR}/randomfile2:/rnd2
 }
 
 @test "mkcw-commit" {

diff --git a/tests/sbom.bats b/tests/sbom.bats
@@ -12,19 +12,24 @@ load helpers
       # clear out one file that we might need to overwrite, but leave the other to
       # ensure that we don't accidentally append content to files that are already
       # present
-      rm -f localpurl.json
+      rm -f ${TEST_SCRATCH_DIR}/localpurl.json
       # write to both the image and the local filesystem
-      run_buildah commit $WITH_POLICY_JSON --sbom ${sbomtype} --sbom-output=localsbom.json --sbom-purl-output=localpurl.json --sbom-image-output=/root/sbom.json --sbom-image-purl-output=/root/purl.json $squash $cid alpine-derived-image
+      run_buildah commit $WITH_POLICY_JSON --sbom ${sbomtype} \
+                  --sbom-output=${TEST_SCRATCH_DIR}/localsbom.json \
+                  --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.json \
+                  --sbom-image-output=/root/sbom.json \
+                  --sbom-image-purl-output=/root/purl.json \
+                  $squash $cid alpine-derived-image
       # both files should exist now, and neither should be empty
-      test -s localsbom.json
-      test -s localpurl.json
+      test -s ${TEST_SCRATCH_DIR}/localsbom.json
+      test -s ${TEST_SCRATCH_DIR}/localpurl.json
       # compare them to their equivalents in the image
       run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine-derived-image
       dcid=$output
       run_buildah mount $dcid
       mountpoint=$output
-      cmp $mountpoint/root/purl.json localpurl.json
-      cmp $mountpoint/root/sbom.json localsbom.json
+      cmp $mountpoint/root/purl.json ${TEST_SCRATCH_DIR}/localpurl.json
+      cmp $mountpoint/root/sbom.json ${TEST_SCRATCH_DIR}/localsbom.json
     done
   done
 }
@@ -37,19 +42,24 @@ load helpers
       # clear out one file that we might need to overwrite, but leave the other to
       # ensure that we don't accidentally append content to files that are already
       # present
-      rm -f localpurl.json
+      rm -f ${TEST_SCRATCH_DIR}/localpurl.json
       # write to both the image and the local filesystem
-      run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} --sbom-output=localsbom.json --sbom-purl-output=localpurl.json --sbom-image-output=/root/sbom.json --sbom-image-purl-output=/root/purl.json $layers -t alpine-derived-image $BUDFILES/simple-multi-step
+      run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} \
+                  --sbom-output=${TEST_SCRATCH_DIR}/localsbom.json \
+                  --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.json \
+                  --sbom-image-output=/root/sbom.json \
+                  --sbom-image-purl-output=/root/purl.json \
+                  $layers -t alpine-derived-image $BUDFILES/simple-multi-step
       # both files should exist now, and neither should be empty
-      test -s localsbom.json
-      test -s localpurl.json
+      test -s ${TEST_SCRATCH_DIR}/localsbom.json
+      test -s ${TEST_SCRATCH_DIR}/localpurl.json
       # compare them to their equivalents in the image
       run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine-derived-image
       dcid=$output
       run_buildah mount $dcid
       mountpoint=$output
-      cmp $mountpoint/root/purl.json localpurl.json
-      cmp $mountpoint/root/sbom.json localsbom.json
+      cmp $mountpoint/root/purl.json ${TEST_SCRATCH_DIR}/localpurl.json
+      cmp $mountpoint/root/sbom.json ${TEST_SCRATCH_DIR}/localsbom.json
     done
   done
 }
@@ -58,10 +68,15 @@ load helpers
   _prefetch alpine ghcr.io/anchore/syft ghcr.io/aquasecurity/trivy
   for sbomtype in syft syft-cyclonedx syft-spdx trivy trivy-cyclonedx trivy-spdx; do
     echo "[sbom type $sbomtype with $layers]"
-    run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} --sbom-output=localsbom.json --sbom-purl-output=localpurl.json --sbom-image-output=/root/sbom.json --sbom-image-purl-output=/root/purl.json -t busybox-derived-image $BUDFILES/pull
+    run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} \
+                --sbom-output=${TEST_SCRATCH_DIR}/localsbom.json \
+                --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.json \
+                --sbom-image-output=/root/sbom.json \
+                --sbom-image-purl-output=/root/purl.json \
+                -t busybox-derived-image $BUDFILES/pull
     # both files should exist now, and neither should be empty
-    test -s localsbom.json
-    test -s localpurl.json
+    test -s ${TEST_SCRATCH_DIR}/localsbom.json
+    test -s ${TEST_SCRATCH_DIR}/localpurl.json
   done
 }
 
@@ -73,19 +88,33 @@ load helpers
       # clear out one file that we might need to overwrite, but leave the other to
       # ensure that we don't accidentally append content to files that are already
       # present
-      rm -f localpurl.json
-      run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} --sbom-output=localsbom.json --sbom-purl-output=localpurl.json --sbom-image-output=/root/sbom.json --sbom-image-purl-output=/root/purl.json $layers -t alpine-derived-image -f $BUDFILES/env/Dockerfile.check-env $BUDFILES/env
+      rm -f ${TEST_SCRATCH_DIR}/localpurl.json
+      run_buildah build $WITH_POLICY_JSON --sbom ${sbomtype} \
+                  --sbom-output=${TEST_SCRATCH_DIR}/localsbom.json \
+                  --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.json \
+                  --sbom-image-output=/root/sbom.json \
+                  --sbom-image-purl-output=/root/purl.json \
+                  $layers -t alpine-derived-image -f $BUDFILES/env/Dockerfile.check-env $BUDFILES/env
       # both files should exist now, and neither should be empty
-      test -s localsbom.json
-      test -s localpurl.json
+      test -s ${TEST_SCRATCH_DIR}/localsbom.json
+      test -s ${TEST_SCRATCH_DIR}/localpurl.json
     done
   done
 }
 
 @test "bud-sbom-with-non-presets" {
   _prefetch alpine busybox
-  run_buildah build --debug $WITH_POLICY_JSON --sbom-output=localsbom.txt --sbom-purl-output=localpurl.txt --sbom-image-output=/root/sbom.txt --sbom-image-purl-output=/root/purl.txt --sbom-scanner-image=alpine --sbom-scanner-command='echo SCANNED ROOT {ROOTFS} > {OUTPUT}' --sbom-scanner-command='echo SCANNED BUILD CONTEXT {CONTEXT} > {OUTPUT}' --sbom-merge-strategy=cat -t busybox-derived-image $BUDFILES/pull
+  run_buildah build --debug $WITH_POLICY_JSON \
+              --sbom-output=${TEST_SCRATCH_DIR}/localsbom.txt \
+              --sbom-purl-output=${TEST_SCRATCH_DIR}/localpurl.txt \
+              --sbom-image-output=/root/sbom.txt \
+              --sbom-image-purl-output=/root/purl.txt \
+              --sbom-scanner-image=alpine \
+              --sbom-scanner-command='echo SCANNED ROOT {ROOTFS} > {OUTPUT}' \
+              --sbom-scanner-command='echo SCANNED BUILD CONTEXT {CONTEXT} > {OUTPUT}' \
+              --sbom-merge-strategy=cat \
+              -t busybox-derived-image $BUDFILES/pull
   # both files should exist now, and neither should be empty
-  test -s localsbom.json
-  test -s localpurl.json
+  test -s ${TEST_SCRATCH_DIR}/localsbom.txt
+  test -s ${TEST_SCRATCH_DIR}/localpurl.txt
 }
diff --git a/tests/source.bats b/tests/source.bats
@@ -153,6 +153,4 @@ load helpers
   run diff -r $srcdir $pulldir
   # FIXME: if there's a nonzero chance of this failing, include actual diffs
   assert "$status" -eq 0 "status from diff of srcdir vs pulldir"
-
-  stop_registry
 }
diff --git a/tests/test_runner.sh b/tests/test_runner.sh
@@ -19,4 +19,4 @@ function execute() {
 TESTS=${@:-.}
 
 # Run the tests.
-execute time bats --tap $TESTS
+execute time bats -j 4 --tap $TESTS