diff --git a/.gitignore b/.gitignore index 0eb4a4d..16a4a81 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +builds tar-scrubber diff --git a/.go-env.sh b/.go-env.sh new file mode 100755 index 0000000..570e829 --- /dev/null +++ b/.go-env.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +dir="$(dirname "$BASH_SOURCE")" +dir="$(readlink -ve "$dir")" + +user="$(id -u):$(id -g)" +args=( + --interactive --rm --init + --user "$user" + --mount "type=bind,src=$dir,dst=/app" + --workdir /app + --tmpfs /tmp,exec + --env HOME=/tmp + + # "go mod" cache is stored in /go/pkg/mod/cache + --env GOPATH=/go + --mount type=volume,src=doi-meta-gopath,dst=/go + --env GOCACHE=/go/.cache + + --env "CGO_ENABLED=${CGO_ENABLED-0}" + --env "GOTOOLCHAIN=${GOTOOLCHAIN-local}" + --env GOFLAGS + --env GOOS --env GOARCH + --env GO386 + --env GOAMD64 + --env GOARM +) +if [ -t 0 ] && [ -t 1 ]; then + args+=( --tty ) +fi +go="$(awk '$1 == "go" { print $2; exit }' "$dir/go.mod")" +if [[ "$go" == *.*.* ]]; then + go="${go%.*}" # strip to just X.Y +fi +args+=( + "golang:$go" + "$@" +) +set -x +exec docker run "${args[@]}" diff --git a/builds.sh b/builds.sh index 0d231e8..77f3d1c 100755 --- a/builds.sh +++ b/builds.sh @@ -1,228 +1,19 @@ #!/usr/bin/env bash set -Eeuo pipefail -json="$1" - +# TODO drop this from the defaults and set it explicitly in DOI instead (to prevent accidents) : "${BASHBREW_STAGING_TEMPLATE:=oisupport/staging-ARCH:BUILD}" export BASHBREW_STAGING_TEMPLATE -shell="$(jq -r ' - [ - "set -- \(keys_unsorted | map(@sh) | join(" "))", - "declare -A sources=(", - ( - to_entries[] - | "\t[\(.key | @sh)]=\(.value | tojson | @sh)" - ), - ")" - ] | join("\n") -' "$json")" -eval "$shell" - -_resolveRemoteArch() { - local img="$1"; shift - - local arches - if ! arches="$(bashbrew remote arches --json "$img" 2>/dev/null)"; then # TODO somehow differentiate errors like 404 / 403, "insufficient_scope" from other bashbrew errors here so we can stop eating stderr - printf 'null' - return - fi - - echo "$arches" -} - -# resolve an image reference to an architecture-specific imageId (digest of the image manifest) -_resolve() { - local img="$1"; shift - local arch="$1"; shift - local bashbrewRemoteArches="$1"; shift - - if [ "$bashbrewRemoteArches" = 'null' ]; then - printf 'null' - return - fi - - jq <<<"$bashbrewRemoteArches" -c --arg arch "$arch" --arg img "$img" ' - if .arches | has($arch) then - ($img | sub("@[^@]+$"; "")) as $base - | { - # ref + descriptor of the arch-specific manifest - manifest: ( - # TODO warn/error on multiple entries for $arch? - .arches[$arch][0] - | { - ref: ($base + "@" + .digest), - desc: ., - } - ), - # ref + descriptor of the index - index: { - ref: ($base + "@" + .desc.digest), - desc: .desc, - }, - } - | if .index.desc.digest == .manifest.desc.digest then - del(.index) - else . end - else null end - ' || exit 1 -} - -declare -A imageArchResolved=( - #["image"]="{...}" # JSON result of bashbrew remote arches -) - -# for each sourceId, try to calculate a buildId, then do a registry lookup to get an imageId -# then, anything that has a calculated buildId but *not* an imageId is something that needs a build -# (and each buildId needs to include the imageIds of all the parent images -- without those, the buildId is invalid / impossible to calculate, which forces us to build everything in order) -declare -A sourceArchResolved=( - #["$sourceId-$arch"]="xxx/staging-xxx:$buildId@sha256:xxx" -) -builds='{}' -for sourceId; do - obj="${sources["$sourceId"]}" - - shell="$(jq <<<"$obj" -r ' - [ - "tag=\(.allTags[0] | @sh)", - ( .arches | - "arches=( \(keys_unsorted | map(@sh) | join(" ")) )", - "declare -A archObjs=(", - ( - to_entries[] - | "\t[\(.key | @sh)]=\(.value | tojson | @sh)" - ), - ")" - ) - ] | join("\n") - ')" - eval "$shell" - - printf >&2 '%s (%s):\n' "$sourceId" "$tag" - - for arch in "${arches[@]}"; do - archObj="${archObjs["$arch"]}" - - printf >&2 ' -> %s: ' "$arch" - - buildIdParts="$(jq -nc --arg sourceId "$sourceId" --arg arch "$arch" ' - { - sourceId: $sourceId, - arch: $arch, - parents: {}, - - # this is included for data tracking purposes, but is not part of the final "buildId" calculation like the above fields are - resolvedParents: {}, - # (we only include parent descriptor in the buildId so that EOL parents do not cause a build cache bust) - } - ')" - - shell="$(jq <<<"$archObj" -r ' - [ - "parents=(", - ( - .parents - | to_entries[] - | select(.key != "scratch") - | { from: .key } + .value - | tojson | @sh - | "\t" + . - ), - ")" - ] | join("\n") - ')" - eval "$shell" - - missingParents=0 - for parent in "${parents[@]}"; do - lookup="$(jq <<<"$parent" -r ' - if .sourceId then - .sourceId - elif .pin then - .from + "@" + .pin - else - # cases like "FROM alpine:3.11" will fall back here (unsupported/deprecated/"naughty" base images) - .from - end - ')" - - # if "$lookup" is a valid/known sourceId, we should look up the (pre-resolved) imageId for it - if [ -n "$lookup" ] && [ -n "${sources["$lookup"]:+x}" ]; then - resolved="${sourceArchResolved["$lookup-$arch"]:-}" - elif [ -n "$lookup" ]; then - if [ -z "${imageArchResolved["$lookup"]:+x}" ]; then - remoteArches="$(_resolveRemoteArch "$lookup")" - imageArchResolved["$lookup"]="$remoteArches" - fi - - resolved="$(_resolve "$lookup" "$arch" "${imageArchResolved["$lookup"]}")" - else - resolved= - fi - : "${resolved:=null}" - - buildIdParts="$( - jq <<<"$buildIdParts" -c \ - --argjson parent "$parent" \ - --argjson resolved "$resolved" \ - ' - .resolvedParents[$parent.from] = $resolved - | .parents[$parent.from] = $resolved.manifest?.desc?.digest? - ' - )" - - if [ "$resolved" = 'null' ] || [ -z "$resolved" ]; then - (( missingParents++ )) || : - fi - done - - # if we're missing *any* parents, we cannot have a buildId - if [ "$missingParents" = 0 ]; then - buildIdJson="$(jq <<<"$buildIdParts" -c 'del(.resolvedParents)')" # see notes above (where buildIdParts is first defined) - buildId="$(sha256sum <<<"$buildIdJson" | cut -d' ' -f1)" # see notes above (where buildIdParts is first defined) - printf >&2 '%s\n' "$buildId" - - img="$BASHBREW_STAGING_TEMPLATE" - img="${img//BUILD/$buildId}" - [ "$img" != "$BASHBREW_STAGING_TEMPLATE" ] # BUILD is required, for proper uniqueness (ARCH is optional) - img="${img//ARCH/$arch}" - - if [ -z "${imageArchResolved["$img"]:+x}" ]; then - remoteArches="$(_resolveRemoteArch "$img")" - imageArchResolved["$img"]="$remoteArches" - fi - - resolved="$(_resolve "$img" "$arch" "${imageArchResolved["$img"]}")" - : "${resolved:=null}" - sourceArchResolved["$sourceId-$arch"]="$resolved" - builds="$( - jq <<<"$builds" \ - --argjson source "$obj" \ - --argjson buildIdParts "$buildIdParts" \ - --arg buildId "$buildId" \ - --arg img "$img" \ - --argjson resolved "$resolved" \ - ' - # TODO error out if $buildId already exists somehow (should be impossible) - .[$buildId] = { - buildId: $buildId, - build: ({ - img: $img, - resolved: $resolved, - } + $buildIdParts), - source: ( - $source - | .arches |= ( - (keys_unsorted - [ $buildIdParts.arch ]) as $otherArches - | del(.[$otherArches[]]) - ) - ), - } - ' - )" - else - printf >&2 'not yet!\n' - fi - done -done -jq <<<"$builds" . +dir="$(dirname "$BASH_SOURCE")" +dir="$(readlink -ve "$dir")" +if [ "$dir/builds.go" -nt "$dir/builds" ] || [ "$dir/.go-env.sh" -nt "$dir/builds" ]; then + { + echo "building '$dir/builds' from 'builds.go'" + "$dir/.go-env.sh" go build -v -o builds builds.go + ls -l "$dir/builds" + } >&2 +fi +[ -x "$dir/builds" ] + +"$dir/builds" "$@" | jq . diff --git a/sources.sh b/sources.sh index ab70b4d..3aa2e85 100755 --- a/sources.sh +++ b/sources.sh @@ -7,6 +7,7 @@ fi # TODO do this for oisupport too! (without arch namespaces; just straight into/from the staging repos) +# TODO drop this from the defaults and set it explicitly in DOI instead (to prevent accidents) defaultArchNamespaces=' amd64 = amd64, arm32v5 = arm32v5, @@ -19,29 +20,16 @@ defaultArchNamespaces=' riscv64 = riscv64, s390x = s390x, windows-amd64 = winamd64 -' # TODO +' : "${BASHBREW_ARCH_NAMESPACES=$defaultArchNamespaces}" export BASHBREW_ARCH_NAMESPACES dir="$(dirname "$BASH_SOURCE")" dir="$(readlink -ve "$dir")" -if [ "$dir/tar-scrubber.go" -nt "$dir/tar-scrubber" ]; then - # TODO this should probably live somewhere else (bashbrew?) +if [ "$dir/tar-scrubber.go" -nt "$dir/tar-scrubber" ] || [ "$dir/.go-env.sh" -nt "$dir/tar-scrubber" ]; then { echo "building '$dir/tar-scrubber' from 'tar-scrubber.go'" - user="$(id -u):$(id -g)" - args=( - --rm - --user "$user" - --mount "type=bind,src=$dir,dst=/app" - --workdir /app - --tmpfs /tmp - --env HOME=/tmp - --env CGO_ENABLED=0 - golang:1.20 - go build -v -o tar-scrubber tar-scrubber.go - ) - docker run "${args[@]}" + "$dir/.go-env.sh" go build -v -o tar-scrubber tar-scrubber.go ls -l "$dir/tar-scrubber" } >&2 fi