diff --git a/CHANGELOG.md b/CHANGELOG.md index 098b4ba..e93e324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Release Notes +## 2.0.0-beta1 + +- Use a custom `in` script instead of the script provided by the embedded [git-resource](https://github.com/concourse/git-resource). + This allows us to efficiently fetch shallow clones at exactly a specified + revision without incurring needless `git fetch --deepen` roundtrips. + +Note: this release can break configuration for the `get` step in your pipelines as +it removes configuration options inherited from `git-resource` that don't make +sense in the context of gate-resource anymore. This helps to achieve better `get` performance and is critical for large gate repository. + +Please review the [README](./README.md) for updated configuration options. + ## 1.1.1 - Patch an issue with shallow-clones not fetching at the correct depth in git-resource. diff --git a/Dockerfile b/Dockerfile index 676f6e7..dd9e72c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,6 @@ FROM concourse/git-resource:1.7.0 as resource RUN mv /opt/resource /opt/git-resource -# patch the git-resource deepen script with our fixes, see https://github.com/concourse/git-resource/pull/316 -COPY git-resource/deepen_shallow_clone_until_ref_is_found_then_check_out /opt/git-resource/deepen_shallow_clone_until_ref_is_found_then_check_out - ADD assets/ /opt/resource/ RUN chmod +x /opt/resource/* diff --git a/Makefile b/Makefile index ec5b4ae..d8d8fcc 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ user=meshcloud name=gate-resource image=$(user)/$(name) -tag=1.1.1 +tag=2.0.0-beta1 docker=docker dockerfile = Dockerfile diff --git a/README.md b/README.md index 8589066..d9e199f 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,9 @@ Since concourse detects that the `none` version already exists after the first t ## Source Configuration -* `git`: *Required.* Configuration of the repository. See [git-resource](https://github.com/concourse/git-resource) for options. - * **note:** the `paths` and `ignore_path` parameters are not supported - +* `git`: *Required.* Configuration of the repository. Supports the following subset of [git-resource](https://github.com/concourse/git-resource) configuration options (review the link for descriptions) + * *Required*: `uri`, `branch`, `private_key` + * *Optional*: `git_config`, `short_ref_format`, * `gate`: *Optional.* The gate to track. ## Behavior @@ -77,6 +77,9 @@ Outputs 2 files: * `metadata`: Contains the contents of whatever was in your gate item. This is useful for environment configuration settings or documenting approval workflows. +> note: the git repository cloned during `in` is a shallow clone and does not track an upstream branch. +> This is not a problem when using the `out` step to create or update gates. + ### `out`: Pass an item through a gate Performs one of the following actions to change the state of a gate. diff --git a/assets/git-in b/assets/git-in new file mode 100755 index 0000000..14f2dab --- /dev/null +++ b/assets/git-in @@ -0,0 +1,95 @@ +#!/bin/bash +# vim: set ft=sh + +set -e + +exec 3>&1 # make stdout available as fd 3 for the result +exec 1>&2 # redirect all output to stderr for logging + +source /opt/git-resource/common.sh + +destination=$1 + +if [ -z "$destination" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# for jq +PATH=/usr/local/bin:$PATH + +bin_dir="${0%/*}" +if [ "${bin_dir#/}" == "$bin_dir" ]; then + bin_dir="$PWD/$bin_dir" +fi + +payload=$(mktemp $TMPDIR/git-resource-request.XXXXXX) + +cat > $payload <&0 + +load_pubkey $payload +configure_https_tunnel $payload +configure_git_ssl_verification $payload +configure_credentials $payload + +uri=$(jq -r '.source.uri // ""' < $payload) +git_config_payload=$(jq -r '.source.git_config // []' < $payload) +ref=$(jq -r '.version.ref // "HEAD"' < $payload) +depth=$(jq -r '(.params.depth // 0)' < $payload) +short_ref_format=$(jq -r '(.params.short_ref_format // "%s")' < $payload) + +configure_git_global "${git_config_payload}" + +if [ -z "$uri" ]; then + echo "invalid payload (missing uri):" >&2 + cat $payload >&2 + exit 1 +fi + +depthflag="" +if test "$depth" -gt 0 2> /dev/null; then + depthflag="--depth $depth" +fi + +git init $destination +cd $destination + +git remote add origin $uri +git fetch origin "$ref" $depthflag + +# this will set master to the current commit +git reset --hard FETCH_HEAD + +git log -1 --oneline +git clean --force --force -d + +if [ "$ref" == "HEAD" ]; then + return_ref=$(git rev-parse HEAD) +else + return_ref=$ref +fi + +# Store committer email in .git/committer. Can be used to send email to last committer on failed build +# Using https://github.com/mdomke/concourse-email-resource for example +git --no-pager log -1 --pretty=format:"%ae" > .git/committer + +# Store git-resource returned version ref .git/ref. Useful to know concourse +# pulled ref in following tasks and resources. +echo "${return_ref}" > .git/ref + +# Store short ref with templating. Useful to build Docker images with +# a custom tag +echo "${return_ref}" | cut -c1-7 | awk "{ printf \"${short_ref_format}\", \$1 }" > .git/short_ref + +# Store commit message in .git/commit_message. Can be used to inform about +# the content of a successfull build. +# Using https://github.com/cloudfoundry-community/slack-notification-resource +# for example +git log -1 --format=format:%B > .git/commit_message + +metadata=$(git_metadata) + +jq -n "{ + version: {ref: $(echo $return_ref | jq -R .)}, + metadata: $metadata +}" >&3 diff --git a/assets/in b/assets/in index 28c280c..f229362 100755 --- a/assets/in +++ b/assets/in @@ -35,14 +35,12 @@ else "source": '$git_source', "version": '$version', "params": { - "disable_git_lfs": true, - "submodules": "none", "depth": 2 } }' | jq -r) # forward to git-resource to let it fetch the repository - echo "$git_payload" | /opt/git-resource/in $destination + echo "$git_payload" | /opt/resource/git-in $destination cd $destination diff --git a/assets/out b/assets/out index dcbb959..f366a65 100755 --- a/assets/out +++ b/assets/out @@ -30,7 +30,6 @@ jq -n "{ }" > $git_source_payload load_pubkey $git_source_payload -load_git_crypt_key $git_source_payload configure_https_tunnel $git_source_payload configure_git_ssl_verification $git_source_payload configure_credentials $git_source_payload @@ -116,7 +115,8 @@ try_autoclose() { echo "$dashed_line" } -# clone the repository. by default we only need a single branch and a single +# clone the repository at its current latest commit - we want to make more commits on top. +# by default we only need to fetch a single branch and the latest commit repository=$(mktemp -d "$TMPDIR/gate-resource-repo.XXXXXX") git clone $uri --branch $branch $repository --depth=1 diff --git a/git-resource/deepen_shallow_clone_until_ref_is_found_then_check_out b/git-resource/deepen_shallow_clone_until_ref_is_found_then_check_out deleted file mode 100755 index 7f7a373..0000000 --- a/git-resource/deepen_shallow_clone_until_ref_is_found_then_check_out +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# vim: set ft=sh - -set -e - -readonly max_depth=127 - -declare depth="$1" -readonly ref="$2" -readonly tagflag="$3" - -declare total_depth="$depth" -declare depth_from_ref=$((depth - 1)) - -# A shallow clone may not contain the Git commit $ref: -# 1. The depth of the shallow clone is measured backwards from the latest -# commit on the given head (master or branch), and in the meantime there may -# have been more than $depth commits pushed on top of our $ref. -# 2. If there's a path filter (`paths`/`ignore_paths`), then there may be more -# than $depth such commits pushed to the head (master or branch) on top of -# $ref that are not affecting the filtered paths. -# -# In either case we try to deepen the shallow clone until we find $ref, reach -# the max depth of the repo, or give up after a given depth and resort to deep -# clone. - -git_dir="$(git rev-parse --git-dir)" -readonly git_dir - -while ! git checkout -q "$ref~$depth_from_ref" &>/dev/null; do - # once the depth of a shallow clone reaches the max depth of the origin - # repo, Git silenty turns it into a deep clone - if [ ! -e "$git_dir"/shallow ]; then - echo "Reached max depth of the origin repo while deepening the shallow clone, it's a deep clone now" - break - fi - - echo "Could not find ref ${ref}~${depth_from_ref} in a shallow clone of depth ${total_depth}" - - (( depth *= 2 )) - - if [ "$total_depth" -ge "$max_depth" ]; then - echo "Reached depth threshold ${max_depth}, falling back to deep clone..." - git fetch --unshallow origin $tagflag - - break - fi - - echo "Deepening the shallow clone by an additional ${depth}..." - git fetch --deepen "$depth" origin $tagflag - (( total_depth += depth )) -done - -git checkout -q "$ref" diff --git a/test/helpers.sh b/test/helpers.sh index 66bc5a3..56b795b 100644 --- a/test/helpers.sh +++ b/test/helpers.sh @@ -148,7 +148,8 @@ get_gate_at_ref() { jq -n "{ source: { git: { - uri: $(echo file://$uriPath | jq -R .) + uri: $(echo file://$uriPath | jq -R .), + branch: \"master\" }, gate: $(echo $gate | jq -R .) }, diff --git a/test/in.sh b/test/in.sh index a14b861..1f47824 100755 --- a/test/in.sh +++ b/test/in.sh @@ -12,6 +12,10 @@ it_can_get_from_regular_version() { local gate_ref=$(make_commit_to_file $repo "$gate/$item") + echo "xxxx" + git -C "$repo" branch -a + echo "xxxx" + result=$(get_gate_at_ref "$repo" "$gate_ref" "$gate" "$dest") echo "$result" | jq -e ' .version == { "ref": "'$gate_ref'" }