Merge pull request #75 from buildkite-plugins/toote_ignore_missing

Add option to ignore missing files
buildkite-plugins · Oct 24, 2022 · aff21d6 · aff21d6
2 parents 884b0aa + 3cfbd16
commit aff21d6
Show file tree

Hide file tree

Showing 7 changed files with 460 additions and 63 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,7 +1,7 @@
 steps:
   - label: "🤡 :hammer:"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.5.0:
           run: tests
   - label: ":sparkles: lint"
     plugins:

diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@ This functionality duplicates the [artifact_paths](https://buildkite.com/docs/pi
 steps:
   - command: ...
     plugins:
-    - artifacts#v1.5.0:
+    - artifacts#v1.7.0:
         upload: "log/**/*.log"
 ```
 
@@ -20,7 +20,7 @@ or
 steps:
   - command: ...
     plugins:
-    - artifacts#v1.5.0:
+    - artifacts#v1.7.0:
         upload: [ "log/**/*.log", "debug/*.error" ]
 ```
 
@@ -30,7 +30,7 @@ or
 steps:
   - command: ...
     plugins:
-    - artifacts#v1.5.0:
+    - artifacts#v1.7.0:
         upload: 
           from: log1.log
           to: log2.log
@@ -42,7 +42,7 @@ or
 steps:
   - command: ...
     plugins:
-    - artifacts#v1.5.0:
+    - artifacts#v1.7.0:
         upload: 
         - from: log1.log
           to: log2.log
@@ -58,7 +58,7 @@ eg: uploading a public file when using S3
 steps:
   - command: ...
     plugins:
-    - artifacts#v1.5.0:
+    - artifacts#v1.7.0:
         upload: "coverage-report/**/*"
         s3-upload-acl: public-read
 ```
@@ -68,7 +68,7 @@ eg: uploading a private file when using GS
 steps:
   - command: ...
     plugins:
-    - artifacts#v1.5.0:
+    - artifacts#v1.7.0:
         upload: "coverage-report/**/*"
         gs-upload-acl: private
 ```
@@ -81,7 +81,7 @@ This downloads artifacts matching globs to the local filesystem. See [downloadin
 steps:
   - command: ...
     plugins:
-      - artifacts#v1.5.0:
+      - artifacts#v1.7.0:
           download: "log/**/*.log"
 ```
 
@@ -91,7 +91,7 @@ or
 steps:
   - command: ...
     plugins:
-      - artifacts#v1.5.0:
+      - artifacts#v1.7.0:
           download: [ "log/**/*.log", "debug/*.error" ]
 ```
 
@@ -101,7 +101,7 @@ or
 steps:
   - command: ...
     plugins:
-      - artifacts#v1.5.0:
+      - artifacts#v1.7.0:
           download: 
             from: log1.log
             to: log2.log
@@ -113,7 +113,7 @@ or
 steps:
   - command: ...
     plugins:
-      - artifacts#v1.5.0:
+      - artifacts#v1.7.0:
           download: 
           - from: log1.log
             to: log2.log
@@ -149,7 +149,7 @@ When uploading, globs specified in the `upload` option will be compressed in a s
 steps:
   - command: ...
     plugins:
-    - artifacts#v1.5.0:
+    - artifacts#v1.7.0:
         upload: "log/*.log"
         compressed: logs.zip
 ```
@@ -160,11 +160,14 @@ When downloading, this option states the actual name of the artifact to be downl
 steps:
   - command: ...
     plugins:
-      - artifacts#v1.5.0:
+      - artifacts#v1.7.0:
           download: "log/*.log"
           compressed: logs.tgz
 ```
 
+### `ignore-missing` (optional, boolean)
+
+If set to `true`, it will ignore errors caused when calling `buildkite-agent artifact` to prevent failures if you expect artifacts not to be present in some situations.
 
 ### Relocation
 

diff --git a/hooks/post-command b/hooks/post-command
@@ -60,6 +60,36 @@ workdir=${BUILDKITE_PLUGIN_ARTIFACTS_WORKDIR:-.}
 pushd "${workdir}"
 trap popd EXIT
 
+bk_agent() {
+  if ! buildkite-agent artifact "${@}"; then
+    if [[ "${BUILDKITE_PLUGIN_ARTIFACTS_IGNORE_MISSING:-"false"}" != "false" ]]; then
+      echo "Ignoring error in upload of" "${@: -1}"
+    else
+      echo "Error in upload of" "${@:-1}"
+      exit 1
+    fi
+  fi
+}
+
+handle_relocation() {
+  local index="$1"
+  local var_string=""
+  if [[ -n "${index}" ]]; then
+    var_string="${index}_"
+  fi
+
+  source_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${var_string}FROM"
+  dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${var_string}TO"
+
+  if [[ -n "${!source_env_var:-}" ]] && [[ -n "${!dest_env_var:-}" ]]; then
+    if [[ -e ${!source_env_var} ]]; then
+      echo "~~~ Moving [${!source_env_var}] to [${!dest_env_var}]..."
+      mv "${!source_env_var}" "${!dest_env_var}"
+    elif [[ ${BUILDKITE_PLUGIN_ARTIFACTS_IGNORE_MISSING:-"false"} == "true" ]]; then
+      echo "Ignoring missing file ${!source_env_var} for relocation"
+    fi
+  fi
+}
 
 # Set user-provided object ACL for AWS S3
 if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_S3_UPLOAD_ACL:-}" ]] ; then
@@ -81,10 +111,7 @@ fi
 
 if [[ "${SINGULAR_UPLOAD_OBJECT}" == "true" ]]; then
   if [[ "${RELOCATION}" == "true" ]]; then
-    if [[ -e "${BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_FROM:-}" ]]; then
-      echo "~~~ Moving [${BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_FROM}] to [${BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_TO}]..."
-      mv "${BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_FROM}" "${BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_TO}"
-    fi
+    handle_relocation ""
     path="${BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_TO}"
   else
     path="${paths[0]}"
@@ -97,18 +124,14 @@ if [[ "${SINGULAR_UPLOAD_OBJECT}" == "true" ]]; then
   fi
 
   echo "~~~ Uploading artifacts ${EXTRA_MESSAGE}"
-  buildkite-agent artifact "${args[@]}" "${path}"
+  bk_agent "${args[@]}" "${path}"
 elif [[ "${COMPRESSED}" == "true" ]]; then
   final_paths=()
   index=0
   for path in "${paths[@]}"; do
-    source_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${index}_FROM"
+    handle_relocation "${index}"
     dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${index}_TO"
-    if [[ -n "${!source_env_var:-}" ]] && [[ -n "${!dest_env_var:-}" ]]; then
-      if [[ -e ${!source_env_var} ]]; then
-        echo "~~~ Moving [${!source_env_var}] to [${!dest_env_var}]..."
-        mv ${!source_env_var} ${!dest_env_var}
-      fi
+    if [[ -n "${!dest_env_var:-}" ]]; then
       path="${!dest_env_var}"
     fi
 
@@ -120,23 +143,19 @@ elif [[ "${COMPRESSED}" == "true" ]]; then
   "${compress[@]}" "${final_paths[@]}"
 
   echo "~~~ Uploading artifacts ${EXTRA_MESSAGE}"
-  buildkite-agent artifact "${args[@]}" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
+  bk_agent "${args[@]}" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
 elif [[ "${MULTIPLE_UPLOADS}" == "true" ]]; then
   index=0
   echo "~~~ Uploading artifacts ${EXTRA_MESSAGE}"
 
   for path in "${paths[@]}"; do
-    source_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${index}_FROM"
+    handle_relocation "${index}"
     dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${index}_TO"
-    if [[ -n "${!source_env_var:-}" ]] && [[ -n "${!dest_env_var:-}" ]]; then
-      if [[ -e ${!source_env_var} ]]; then
-        echo "~~~ Moving [${!source_env_var}] to [${!dest_env_var}]..."
-        mv ${!source_env_var} ${!dest_env_var}
-      fi
+    if [[ -n "${!dest_env_var:-}" ]]; then
       path="${!dest_env_var}"
     fi
 
-    buildkite-agent artifact "${args[@]}" "$path"
+    bk_agent "${args[@]}" "$path"
     ((index+=1))
   done
 fi
diff --git a/hooks/pre-command b/hooks/pre-command
@@ -46,6 +46,42 @@ if [[ "${#paths[@]}" -le 0 ]]; then
   exit 0
 fi
 
+bk_agent() {
+  if ! buildkite-agent artifact "${@}"; then
+    if [[ "${BUILDKITE_PLUGIN_ARTIFACTS_IGNORE_MISSING:-"false"}" != "false" ]]; then
+      echo "Ignoring error in download of" "${@: -2}"
+    else
+      echo "Error in download of" "${@: -2}"
+      return 1
+    fi
+  fi
+
+  return 0
+}
+
+handle_relocation() {
+  local index="$1"
+  local var_string=""
+  if [[ -n "${index}" ]]; then
+    var_string="${index}_"
+  fi
+
+  source_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${var_string}FROM"
+  dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${var_string}TO"
+  if [[ -n "${!dest_env_var:-}" ]]; then
+    if ! [[ -d $(dirname "${!dest_env_var}") ]]; then
+      mkdir -p "$(dirname "${!dest_env_var}")"
+    fi
+
+    if [[ ! -e "${!source_env_var}" ]] && [[ ${BUILDKITE_PLUGIN_ARTIFACTS_IGNORE_MISSING:-"false"} != "false" ]]; then
+      echo "Ignoring missing file ${!source_env_var} for relocation"
+    else
+      echo "~~~ Moving [${!source_env_var}] to [${!dest_env_var}]..."
+      mv "${!source_env_var}" "${!dest_env_var}"
+    fi
+  fi
+}
+
 if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED:-}" ]]; then
   COMPRESSED="true"
   if [[ "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" =~ .*\.zip ]]; then
@@ -68,34 +104,20 @@ workdir="${BUILDKITE_PLUGIN_ARTIFACTS_WORKDIR:-.}"
 
 if [[ "${COMPRESSED}" == "true" ]]; then
   echo "~~~ Downloading artifacts ${EXTRA_MESSAGE}"
-  buildkite-agent artifact "${args[@]}" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" "${workdir}"
-
+  bk_agent "${args[@]}" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" "${workdir}"
 
   echo "~~~ Uncompressing ${paths[*]} from ${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
   "${compress[@]}" "${paths[@]}"
 
   # single relocation
   if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO:-}" ]]; then
-    if ! [[ -d $(dirname "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO}") ]]; then
-        mkdir -p "$(dirname "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO}")"
-    fi
-    echo "~~~ Moving [${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM}] to [${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO}]..."
-    mv "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM}" "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO}"
+    handle_relocation ""
   fi
 
   # multiple relocations
   index=0
   for path in "${paths[@]}"; do
-    source_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${index}_FROM"
-    dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${index}_TO"
-    if [[ -n "${!dest_env_var:-}" ]]; then
-      if ! [[ -d $(dirname "${!dest_env_var}") ]]; then
-        mkdir -p "$(dirname "${!dest_env_var}")"
-      fi
-      echo "~~~ Moving [${!source_env_var}] to [${!dest_env_var}]..."
-      mv "${!source_env_var}" "${!dest_env_var}"
-    fi
-
+    handle_relocation "${index}"
     ((index+=1))
   done
 
@@ -107,14 +129,10 @@ elif [[ "${SINGULAR_DOWNLOAD_OBJECT}" == "true" ]]; then
   fi
 
   echo "~~~ Downloading artifacts ${EXTRA_MESSAGE}"
-  buildkite-agent artifact "${args[@]}" "${source}" "${workdir}"
+  bk_agent "${args[@]}" "${source}" "${workdir}"
 
   if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO:-}" ]]; then
-    if ! [[ -d $(dirname "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO}") ]]; then
-        mkdir -p "$(dirname "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO}")"
-    fi
-    echo "~~~ Moving [${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM}] to [${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO}]..."
-    mv "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM}" "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO}"
+    handle_relocation ""
   fi
 elif [[ "${MULTIPLE_DOWNLOADS}" == "true" ]]; then
   index=0
@@ -128,14 +146,8 @@ elif [[ "${MULTIPLE_DOWNLOADS}" == "true" ]]; then
     else
       source="${path}"
     fi
-    buildkite-agent artifact "${args[@]}" "${source}" "${workdir}"
-    if [[ -n "${!dest_env_var:-}" ]]; then
-      if ! [[ -d $(dirname "${!dest_env_var}") ]]; then
-        mkdir -p "$(dirname "${!dest_env_var}")"
-      fi
-      echo "~~~ Moving [${source}] to [${!dest_env_var}]..."
-      mv "${source}" "${!dest_env_var}"
-    fi
+    bk_agent "${args[@]}" "${source}" "${workdir}"
+    handle_relocation "${index}"
     ((index+=1))
   done
 fi
diff --git a/plugin.yml b/plugin.yml
@@ -47,6 +47,8 @@ configuration:
     compressed:
       type: string
       pattern: '.*(.zip|.tgz)$'
+    ignore-missing:
+      type: boolean
   oneOf:
     - required:
       - upload