Optional failure for a tile that is already staged

ci/pcf-pipelines/pipeline.yml

@@ -193,7 +193,11 @@ resources:
   type: pool
   source:
     pool: vsphere
-    <<: *norm-ci-locks-params
+    <<: &norm-ci-locks-params
+      uri: [email protected]:pivotal-cf/norm-ci-locks.git
+      branch: master
+      private_key: {{locks_git_ssh_key}}
+      retry_delay: 1m
 
 - name: vsphere-offline-lock
   type: pool
@@ -312,7 +316,10 @@ jobs:
 
 - name: test
   on_failure:
-    <<: *notify_slack
+    <<: &notify_slack
+      put: slack
+      params:
+        text: "$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME failed: $ATC_EXTERNAL_URL/builds/$BUILD_ID"
   plan:
   - get: pcf-pipelines
     trigger: true
@@ -428,7 +435,11 @@ jobs:
   - task: destroy-rc-vsphere-install
     params:
       PIPELINE_NAME: rc-vsphere-install
-      <<: *vsphere_atc_creds
+      <<: &vsphere_atc_creds
+        ATC_EXTERNAL_URL: {{atc_external_url}}
+        ATC_BASIC_AUTH_USERNAME: {{fly_basic_auth_username}}
+        ATC_BASIC_AUTH_PASSWORD: {{fly_basic_auth_password}}
+        ATC_TEAM_NAME: vsphere
     file: pcf-pipelines/ci/tasks/destroy_pipeline.yml
 
 - name: set-rc-vsphere-install
@@ -560,7 +571,11 @@ jobs:
   - task: destroy-rc-gcp-install
     params:
       PIPELINE_NAME: rc-gcp-install
-      <<: *atc_creds
+      <<: &atc_creds
+        ATC_EXTERNAL_URL: {{atc_external_url}}
+        ATC_BASIC_AUTH_USERNAME: {{fly_basic_auth_username}}
+        ATC_BASIC_AUTH_PASSWORD: {{fly_basic_auth_password}}
+        ATC_TEAM_NAME: {{atc_team_name}}
     file: pcf-pipelines/ci/tasks/destroy_pipeline.yml
 
 - name: set-rc-gcp-install
@@ -1452,27 +1467,4 @@ jobs:
       params: {release: vsphere-offline-lock}
   on_failure:
     <<: *notify_slack
-####### END VSPHERE OFFLINE
-
-atc_creds: &atc_creds
-  ATC_EXTERNAL_URL: {{atc_external_url}}
-  ATC_BASIC_AUTH_USERNAME: {{fly_basic_auth_username}}
-  ATC_BASIC_AUTH_PASSWORD: {{fly_basic_auth_password}}
-  ATC_TEAM_NAME: {{atc_team_name}}
-
-vsphere_atc_creds: &vsphere_atc_creds
-  ATC_EXTERNAL_URL: {{atc_external_url}}
-  ATC_BASIC_AUTH_USERNAME: {{fly_basic_auth_username}}
-  ATC_BASIC_AUTH_PASSWORD: {{fly_basic_auth_password}}
-  ATC_TEAM_NAME: vsphere
-
-notify_slack: &notify_slack
-  put: slack
-  params:
-    text: "$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME failed: $ATC_EXTERNAL_URL/builds/$BUILD_ID"
-
-norm-ci-locks-params: &norm-ci-locks-params
-  uri: [email protected]:pivotal-cf/norm-ci-locks.git
-  branch: master
-  private_key: {{locks_git_ssh_key}}
-  retry_delay: 1m
+####### END VSPHERE OFFLINE

install-pcf/aws/pipeline.yml

@@ -249,6 +249,7 @@ jobs:
       OPSMAN_CLIENT_SECRET: ""
       OPSMAN_USERNAME: {{opsman_admin_username}}
       OPSMAN_PASSWORD: {{opsman_admin_password}}
+      SKIP_IF_ALREADY_STAGED: no
 
 - name: deploy-ert
   serial_groups: [opsman]

install-pcf/azure/pipeline.yml

@@ -345,6 +345,7 @@ jobs:
       OPSMAN_CLIENT_SECRET: ""
       OPSMAN_USERNAME: {{opsman_admin_username}}
       OPSMAN_PASSWORD: {{opsman_admin_password}}
+      SKIP_IF_ALREADY_STAGED: no
 
 ###########################
 ## Job - Configure ERT   ##

install-pcf/gcp/pipeline.yml

@@ -238,6 +238,7 @@ jobs:
       OPSMAN_PASSWORD: {{opsman_admin_password}}
       OPSMAN_CLIENT_ID: ""
       OPSMAN_CLIENT_SECRET: ""
+      SKIP_IF_ALREADY_STAGED: no
 
 
 - name: deploy-ert

install-pcf/openstack/pipeline.yml

@@ -271,6 +271,7 @@ jobs:
       OPSMAN_PASSWORD: {{opsman_admin_password}}
       OPSMAN_CLIENT_ID: ""
       OPSMAN_CLIENT_SECRET: ""
+      SKIP_IF_ALREADY_STAGED: no
 
 
 - name: configure-ert

install-pcf/vsphere/pipeline.yml

@@ -210,6 +210,7 @@ jobs:
       OPSMAN_PASSWORD: {{opsman_admin_password}}
       OPSMAN_CLIENT_ID: ""
       OPSMAN_CLIENT_SECRET: ""
+      SKIP_IF_ALREADY_STAGED: no
 
 
 - name: deploy-ert

tasks/disable-all-errands/task.yml

@@ -0,0 +1,18 @@
+---
+platform: linux
+image_resource:
+  type: docker-image
+  source:
+    repository: cloudopsci/eu-ci
+    tag: stable
+run:
+  path: /bin/bash
+  args:
+  - -exc
+  - |-
+    #!/usr/bin/env bash
+    omen toggle-errands --action disable
+params:
+  OPSMAN_HOSTNAME:
+  OPSMAN_PASSWORD:
+  OPSMAN_USER:

tasks/enable-only-updated-tile-errands/task.yml

@@ -0,0 +1,20 @@
+---
+platform: linux
+image_resource:
+  type: docker-image
+  source:
+    repository: cloudopsci/eu-ci
+    tag: stable
+run:
+  path: /bin/bash
+  args:
+  - -exc
+  - |-
+    #!/usr/bin/env bash
+    omen toggle-errands --action disable
+    omen toggle-errands --action enable --products "${PRODUCT_NAME}"
+params:
+  OPSMAN_HOSTNAME:
+  OPSMAN_PASSWORD:
+  OPSMAN_USER:
+  PRODUCT_NAME:

tasks/stage-product/task.sh

@@ -51,10 +51,15 @@ UNSTAGED_PRODUCT=$(echo "$UNSTAGED_ALL" | jq \
 )
 
 # There should be only one such unstaged product.
-if [ "$(echo $UNSTAGED_PRODUCT | jq '. | length')" -ne "1" ]; then
-  echo "Need exactly one unstaged build for $PRODUCT_NAME version $desired_version"
+if [[ "$(echo $UNSTAGED_PRODUCT | jq '. | length')" -ne "1" ]]; then
+  echo "Need exactly one unstaged build for staging $PRODUCT_NAME version $desired_version"
   jq -n "$UNSTAGED_PRODUCT"
-  exit 1
+  if echo "$SKIP_IF_ALREADY_STAGED" | egrep -iq '(true|yes|y|t|1)' ; then
+    echo "Skipping staging for $PRODUCT_NAME version $desired_version"
+    exit 0
+  else
+    exit 1
+  fi
 fi
 
 full_version=$(echo "$UNSTAGED_PRODUCT" | jq -r '.[].product_version')

tasks/stage-product/task.yml

@@ -31,6 +31,7 @@ params:
   OPSMAN_USERNAME:
   OPSMAN_PASSWORD:
   OPSMAN_DOMAIN_OR_IP_ADDRESS:
+  SKIP_IF_ALREADY_STAGED:
 
 run:
   path: pcf-pipelines/tasks/stage-product/task.sh

tasks/upload-product-and-stemcell/task.sh

@@ -51,6 +51,8 @@ if [ -n "$STEMCELL_VERSION" ]; then
         '
         if any(.Dependencies[]; select(.Release.Product.Name | contains("Stemcells for PCF (Windows)"))) then
           "stemcells-windows-server"
+        elif any(.Dependencies[]; select(.Release.Product.Name | contains("Xenial"))) then
+          "stemcells-ubuntu-xenial"
         else
           "stemcells"
         end

upgrade-buildpacks/pipeline.yml

@@ -146,7 +146,10 @@ jobs:
   - task: stage
     file: pcf-pipelines/tasks/stage-buildpack/task.yml
     params:
-      <<: *cf_api_params
+      <<: &cf_api_params
+        CF_API_URI: {{cf_api_uri}}
+        CF_USERNAME: {{cf_user}}
+        CF_PASSWORD: {{cf_password}}
       BUILDPACK_NAME: binary_buildpack_latest
 
 - name: promote-binary-buildpack
@@ -459,8 +462,3 @@ jobs:
       <<: *cf_api_params
       SOURCE_BUILDPACK_NAME: tc_buildpack_latest
       TARGET_BUILDPACK_NAME: tc_buildpack
-
-cf_api_params: &cf_api_params
-  CF_API_URI: {{cf_api_uri}}
-  CF_USERNAME: {{cf_user}}
-  CF_PASSWORD: {{cf_password}}

upgrade-tile/pipeline.yml

@@ -96,6 +96,7 @@ jobs:
       OPSMAN_USERNAME: {{opsman_admin_username}}
       OPSMAN_PASSWORD: {{opsman_admin_password}}
       OPSMAN_DOMAIN_OR_IP_ADDRESS: {{opsman_domain_or_ip_address}}
+      SKIP_IF_ALREADY_STAGED: no
 
   - task: toggle-errands
     file: pcf-pipelines/tasks/toggle-errands/task.yml