From 6a0dd2b5b44fecac6d6fc757c430bae2c16666fa Mon Sep 17 00:00:00 2001 From: Ivan-Clare Ngong Date: Mon, 28 Nov 2022 19:03:01 +0100 Subject: [PATCH] switching from GCP gcs to AWS s3 buckets (#3563) * switching from GCP gcs to AWS s3 buckets * move awscli installation to Dockerfile --- .drone.yml | 14 +++++++---- Dockerfile | 4 +++- install.ps1 | 2 +- install.sh | 2 +- scripts/build-upload | 34 ++++++--------------------- scripts/log-upload | 36 +++++++++-------------------- scripts/publish-dev-rpm | 34 ++++++--------------------- tests/e2e/scripts/latest_commit.ps1 | 4 ++-- tests/e2e/scripts/latest_commit.sh | 6 ++--- 9 files changed, 44 insertions(+), 92 deletions(-) diff --git a/.drone.yml b/.drone.yml index ad1f493e09..4c2c3dc53a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -26,8 +26,10 @@ steps: image: rancher/dapper:v0.5.5 environment: ENABLE_REGISTRY: 'true' - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader commands: - docker pull --quiet rancher/hardened-build-base:v1.18.1b7 - docker pull --quiet alpine:3.15 @@ -83,11 +85,13 @@ steps: - name: test image: rancher/dapper:v0.5.5 - secrets: [ gcloud_auth ] + secrets: [ AWS_ACCESS_KEY_ID-artifact-uploader, AWS_SECRET_ACCESS_KEY-artifact-uploader ] environment: ENABLE_REGISTRY: 'true' - GCLOUD_AUTH: - from_secret: gcloud_auth + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID-artifact-uploader + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader commands: - dapper -f Dockerfile --target dapper make test volumes: diff --git a/Dockerfile b/Dockerfile index f92314cade..92172ff63e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN set -x \ gcc \ bsd-compat-headers \ py-pip \ + py3-pip \ pigz \ tar \ yq @@ -46,6 +47,7 @@ RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \ chmod a+x /usr/local/bin/kubectl; \ pip install codespell +RUN python3 -m pip install awscli RUN curl -sL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.50.1 RUN set -x \ && apk --no-cache add \ @@ -203,6 +205,6 @@ RUN set -x \ jq \ less \ socat \ - vim + vim ENTRYPOINT ["/bin/rke2"] CMD ["server"] diff --git a/install.ps1 b/install.ps1 index 50ae4b3d54..4eb1e443a3 100644 --- a/install.ps1 +++ b/install.ps1 @@ -664,7 +664,7 @@ function Install-AirgapTarball() { } # Globals -$STORAGE_URL = "https://storage.googleapis.com/rke2-ci-builds" +$STORAGE_URL = "https://rke2-ci-builds.s3.amazonaws.com" $INSTALL_RKE2_GITHUB_URL = "https://github.com/rancher/rke2" Confirm-WindowsFeatures -RequiredFeatures @("Containers") diff --git a/install.sh b/install.sh index 9127328343..dcd046d025 100755 --- a/install.sh +++ b/install.sh @@ -94,7 +94,7 @@ check_target_ro() { # setup_env defines needed environment variables. setup_env() { - STORAGE_URL="https://storage.googleapis.com/rke2-ci-builds" + STORAGE_URL="https://rke2-ci-builds.s3.amazonaws.com" INSTALL_RKE2_GITHUB_URL="https://github.com/rancher/rke2" DEFAULT_TAR_PREFIX="/usr/local" # --- bail if we are not root --- diff --git a/scripts/build-upload b/scripts/build-upload index 616131c71b..ea3b8ef206 100755 --- a/scripts/build-upload +++ b/scripts/build-upload @@ -1,7 +1,11 @@ #!/bin/bash set -ex -[ -n "$GCLOUD_AUTH" ] || { +[ -n "$AWS_ACCESS_KEY_ID" ] || { + exit 0 +} + +[ -n "$AWS_SECRET_ACCESS_KEY" ] || { exit 0 } @@ -31,25 +35,6 @@ cleanup() { } trap cleanup EXIT INT -GCLOUD_JSON=${TMPDIR}/.gcloud.json -[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON} -[ -s "${GCLOUD_JSON}" ] || { - echo "gcloud auth not defined" >&2 - exit 1 -} - -BOTO_CONF=${TMPDIR}/.boto -[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <${TMPDIR}/${BUNDLE_NAME}.sha256sum cp $1 ${TMPDIR}/${BUNDLE_NAME} @@ -58,12 +43,7 @@ TARBALL_NAME=$(basename $2 .tar.zst)-$3.tar.zst (cd $(dirname $2) && sha256sum $(basename $2)) >${TMPDIR}/${TARBALL_NAME}.sha256sum cp $2 ${TMPDIR}/${TARBALL_NAME} -[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR} - -HOME=${TMPDIR} -PATH=${PATH}:${HOME}/gsutil - -gsutil cp ${TMPDIR}/${BUNDLE_NAME}* ${TMPDIR}/${TARBALL_NAME}* gs://rke2-ci-builds || exit 1 +aws s3 cp ${TMPDIR}/${BUNDLE_NAME}* ${TMPDIR}/${TARBALL_NAME}* s3://rke2-ci-builds || exit 1 echo "Build uploaded" >&2 -echo "https://storage.googleapis.com/rke2-ci-builds/${BUNDLE_NAME}" +echo "https://rke2-ci-builds.s3.amazonaws.com/${BUNDLE_NAME}" diff --git a/scripts/log-upload b/scripts/log-upload index ecff03f860..aebd233c12 100755 --- a/scripts/log-upload +++ b/scripts/log-upload @@ -1,5 +1,13 @@ #!/bin/bash +[ -n "$AWS_ACCESS_KEY_ID" ] || { + exit 0 +} + +[ -n "$AWS_SECRET_ACCESS_KEY" ] || { + exit 0 +} + [ -d "$1" ] || { echo "First argument should be a directory" >&2 exit 1 @@ -18,33 +26,11 @@ cleanup() { } trap cleanup EXIT INT -GCLOUD_JSON=${TMPDIR}/.gcloud.json -[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON} -[ -s "${GCLOUD_JSON}" ] || { - echo "gcloud auth not defined" >&2 - exit 1 -} - -BOTO_CONF=${TMPDIR}/.boto -[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <&2 -echo "https://storage.googleapis.com/rke2-ci-logs/${LOG_TGZ}" +echo "https://rke2-ci-logs.s3.amazonaws.com/${LOG_TGZ}" + diff --git a/scripts/publish-dev-rpm b/scripts/publish-dev-rpm index 5ec7145fd4..7ee35658b4 100755 --- a/scripts/publish-dev-rpm +++ b/scripts/publish-dev-rpm @@ -1,7 +1,11 @@ #!/usr/bin/env bash set -ex -[ -n "${GCLOUD_AUTH}" ] || { +[ -n "$AWS_ACCESS_KEY_ID" ] || { + exit 0 +} + +[ -n "$AWS_SECRET_ACCESS_KEY" ] || { exit 0 } @@ -30,31 +34,7 @@ done <"${RPM_LIST}" umask 077 -GCLOUD_JSON=${TMPDIR}/.gcloud.json -[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >"${GCLOUD_JSON}" -[ -s "${GCLOUD_JSON}" ] || { - echo "gcloud auth not defined" >&2 - exit 1 -} - -BOTO_CONF=${TMPDIR}/.boto -[ -s "${BOTO_CONF}" ] || cat >"${BOTO_CONF}" <&2 -echo "https://storage.googleapis.com/rke2-ci-builds/${BUNDLE_NAME}" +echo "https://rke2-ci-builds.s3.amazonaws.com/${BUNDLE_NAME}" diff --git a/tests/e2e/scripts/latest_commit.ps1 b/tests/e2e/scripts/latest_commit.ps1 index 52c52be22c..67a9742b20 100644 --- a/tests/e2e/scripts/latest_commit.ps1 +++ b/tests/e2e/scripts/latest_commit.ps1 @@ -3,7 +3,7 @@ param ($Branch, $CommitFile) (Invoke-RestMethod "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$Branch").sha | ` Out-File -FilePath $CommitFile -$StorageUrl = "https://storage.googleapis.com/rke2-ci-builds/rke2-images.windows-amd64-" +$StorageUrl = "https://rke2-ci-builds.s3.amazonaws.com/rke2-images.windows-amd64-" $TopCommit = (Get-Content -TotalCount 1 $CommitFile) $StatusCode = Invoke-WebRequest $StorageUrl$TopCommit".tar.zst.sha256sum" -DisableKeepAlive -UseBasicParsing -Method head | % {$_.StatusCode} $Iterations = 0 @@ -17,4 +17,4 @@ while (($StatusCode -ne 200) -AND ($Iterations -lt 6)) { if ($Iterations -ge 6){ Write-Host echo "No valid commits found" Exit 1 -} \ No newline at end of file +} diff --git a/tests/e2e/scripts/latest_commit.sh b/tests/e2e/scripts/latest_commit.sh index 165be9aa59..af5f5f5e95 100755 --- a/tests/e2e/scripts/latest_commit.sh +++ b/tests/e2e/scripts/latest_commit.sh @@ -2,7 +2,7 @@ # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build iterations=0 curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$1" | jq -r '.[] | .sha' &> "$2" -curl -s --fail https://storage.googleapis.com/rke2-ci-builds/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum +curl -s --fail https://rke2-ci-builds.s3.amazonaws.com/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum while [ $? -ne 0 ]; do ((iterations++)) if [ "$iterations" -ge 6 ]; then @@ -11,5 +11,5 @@ while [ $? -ne 0 ]; do fi sed -i 1d "$2" sleep 1 - curl -s --fail https://storage.googleapis.com/rke2-ci-builds/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum -done \ No newline at end of file + curl -s --fail https://rke2-ci-builds.s3.amazonaws.com/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum +done