switching from GCP gcs to AWS s3 buckets (#3563)

* switching from GCP gcs to AWS s3 buckets * move awscli installation to Dockerfile
rancher · Nov 28, 2022 · 6a0dd2b · 6a0dd2b
1 parent b7c5c86
commit 6a0dd2b
Show file tree

Hide file tree

Showing 9 changed files with 44 additions and 92 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -26,8 +26,10 @@ steps:
     image: rancher/dapper:v0.5.5
     environment:
       ENABLE_REGISTRY: 'true'
-      GCLOUD_AUTH:
-        from_secret: gcloud_auth
+      AWS_ACCESS_KEY_ID:
+        from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader     
     commands:
       - docker pull --quiet rancher/hardened-build-base:v1.18.1b7
       - docker pull --quiet alpine:3.15
@@ -83,11 +85,13 @@ steps:
 
   - name: test
     image: rancher/dapper:v0.5.5
-    secrets: [ gcloud_auth ]
+    secrets: [ AWS_ACCESS_KEY_ID-artifact-uploader, AWS_SECRET_ACCESS_KEY-artifact-uploader ]
     environment:
       ENABLE_REGISTRY: 'true'
-      GCLOUD_AUTH:
-        from_secret: gcloud_auth
+      AWS_ACCESS_KEY_ID:
+        from_secret: AWS_ACCESS_KEY_ID-artifact-uploader
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: AWS_SECRET_ACCESS_KEY-artifact-uploader  
     commands:
       - dapper -f Dockerfile --target dapper make test
     volumes:

diff --git a/Dockerfile b/Dockerfile
@@ -15,6 +15,7 @@ RUN set -x \
     gcc \
     bsd-compat-headers \
     py-pip \
+    py3-pip \
     pigz \
     tar \
     yq
@@ -46,6 +47,7 @@ RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \
     chmod a+x /usr/local/bin/kubectl; \
     pip install codespell
 
+RUN python3 -m pip install awscli
 RUN curl -sL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.50.1
 RUN set -x \
     && apk --no-cache add \
@@ -203,6 +205,6 @@ RUN set -x \
     jq \
     less \
     socat \
-    vim
+    vim 
 ENTRYPOINT ["/bin/rke2"]
 CMD ["server"]
diff --git a/install.ps1 b/install.ps1
@@ -664,7 +664,7 @@ function Install-AirgapTarball() {
 }
 
 # Globals
-$STORAGE_URL = "https://storage.googleapis.com/rke2-ci-builds"
+$STORAGE_URL = "https://rke2-ci-builds.s3.amazonaws.com"
 $INSTALL_RKE2_GITHUB_URL = "https://github.com/rancher/rke2"
 
 Confirm-WindowsFeatures -RequiredFeatures @("Containers")

diff --git a/install.sh b/install.sh
@@ -94,7 +94,7 @@ check_target_ro() {
 
 # setup_env defines needed environment variables.
 setup_env() {
-    STORAGE_URL="https://storage.googleapis.com/rke2-ci-builds"
+    STORAGE_URL="https://rke2-ci-builds.s3.amazonaws.com"
     INSTALL_RKE2_GITHUB_URL="https://github.com/rancher/rke2"
     DEFAULT_TAR_PREFIX="/usr/local"
     # --- bail if we are not root ---

diff --git a/scripts/build-upload b/scripts/build-upload
@@ -1,7 +1,11 @@
 #!/bin/bash
 set -ex
 
-[ -n "$GCLOUD_AUTH" ] || {
+[ -n "$AWS_ACCESS_KEY_ID" ] || {
+  exit 0
+}
+
+[ -n "$AWS_SECRET_ACCESS_KEY" ] || {
   exit 0
 }
 
@@ -31,25 +35,6 @@ cleanup() {
 }
 trap cleanup EXIT INT
 
-GCLOUD_JSON=${TMPDIR}/.gcloud.json
-[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON}
-[ -s "${GCLOUD_JSON}" ] || {
-  echo "gcloud auth not defined" >&2
-  exit 1
-}
-
-BOTO_CONF=${TMPDIR}/.boto
-[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <<END
-[Credentials]
-gs_service_key_file = ${GCLOUD_JSON}
-[Boto]
-https_validate_certificates = True
-[GSUtil]
-content_language = en
-default_api_version = 2
-default_project_id = rancher-dev
-END
-
 BUNDLE_NAME=$(basename $1 .tar.gz)-$3.tar.gz
 (cd $(dirname $1) && sha256sum $(basename $1)) >${TMPDIR}/${BUNDLE_NAME}.sha256sum
 cp $1 ${TMPDIR}/${BUNDLE_NAME}
@@ -58,12 +43,7 @@ TARBALL_NAME=$(basename $2 .tar.zst)-$3.tar.zst
 (cd $(dirname $2) && sha256sum $(basename $2)) >${TMPDIR}/${TARBALL_NAME}.sha256sum
 cp $2 ${TMPDIR}/${TARBALL_NAME}
 
-[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR}
-
-HOME=${TMPDIR}
-PATH=${PATH}:${HOME}/gsutil
-
-gsutil cp ${TMPDIR}/${BUNDLE_NAME}* ${TMPDIR}/${TARBALL_NAME}* gs://rke2-ci-builds || exit 1
+aws s3 cp ${TMPDIR}/${BUNDLE_NAME}* ${TMPDIR}/${TARBALL_NAME}* s3://rke2-ci-builds || exit 1
 
 echo "Build uploaded" >&2
-echo "https://storage.googleapis.com/rke2-ci-builds/${BUNDLE_NAME}"
+echo "https://rke2-ci-builds.s3.amazonaws.com/${BUNDLE_NAME}"
diff --git a/scripts/log-upload b/scripts/log-upload
@@ -1,5 +1,13 @@
 #!/bin/bash
 
+[ -n "$AWS_ACCESS_KEY_ID" ] || {
+  exit 0
+}
+
+[ -n "$AWS_SECRET_ACCESS_KEY" ] || {
+  exit 0
+}
+
 [ -d "$1" ] || {
   echo "First argument should be a directory" >&2
   exit 1
@@ -18,33 +26,11 @@ cleanup() {
 }
 trap cleanup EXIT INT
 
-GCLOUD_JSON=${TMPDIR}/.gcloud.json
-[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON}
-[ -s "${GCLOUD_JSON}" ] || {
-  echo "gcloud auth not defined" >&2
-  exit 1
-}
-
-BOTO_CONF=${TMPDIR}/.boto
-[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <<END
-[Credentials]
-gs_service_key_file = ${GCLOUD_JSON}
-[Boto]
-https_validate_certificates = True
-[GSUtil]
-content_language = en
-default_api_version = 2
-default_project_id = rancher-dev
-END
-
-[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR}
-
-HOME=${TMPDIR}
-PATH=$PATH:${HOME}/gsutil
 
 LOG_TGZ=rke2-log-$(date +%s)-$("${GO}" env GOARCH)-$(git rev-parse --short HEAD)-$(basename $1).tgz
 
 tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1)
-gsutil cp ${TMPDIR}/${LOG_TGZ} gs://rke2-ci-logs || exit 1
+aws s3 cp ${TMPDIR}/${LOG_TGZ} s3://rke2-ci-logs || exit 1
 echo "Logs uploaded" >&2
-echo "https://storage.googleapis.com/rke2-ci-logs/${LOG_TGZ}"
+echo "https://rke2-ci-logs.s3.amazonaws.com/${LOG_TGZ}"
+
diff --git a/scripts/publish-dev-rpm b/scripts/publish-dev-rpm
@@ -1,7 +1,11 @@
 #!/usr/bin/env bash
 set -ex
 
-[ -n "${GCLOUD_AUTH}" ] || {
+[ -n "$AWS_ACCESS_KEY_ID" ] || {
+  exit 0
+}
+
+[ -n "$AWS_SECRET_ACCESS_KEY" ] || {
   exit 0
 }
 
@@ -30,31 +34,7 @@ done <"${RPM_LIST}"
 
 umask 077
 
-GCLOUD_JSON=${TMPDIR}/.gcloud.json
-[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >"${GCLOUD_JSON}"
-[ -s "${GCLOUD_JSON}" ] || {
-  echo "gcloud auth not defined" >&2
-  exit 1
-}
-
-BOTO_CONF=${TMPDIR}/.boto
-[ -s "${BOTO_CONF}" ] || cat >"${BOTO_CONF}" <<END
-[Credentials]
-gs_service_key_file = ${GCLOUD_JSON}
-[Boto]
-https_validate_certificates = True
-[GSUtil]
-content_language = en
-default_api_version = 2
-default_project_id = rancher-dev
-END
-
-[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C "${TMPDIR}"
-
-HOME=${TMPDIR}
-PATH=${PATH}:${HOME}/gsutil
-
-gsutil cp ${RPM_DIR}/*.rpm "gs://rke2-ci-builds/" || exit 1
+aws s3 cp ${RPM_DIR}/*.rpm "s3://rke2-ci-builds" || exit 1
 
 echo "Build uploaded" >&2
-echo "https://storage.googleapis.com/rke2-ci-builds/${BUNDLE_NAME}"
+echo "https://rke2-ci-builds.s3.amazonaws.com/${BUNDLE_NAME}"
diff --git a/tests/e2e/scripts/latest_commit.ps1 b/tests/e2e/scripts/latest_commit.ps1
@@ -3,7 +3,7 @@ param ($Branch, $CommitFile)
 (Invoke-RestMethod "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$Branch").sha | `
 Out-File -FilePath $CommitFile
 
-$StorageUrl = "https://storage.googleapis.com/rke2-ci-builds/rke2-images.windows-amd64-"
+$StorageUrl = "https://rke2-ci-builds.s3.amazonaws.com/rke2-images.windows-amd64-"
 $TopCommit = (Get-Content -TotalCount 1 $CommitFile)
 $StatusCode = Invoke-WebRequest $StorageUrl$TopCommit".tar.zst.sha256sum" -DisableKeepAlive -UseBasicParsing -Method head | % {$_.StatusCode}
 $Iterations = 0
@@ -17,4 +17,4 @@ while (($StatusCode -ne 200) -AND ($Iterations -lt 6)) {
 if ($Iterations -ge 6){
     Write-Host echo "No valid commits found" 
     Exit 1
-}
+}
diff --git a/tests/e2e/scripts/latest_commit.sh b/tests/e2e/scripts/latest_commit.sh
@@ -2,7 +2,7 @@
 # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
 iterations=0
 curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$1" | jq -r '.[] | .sha'  &> "$2"
-curl -s --fail https://storage.googleapis.com/rke2-ci-builds/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum
+curl -s --fail https://rke2-ci-builds.s3.amazonaws.com/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum
 while [ $? -ne 0 ]; do
     ((iterations++))
     if [ "$iterations" -ge 6 ]; then
@@ -11,5 +11,5 @@ while [ $? -ne 0 ]; do
     fi
     sed -i 1d "$2"
     sleep 1
-    curl -s --fail https://storage.googleapis.com/rke2-ci-builds/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum
-done
+    curl -s --fail https://rke2-ci-builds.s3.amazonaws.com/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum
+done