Update S3-rhcossync.sh (#4253)

openshift-eng · Sep 26, 2024 · 30131f2 · 30131f2
1 parent b344afe
commit 30131f2
Showing 1 changed file with 11 additions and 9 deletions.
diff --git a/jobs/build/rhcos_sync/S3-rhcossync.sh b/jobs/build/rhcos_sync/S3-rhcossync.sh
@@ -77,8 +77,8 @@ function emulateSymlinks() {
 
     if [[ "${RHCOS_MIRROR_PREFIX}" == "pre-release" ]]; then
         MAJOR_MINOR_LATEST="latest-${MAJOR_MINOR}"
-        aws s3 sync --no-progress --delete --exact-timestamps "${S3_SOURCE}" s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${MAJOR_MINOR_LATEST}/
-        aws s3 sync --no-progress --delete --exact-timestamps "${S3_SOURCE}" s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${MAJOR_MINOR_LATEST}/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+        aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${MAJOR_MINOR_LATEST}/
+        aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${MAJOR_MINOR_LATEST}/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
 
         # Is this major.minor the latest Y stream? If it is, we need to set
         # the overall 'latest'.
@@ -97,8 +97,8 @@ function emulateSymlinks() {
         # LATEST_LINK will end up being something like 4.9.0-fc.0 if the next major exists or "" if it does not.
 
         if [[ -z "${LATEST_LINK}" ]]; then
-            aws s3 sync --no-progress --delete --exact-timestamps "${S3_SOURCE}" s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/
-            aws s3 sync --no-progress --delete --exact-timestamps "${S3_SOURCE}" s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+            aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/
+            aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
         fi
 
     else
@@ -108,8 +108,8 @@ function emulateSymlinks() {
         LATEST_CONTENT=$(aws s3 ls "s3://art-srv-enterprise${BASEDIR}/${MAJOR_NEXT_MINOR}/" | grep PRE || true)
 
         if [[ -z "${LATEST_CONTENT}" ]]; then
-            aws s3 sync --no-progress --delete --exact-timestamps "${S3_SOURCE}" s3://art-srv-enterprise${BASEDIR}/latest/
-            aws s3 sync --no-progress --delete --exact-timestamps "${S3_SOURCE}" s3://art-srv-enterprise${BASEDIR}/latest/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+            aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/latest/
+            aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/latest/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
         fi
     fi
 
@@ -187,9 +187,11 @@ fi
 aws s3 sync --no-progress --delete --exact-timestamps ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/"
 aws s3 sync --no-progress --delete --exact-timestamps ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/" --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
 if [ $NOLATEST -eq 0 ]; then
-    aws s3 sync --no-progress --delete --exact-timestamps "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/" "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/"
-    aws s3 sync --no-progress --delete --exact-timestamps "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/" "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/" --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
-    emulateSymlinks "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/"
+    aws s3 sync --no-progress --delete --exact-timestamps ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/"
+    aws s3 sync --no-progress --delete --exact-timestamps ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/" --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+    # CloudFlare does not support the full S3 API and we encountered unimplemented APIs using the AWS CLI to sync from one area of R2 to another area.
+    # We therefore re-copy from local content to R2, which does not hit these limitations. This is technically slower, but works with R2.
+    emulateSymlinks "./"
 else
     echo "INFO: Not updating 'latest' symlink because --nolatest was given"
 fi