[Index] Make sync-chart-cloudflare-index more resilient (#31396)

Signed-off-by: Miguel Ruiz <[email protected]>
Signed-off-by: Bitnami Containers <[email protected]>

.github/workflows/archive-full-index.yml

@@ -0,0 +1,128 @@
+name: '[Index] Generate the full bitnami/charts index.yaml'
+on:
+  push:
+    branches:
+      - index
+jobs:
+  get:
+    runs-on: ubuntu-latest
+    name: Get
+    steps:
+      - id: checkout-repo-index
+        name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          ref: index
+          path: index
+      - id: checkout-repo-full-index
+        name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          ref: archive-full-index
+          path: full-index
+      - id: get-last-indexes
+        name: Get indexes
+        run: |
+          cp index/bitnami/index.yaml ./last_index.yaml
+          cp full-index/bitnami/index.yaml ./previous_index.yaml
+      - id: upload-artifact
+        name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: indexes
+          path: ./*index.yaml
+          retention-days: 2
+          if-no-files-found: error
+  merge:
+    runs-on: ubuntu-latest
+    needs: get
+    name: Merge
+    steps:
+      - id: download-artifact
+        name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: indexes
+      - id: merge
+        name: Merge
+        run: yq eval-all '. as $item ireduce ({}; . *+ $item )' previous_index.yaml last_index.yaml > duplicates_index.yaml
+      - id: remove
+        name: Remove duplicates
+        # Removes duplicates per entry using 'digest' as value.
+        run: yq eval '.entries[] |= unique_by(.digest)' duplicates_index.yaml > index.yaml
+      - id: upload-artifact
+        name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: archive-full-index
+          path: index.yaml
+          retention-days: 2
+          if-no-files-found: error
+  checks:
+    runs-on: ubuntu-latest
+    needs: merge
+    name: Checks
+    steps:
+      - id: download-artifacts
+        name: Download artifacts
+        uses: actions/download-artifact@v3
+      - id: index-lint
+        name: Lint archive full index
+        # Lint the resulting archive full index using ignoring identation and lin-length rules.
+        run: |
+          cat << EOF > config
+          extends: relaxed
+
+          rules:
+            indentation:
+              level: error
+            line-length: disable
+          EOF
+          yamllint -c config archive-full-index/index.yaml
+      - id: check-no-dups
+        name: Checks there are not any duplicates
+        # Try to find duplicate digest attributes which would mean there are duplicates.
+        run: |
+          yq eval '.entries[][].digest' archive-full-index/index.yaml | sort | uniq -d | ( ! grep sha256 )
+      - id: check-missing-releases
+        name: Checks there are not missing releases
+        # Available URLs should be fine if everything went well during the merge & deduplication.
+        run: |
+          yq eval '.entries[][].urls[]' indexes/last_index.yaml |sort| uniq > last_index_urls
+          yq eval '.entries[][].urls[]' archive-full-index/index.yaml | sort| uniq > index_urls
+          missing_urls="$(comm -13 index_urls last_index_urls)"
+          if [ -n "${missing_urls}" ]; then
+              echo "Found missing URLs:\n${missing_urls}"
+              exit 1
+          fi
+          echo "No missing releases detected"
+  update:
+    runs-on: ubuntu-latest
+    needs: checks
+    name: Update
+    steps:
+      - id: checkout-repo
+        name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          ref: archive-full-index
+          token: ${{ secrets.BITNAMI_BOT_TOKEN }}
+      - id: download-artifact-archive-full-index
+        name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: archive-full-index
+      - id: update-index
+        name: git-add-push
+        run: |
+          git config user.name "Bitnami Containers"
+          git config user.email "[email protected]"
+          git fetch origin archive-full-index
+          git reset --hard $(git commit-tree origin/archive-full-index^{tree} -m "Update index.yaml")
+          # Compare size of files
+          if [[ $(stat -c%s bitnami/index.yaml) -gt $(stat -c%s index.yaml) ]]; then
+            echo "New index.yaml file is shorter than the current one"
+            exit 1
+          fi
+          cp index.yaml bitnami/index.yaml
+          git add bitnami/index.yaml && git commit --signoff --amend --no-edit && git push origin archive-full-index --force-with-lease

.github/workflows/sync-chart-cloudflare-index.yml

@@ -0,0 +1,91 @@
+name: '[Index] Sync bitnami/charts index.yaml to Cloudflare'
+
+on:
+  push:
+    branches:
+    - index
+  workflow_call:
+    secrets:
+      CLOUDFLARE_CLIENT_ID:
+        required: true
+      CLOUDFLARE_CLIENT_SECRET:
+        required: true
+      CLOUDFLARE_USER_AUTH:
+        required: true
+
+# Remove all permissions by default
+permissions: {}
+
+jobs:
+  deploy:
+    name: Sync bitnami/charts index.yaml to Cloudflare
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      result: ${{ steps.upload.outputs.result }}
+    steps:
+    - uses: actions/checkout@master
+    - name: Upload to Cloudflare using a BCOM upload proxy
+      id: upload
+      env:
+        CLOUDFLARE_CLIENT_ID: ${{ secrets.CLOUDFLARE_CLIENT_ID }}
+        CLOUDFLARE_CLIENT_SECRET: ${{ secrets.CLOUDFLARE_CLIENT_SECRET }}
+        CLOUDFLARE_USER_AUTH: ${{ secrets.CLOUDFLARE_USER_AUTH }}
+      run: |
+        status="fail"
+        retries=0
+        while [[ "${status}" != "ok" && "$retries" -lt 3 ]]; do
+          export TOKEN=$(curl -s --location 'https://api-esp.broadcom.com/auth/oauth/v2/token' \
+          --data-urlencode "client_id=${CLOUDFLARE_CLIENT_ID}" \
+          --data-urlencode "client_secret=${CLOUDFLARE_CLIENT_SECRET}" \
+          --data-urlencode 'grant_type=client_credentials' | jq .access_token -r )
+
+          curl_args=(
+            "--location" "--request" "PUT"
+            "--fail" "--max-time" "10"
+            "--header" "userAuth: Basic ${CLOUDFLARE_USER_AUTH}"
+            "--header" "filePath: /index.yaml"
+            "--header" "Content-Type: text/yaml"
+            "--header" "Authorization: Bearer $TOKEN"
+            "--upload-file" "bitnami/index.yaml"
+          )
+          echo "Uploading index.yaml to Cloudflare"
+          # To avoid the action from failing, we run the request inside a conditional so we can retry
+          if curl "${curl_args[@]}" 'https://api-esp.broadcom.com/crushftp/fileUpload'; then
+            echo "Index upload request succeeded, waiting 20 seconds before integrity check..."
+            # Wait for 20 seconds to ensure the new index.yaml is available
+            sleep 20
+            # Compare the index.yaml checksums remote and locally
+            REMOTE_MD5=($(curl -Ls https://charts.bitnami.com/bitnami/index.yaml | md5sum))
+            REPOSITORY_MD5=($(md5sum bitnami/index.yaml))
+            if [[ "${REPOSITORY_MD5[0]}" == "${REMOTE_MD5[0]}" ]]; then
+              status='ok'
+            else
+              echo "Integrity check failed. Uploading index.yaml again.";
+            fi
+          else
+            echo "Index upload request failed or timed out. Retrying again in 20 seconds...";
+            sleep 20
+          fi
+          retries=$((retries+1))
+        done
+        echo "result=${status}" >> $GITHUB_OUTPUT
+    - name: Show messages
+      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea
+      with:
+        script: |
+          if ("${{ steps.upload.outputs.result }}" != "ok" ) {
+            core.setFailed("Index upload failed");
+          } else {
+            core.info("Index upload succeeded")
+          }
+  notify:
+    name: Send notification
+    needs: [deploy]
+    if: ${{ always() && needs.deploy.outputs.result != 'ok' }}
+    uses: bitnami/charts/.github/workflows/gchat-notification.yml@main
+    with:
+      workflow: ${{ github.workflow }}
+      job-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    secrets: inherit

.github/workflows/sync-chart-s3-index.yml

@@ -0,0 +1,24 @@
+name: '[Index] Sync bitnami/charts index.yaml to S3'
+
+on:
+  push:
+    branches:
+    - index
+
+jobs:
+  deploy:
+    name: Sync bitnami/charts index.yaml to S3
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - name: Upload to S3
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PUBLISH_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUBLISH_SECRET_ACCESS_KEY }}
+        AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_PUBLISH_ROLE_ARN }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+      run: |
+        # Configure AWS account
+        export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" $(aws sts assume-role --role-arn ${AWS_ASSUME_ROLE_ARN} --role-session-name GitHubIndex --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" --output text))
+        aws s3 cp --follow-symlinks bitnami/index.yaml s3://${{ secrets.AWS_S3_BUCKET }}/bitnami/
+        aws s3 cp --follow-symlinks bitnami/index.html s3://${{ secrets.AWS_S3_BUCKET }}/

bitnami/index.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-type" content="text/html;charset=UTF-8">
+    <title>Bitnami Helm Charts</title>
+  </head>
+  <body>
+
+  <h1 id="bitnami-library-kubernetes">The Bitnami Library for Kubernetes</h1>
+
+  Popular applications, provided by <a href="https://bitnami.com">Bitnami</a>, ready to launch on Kubernetes using <a href="https://github.com/helm/helm">Kubernetes Helm</a>.
+
+  <h2 id="tldr">TL;DR</h2>
+
+  <code>
+  $ helm repo add bitnami https://charts.bitnami.com/bitnami<br />
+  $ helm search repo bitnami<br />
+  $ helm install my-release bitnami/&lt;chart&gt;
+  </code>
+
+  <p>For more information, please refer to the <a href="https://github.com/bitnami/charts">Bitnami charts project on GitHub</a>.</p>
+
+  </body>
+</html>