From 9007e98406c3eb71d3cb367f5ae1a87a5827a934 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Thu, 30 Jan 2025 01:39:39 +0100
Subject: [PATCH] fix(ci): cluster pinning with timeout and retry (#2321)

---
 .github/workflows/ci.yml | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e10177277..6b4945cf7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -123,18 +123,40 @@ jobs:
         if: github.ref == 'refs/heads/main'
         continue-on-error: true
         run: |
-          ipfs-cluster-ctl --enc=json \
-              --host "${CLUSTER_HOST}" \
-              --basic-auth "${CLUSTER_USER}:${CLUSTER_PASSWORD}" \
-              add --format car --local \
-              --name "${PIN_NAME}" \
-              ipfs-webui_${{ github.sha }}.car
+        timeout-minutes: 5 # pin should takes a few seconds, 5 min is a hard ceiling
+        run: |
+          echo "Avoid doing initial upload to am6-1 node with experimental pebble datastore"
+          echo '127.0.0.1 collab-cluster-am6-1.cluster.dwebops.net' | sudo tee -a /etc/hosts
+
+          # run in a loop and retry until GA timeout-minutes hits
+          attempt=1
+          while true; do
+            echo "Attempt #$attempt"
+
+            # pinning <100MiB file should not take more than few seconds
+            timeout 1m ipfs-cluster-ctl --enc=json \
+                --host "${CLUSTER_HOST}" \
+                --basic-auth "${CLUSTER_USER}:${CLUSTER_PASSWORD}" \
+                add --format car --local \
+                --name "${PIN_NAME}" \
+                ipfs-webui_${{ github.sha }}.car && {
+                  echo "Pin succeeded on attempt #$attempt"
+                  exit 0
+                }
+
+                echo "Attempt #$attempt failed, retrying in 2 seconds..."
+                attempt=$((attempt + 1))
+                # if first attept failed, retry with debug enabled
+                if [[ $counter -eq 2 ]]; then
+                  export GOLOG_LOG_LEVEL='debug'
+                fi
+                sleep 2
+          done
         env:
           CLUSTER_HOST: "/dnsaddr/ipfs-websites.collab.ipfscluster.io"
           CLUSTER_USER: ${{ secrets.CLUSTER_USER }}
           CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }}
           PIN_NAME: "ipfs-webui@${{ github.sha }}"
-        timeout-minutes: 15
 
       - name: Fail job due to pinning failure
         # only fail if pinning failed for main commits