Overhaul latest_commit scripts for E2E, more logging and reduce GHA a…

…pi calls (rancher#7427)

* Overhaul latest_commit scripts for E2E, improve logging and reduce GHA api calls

Signed-off-by: Derek Nola <[email protected]>

* Address review comments, fix typo

Signed-off-by: Derek Nola <[email protected]>

---------

Signed-off-by: Derek Nola <[email protected]>

tests/e2e/mixedos/Vagrantfile

@@ -28,9 +28,11 @@ def provision(vm, role, role_num, node_num)
     vm.provision "shell", path: "../scripts/install-bgp.ps1"
     if RELEASE_VERSION == "skip"
       install_type = "-ArtifactPath 'C:\tmp'" 
-    elsif !RELEASE_VERSION.empty?
+    elsif !RELEASE_VERSION.empty? && RELEASE_VERSION.start_with?("v1")
       install_type = "-Version #{RELEASE_VERSION}"
-    else 
+    elsif !RELEASE_VERSION.empty?
+      install_type = "-Commit #{RELEASE_VERSION}"
+    else
       vm.provision "shell", path: "../scripts/latest_commit.ps1", args: [GITHUB_BRANCH, "./rke2_commits.txt"]
       install_type = "-Commit (Get-Content -TotalCount 1 ./rke2_commits.txt)"
     end

tests/e2e/scripts/latest_commit.ps1

@@ -1,7 +1,16 @@
 # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
 param ($Branch, $CommitFile)
-(Invoke-RestMethod "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$Branch").sha | `
-Out-File -FilePath $CommitFile
+$response = (Invoke-RestMethod "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$Branch")
+if ($response -is [System.Array]) {
+    $response.sha | Out-File -FilePath $CommitFile
+} if ($response -is [PSCustomObject]) {
+    if ($response.message -like "API rate limit exceeded for *") {
+        Write-Host "Github API rate limit exceeded"
+        Exit 1
+    }
+    Write-Host "Github API returned a non-expected response $($response.message)"
+    Exit 1
+} 
 
 $StorageUrl = "https://rke2-ci-builds.s3.amazonaws.com/rke2-images.windows-amd64-"
 $TopCommit = (Get-Content -TotalCount 1 $CommitFile)

tests/e2e/scripts/latest_commit.sh

@@ -1,7 +1,25 @@
 #!/bin/bash
-# Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
+
+branch=$1
+output_file=$2
+# Grabs the last 10 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
 iterations=0
-response=$(curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$1")
+
+# The VMs take time on startup to hit aws, wait loop until we can
+while ! curl -s --fail https://rke2-ci-builds.s3.amazonaws.com?max-keys=0 > /dev/null; do
+    ((iterations++))
+    if [ "$iterations" -ge 30 ]; then
+        echo "Unable to hit https://rke2-ci-builds.s3.amazonaws.com"
+        exit 1
+    fi
+    sleep 1
+done
+
+if [ -n "$GH_TOKEN" ]; then
+    response=$(curl -s -H "Authorization: token $GH_TOKEN" -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/rancher/rke2/commits?per_page=10&sha=$branch")
+else
+    response=$(curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/rancher/rke2/commits?per_page=10&sha=$branch")
+fi
 type=$(echo "$response" | jq -r type)
 
 # Verify if the response is an array with the rke2 commits
@@ -14,17 +32,17 @@ if [[ $type == "object" ]]; then
     echo "Github API returned a non-expected response ${message}"
     exit 1
 elif [[ $type == "array" ]]; then
-    echo ${response} | jq -r '.[] | .sha'  &> "$2"
+    commits_str=$(echo "$response" | jq -j -r '.[] | .sha, " "')
 fi
 
-curl -s --fail https://rke2-ci-builds.s3.amazonaws.com/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum
-while [ $? -ne 0 ]; do
-    ((iterations++))
-    if [ "$iterations" -ge 6 ]; then
-        echo "No valid commits found"
-        exit 1
+read -a commits <<< "$commits_str"
+
+for commit in "${commits[@]}"; do
+    if curl -s --fail https://rke2-ci-builds.s3.amazonaws.com/rke2-images.linux-amd64-$commit.tar.zst.sha256sum > /dev/null; then
+        echo "$commit" > "$output_file"
+        exit 0
     fi
-    sed -i 1d "$2"
-    sleep 1
-    curl -s --fail https://rke2-ci-builds.s3.amazonaws.com/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum
 done
+
+echo "Failed to find a valid commit, checked: " "${commits[@]}"
+exit 1

tests/e2e/scripts/run_tests.sh

@@ -21,6 +21,11 @@ cd rke2
 git pull --rebase origin master
 /usr/local/go/bin/go mod tidy
 cd tests/e2e
+
+# To reduce GH API requests, we grab the latest commit on the host and pass it to the tests
+./scripts/latest_commit.sh master latest_commit.txt
+E2E_RELEASE_VERSION=$(cat latest_commit.txt) && export E2E_RELEASE_VERSION
+
 # create directory to store reports if it does not exists
 if [ ! -d createreport ]
 then

tests/e2e/vagrantdefaults.rb

@@ -13,12 +13,14 @@ def defaultOSConfigure(vm)
 def getInstallType(vm, version, branch)
   if version == "skip"
     return "INSTALL_RKE2_ARTIFACT_PATH=/tmp" 
-  elsif !version.empty?
+  elsif !version.empty? && version.start_with?("v1")
     return "INSTALL_RKE2_VERSION=#{version}"
+  elsif !version.empty?
+    return "INSTALL_RKE2_COMMIT=#{version}"
   end
-  # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
+  # Grabs the last 10 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
   scripts_location = Dir.exist?("./scripts") ? "./scripts" : "../scripts" 
-  vm.provision "shell", path:  scripts_location + "/latest_commit.sh", args: [branch, "/tmp/rke2_commits"]
+  vm.provision "shell", path:  scripts_location + "/latest_commit.sh", env: {GH_TOKEN:ENV['GH_TOKEN']}, args: [branch, "/tmp/rke2_commits"]
   return "INSTALL_RKE2_COMMIT=$(head\ -n\ 1\ /tmp/rke2_commits)"
 end