feature: 安装脚本健康检查重试机制 (closed #1854)

TencentBlueKing · Oct 13, 2023 · 6ec2568 · 6ec2568
1 parent a7a089f
commit 6ec2568
Show file tree

Hide file tree

Showing 2 changed files with 59 additions and 29 deletions.
diff --git a/script_tools/agent_tools/agent2/setup_agent.sh b/script_tools/agent_tools/agent2/setup_agent.sh
@@ -275,27 +275,42 @@ is_process_ok () {
 }
 
 check_heathz_by_gse () {
-    local result report_result
-    if [ -f "${GSE_AGENT_CONFIG_PATH}" ]; then
-        result=$("${AGENT_SETUP_PATH}"/bin/gse_agent -f "${GSE_AGENT_CONFIG_PATH}" --healthz)
-    else
-        result=$("${AGENT_SETUP_PATH}"/bin/gse_agent --healthz)
-    fi
-    execution_code=$?
+    local SLEEP_TIME=1 RETRY_COUNT=0
+
+    for i in {0..2}; do
+        local result execution_code
+        if [ -f "${GSE_AGENT_CONFIG_PATH}" ]; then
+            result=$("${AGENT_SETUP_PATH}"/bin/gse_agent -f "${GSE_AGENT_CONFIG_PATH}" --healthz 1)
+        else
+            result=$("${AGENT_SETUP_PATH}"/bin/gse_agent --healthz 1)
+        fi
+        execution_code=$?
+        if [[ "${execution_code}" -eq 0 ]]; then
+            break
+        else
+            sleep "${SLEEP_TIME}"
+            RETRY_COUNT=$((RETRY_COUNT + 1))
+            if [[ "${RETRY_COUNT}" -ge 3 ]]; then
+                log healthz_check INFO "gse_agent healthz check return code: ${execution_code}"
+                report_result=$(awk -F': ' '{print $2}' <<< "$result")
+                if is_base64_command_exist; then
+                    report_result=$(echo "$result" | base64 -w 0)
+                else
+                    report_result=$(echo "$result" | tr "\"" "\'")
+                fi
+                log report_healthz INFO "${report_result}"
+                fail healthz_check FAILED "gse healthz check failed with retry count: $RETRY_COUNT"
+            fi
+        fi
+    done
     report_result=$(awk -F': ' '{print $2}' <<< "$result")
     if is_base64_command_exist; then
         report_result=$(echo "$result" | base64 -w 0)
     else
         report_result=$(echo "$result" | tr "\"" "\'")
     fi
-    if [ "${execution_code}" -eq 0 ]; then
-        log report_healthz - "${report_result}"
-        log healthz_check INFO "gse_agent healthz check success"
-    else
-        warn report_healthz INFO "gse_agent healthz check return code: ${execution_code}"
-        warn report_healthz - "${report_result}"
-        fail healthz_check FAILED "gse healthz check failed."
-    fi
+    log report_healthz - "${report_result}"
+    log healthz_check INFO "gse_agent healthz check success"
 }
 
 remove_crontab () {

diff --git a/script_tools/agent_tools/agent2/setup_proxy.sh b/script_tools/agent_tools/agent2/setup_proxy.sh
@@ -407,27 +407,42 @@ is_base64_command_exist() {
 }
 
 check_healthz_by_gse () {
-    local result
-    if [ -f "${GSE_AGENT_CONFIG_PATH}" ]; then
-        result=$("${AGENT_SETUP_PATH}"/bin/gse_agent -f "${GSE_AGENT_CONFIG_PATH}" --healthz)
-    else
-        result=$("${AGENT_SETUP_PATH}"/bin/gse_agent --healthz)
-    fi
-    execution_code=$?
+    local SLEEP_TIME=1 RETRY_COUNT=0
 
+    for i in {0..2}; do
+        local result execution_code
+        if [ -f "${GSE_AGENT_CONFIG_PATH}" ]; then
+            result=$("${AGENT_SETUP_PATH}"/bin/gse_agent -f "${GSE_AGENT_CONFIG_PATH}" --healthz 1)
+        else
+            result=$("${AGENT_SETUP_PATH}"/bin/gse_agent --healthz 1)
+        fi
+        execution_code=$?
+        if [[ "${execution_code}" -eq 0 ]]; then
+            break
+        else
+            sleep "${SLEEP_TIME}"
+            RETRY_COUNT=$((RETRY_COUNT + 1))
+            if [[ "${RETRY_COUNT}" -ge 3 ]]; then
+                log healthz_check INFO "gse_agent healthz check return code: ${execution_code}"
+                report_result=$(awk -F': ' '{print $2}' <<< "$result")
+                if is_base64_command_exist; then
+                    report_result=$(echo "$result" | base64 -w 0)
+                else
+                    report_result=$(echo "$result" | tr "\"" "\'")
+                fi
+                log report_healthz INFO "${report_result}"
+                fail healthz_check FAILED "gse healthz check failed with retry count: $RETRY_COUNT"
+            fi
+        fi
+    done
     report_result=$(awk -F': ' '{print $2}' <<< "$result")
     if is_base64_command_exist; then
         report_result=$(echo "$result" | base64 -w 0)
     else
         report_result=$(echo "$result" | tr "\"" "\'")
     fi
-    if [ "${execution_code}" -eq 0 ]; then
-        log report_healthz - "${report_result}"
-        log healthz_check INFO "gse_agent healthz check success"
-    else
-        warn report_healthz INFO "gse_agent healthz check return code: ${execution_code}"
-        warn report_healthz - "${report_result}"
-        fail healthz_check FAILED "gse healthz check failed."
+    log report_healthz - "${report_result}"
+    log healthz_check INFO "gse_agent healthz check success"
     fi
 }