Merge pull request #8 from liulxb/main

add shellcheck and optimize code
opensourceways · Aug 26, 2024 · c844c40 · c844c40
2 parents c7184db + 1ba25d7
commit c844c40
Show file tree

Hide file tree

Showing 5 changed files with 205 additions and 113 deletions.
diff --git a/.github/workflows/checkstyle-shell.yml b/.github/workflows/checkstyle-shell.yml
@@ -0,0 +1,14 @@
+name: "checkstyle: shell"
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  shellcheck:
+    name: Shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
diff --git a/ai-repair/AiRepairSpec.py b/ai-repair/AiRepairSpec.py
@@ -15,10 +15,10 @@
 
 print("Ai repair spec begin ...")
 
-if len(sys.argv) != 5 :
+if len(sys.argv) != 5:
     print('error argumengs num [{}]'.format(len(sys.argv)))
     for i in sys.argv:
-        print (i)
+        print(i)
     sys.exit(1)
 
 if not sys.argv[1].endswith(".spec"):
@@ -40,20 +40,24 @@
 # 修改后的spec脚本保存地址
 spec_repair_file = sys.argv[3]
 spec_repair_path = os.path.dirname(spec_repair_file)
- 
+
 # 修改建议
 suggestion_file = sys.argv[4]
 suggestion_path = os.path.dirname(suggestion_file)
- 
+
 
 arr_path = [spec_src_path, spec_repair_path, log_src_path, suggestion_path]
 for path in arr_path:
     if not os.path.exists(path):
         os.makedirs(path)
 
 specbot = SpecBot()
-suggestion, flag = specbot.repair(spec_src_file, log_src_file, spec_repair_file)
-
+suggestion, flag = specbot.repair(
+    spec_src_file,
+    log_src_file,
+    spec_repair_file
+)
+
 # 打开文件用于写入，如果文件不存在则创建
 with open(suggestion_file, 'w') as file:
     # 将字符串写入文件

diff --git a/utils/HaveProcRPM-new.sh b/utils/HaveProcRPM-new.sh
@@ -30,10 +30,10 @@ done
 start_date="2024-08-12"
 
 # os_arch = 2p16g  2288hv3-2s24p-768g--b26
-#os_arch=2288hv3-2s24p-768g--b26
+# os_arch=2288hv3-2s24p-768g--b26
 os_arch=vm-2p16g
 
-# testbox = x86_64  
+# testbox = x86_64   aarch64
 testbox=x86_64
 
 # job日志路径
@@ -72,148 +72,146 @@ fi
 # 备份上次统计的结果文件
 file_date=$(date +"%Y%m%d-%H%M%S")
 old_dir="${RESULT_PATH}/../result_${file_date}"
-mv ${RESULT_PATH} ${old_dir}
+mv "${RESULT_PATH}" "${old_dir}"
 
-mkdir -p ${dir_fail_spec}
-mkdir -p ${dir_fail_log}
+mkdir -p "${dir_fail_spec}"
+mkdir -p "${dir_fail_log}"
 
 # 新建文件
-files=(${res_job_rpm} ${res_proc_ed} ${res_proc_ing} ${res_repeat} ${res_err}  ${rpm_build_succ_file} ${rpm_build_installed_file} ${execute_log})
+files=("${res_job_rpm}" "${res_proc_ed}" "${res_proc_ing}" "${res_repeat}" "${res_err}"  "${rpm_build_succ_file}" "${rpm_build_installed_file}" "${execute_log}")
 for file in "${files[@]}"
 do
-    $(touch ${file}) 
+    touch "${file}"
 done
 
 #---------init file end---------
 
 # 根据源list文件，获取成功的列表（如果源码生成，则表示构建成功）
-while read file; do
+while read -r file; do
     rpm_name=$(echo "$file" | awk -F'/' '{print "/" $NF }')
     if [ -f "${RPM_BUILD_SRC_PATH}${rpm_name}" ]; then
-        echo "${rpm_name}" >> ${rpm_build_succ_file}
+        echo "${rpm_name}" >> "${rpm_build_succ_file}"
     fi
-done < ${LIST_PATH}
+done < "${LIST_PATH}"
 
 end_date=$(date +"%Y-%m-%d")
 end=$(date -d "${end_date}" +%s)
 current=$(date -d "${start_date}" +%s)
 
 log_msg()
 {
-    echo "$@" | tee -a ${execute_log}
+    echo "$@" | tee -a "${execute_log}"
 }
 
 log_msg ""
 log_msg "begin [${start_date}] to [${end_date}] ..."
 log_msg ""
 
-while [ ${current} -le ${end} ]; do
+while [ "${current}" -le "${end}" ]; do
     # 结果路径
     current_date=$(date -d "@${current}" +%Y-%m-%d)
     CURRENT_PATH="${RPM_BUILD_RES_PATH}/${current_date}/${os_arch}/openeuler-24.03-LTS-${testbox}/${testbox}"
     log_msg "[${CURRENT_PATH}] ..."
 
     # 分别对每个job的结果目录进行处理
-    for file_path in $(ls -d ${CURRENT_PATH}/*/ | sort )
+    for file_path in $(find "${CURRENT_PATH}/*/" -type d -maxdepth 1 | sort )
     do
         dir_name=$(basename "${file_path}")
         dir_path="${CURRENT_PATH}/${dir_name}"
 
         # 获取repo_addr，以获取rpm包名
-        rpm_name=$(grep "^repo_addr:"  ${dir_path}/job.yaml | awk -F'/' '{print $NF}')
+        rpm_name=$(grep "^repo_addr:"  "${dir_path}/job.yaml" | awk -F'/' '{print $NF}')
         if [ -z "${rpm_name}" ]; then
-            echo "proc error: rpm name is null [${dir_path}]" >> ${res_err}
+            echo "proc error: rpm name is null [${dir_path}]" >> "${res_err}"
             continue
         fi
 
         # 检查是否属于本list
-        grep -q "/${rpm_name}" ${LIST_PATH}
-        if [ $? -ne 0 ]; then
-            echo "[warn] not belong to this batch: [${rpm_name}}" >> ${res_err}
+        if grep -q "/${rpm_name}" "${LIST_PATH}"; then
+            echo "[warn] not belong to this batch: [${rpm_name}}" >> "${res_err}"
             continue
         fi
 
         # 检查rpm是否在成功清单中
-        grep -q "/${rpm_name}" ${rpm_build_succ_file}
-        if [ $? -ne 0 ]; then
+        if grep -q "/${rpm_name}" "${rpm_build_succ_file}"; then
             # 不在成功清单中
 
             # 检查包是否已经安装（build之前已经存在）
-            if [ -f ${dir_path}/dmesg ] && grep -q -F '[---self-log---]: success, rpm already installed' ${dir_path}/dmesg; then 
-                echo "/${rpm_name}" >> ${rpm_build_installed_file}
+            if [ -f "${dir_path}/dmesg" ] && grep -q -F '[---self-log---]: success, rpm already installed' "${dir_path}/dmesg"; then 
+                echo "/${rpm_name}" >> "${rpm_build_installed_file}"
             else
                 output_file="${dir_path}/output"
                 if [ ! -f "${output_file}" ]; then
                     # 处理中的job， 则跳过
-                    echo "${dir_name} ## ${rpm_name} ## ${dir_path}" >> ${res_proc_ing}
+                    echo "${dir_name} ## ${rpm_name} ## ${dir_path}" >> "${res_proc_ing}"
 
-                    rpm_package_line=$(grep "/${rpm_name}" ${LIST_PATH})
+                    rpm_package_line=$(grep "/${rpm_name}" "${LIST_PATH}")
                     rpm_package_name="Packages/${rpm_package_line##*Packages/}"
-                    echo "${rpm_name}   ${rpm_package_name}" >> ${rpm_build_procing_file}
+                    echo "${rpm_name}   ${rpm_package_name}" >> "${rpm_build_procing_file}"
                     continue
                 fi
             fi
         fi
 
         # 保存记录到已处理文件
         record=${dir_name}' ## '${rpm_name}' ## '${dir_path}
-        if grep -q "## ${rpm_name} ##" ${res_job_rpm}; then
+        if grep -q "## ${rpm_name} ##" "${res_job_rpm}"; then
             # 如果已经存在rpm的记录，则可能是重复提交，删除最开始的记录
-            $(grep "${rpm_name}" ${res_job_rpm} >> ${res_repeat})
-            $(sed "/${rpm_name}/d" ${res_job_rpm} > ${tmp_file} && mv -f ${tmp_file} ${res_job_rpm})
+            grep "${rpm_name}" "${res_job_rpm}" >> "${res_repeat}"
+            sed "/${rpm_name}/d" "${res_job_rpm}" > "${tmp_file}" && mv -f "${tmp_file}" "${res_job_rpm}"
         fi
 
-        echo ${record} >> ${res_job_rpm};
+        echo "${record}" >> "${res_job_rpm}";
     done
 
     # 当前日期下的目录处理完成，下一个日期
     current=$((current + 86400))
 done
 
 # 对已处理的job进行分析
-while read line; do
-    rpm_name=$(echo ${line} | awk -F' ## ' '{print $2}')
-    echo ${rpm_name} >> ${res_proc_ed}
+while read -r line; do
+    rpm_name=$(echo "${line}" | awk -F' ## ' '{print $2}')
+    echo "${rpm_name}" >> "${res_proc_ed}"
 
-    if grep -q "/${rpm_name}" ${rpm_build_succ_file}; then
+    if grep -q "/${rpm_name}" "${rpm_build_succ_file}"; then
         # 在成功列表中
         continue
-    elif grep -q "/${rpm_name}" ${rpm_build_installed_file}; then 
+    elif grep -q "/${rpm_name}" "${rpm_build_installed_file}"; then 
         # 在已安装列表中
         continue
     else
-        job_id=$(echo ${line} | awk -F' ## ' '{print $1}')
-        file_path=$(echo ${line} | awk -F' ## ' '{print $NF}')
+        #job_id=$(echo ${line} | awk -F' ## ' '{print $1}')
+        file_path=$(echo "${line}" | awk -F' ## ' '{print $NF}')
 
         if [ -e "${file_path}/output" ]; then
             # output文件存在
-            if ls ${file_path} | grep -q '\.spec$'; then 
+            if find "${file_path}" -maxdepth 1 -type f -name "*.spec" > /dev/null; then 
                 # spec文件存在
                 # 保存log、spec文件
-                spec_file_name=$(find ${file_path} -name "*.spec" | awk -F'/' '{print $NF}')
-                spec_name=$(basename ${spec_file_name} .spec)
+                spec_file_name=$(find "${file_path}" -name "*.spec" | awk -F'/' '{print $NF}')
+                spec_name=$(basename "${spec_file_name}" .spec)
 
-                rpm_package_line=$(grep "/${rpm_name}" ${LIST_PATH})
+                rpm_package_line=$(grep "/${rpm_name}" "${LIST_PATH}")
                 rpm_package_name="Packages/${rpm_package_line##*Packages/}"
-                echo "${spec_file_name}   ${rpm_package_name}" >> ${rpm_build_fail_file}
+                echo "${spec_file_name}   ${rpm_package_name}" >> "${rpm_build_fail_file}"
 
-                $(cp ${file_path}/*.spec ${dir_fail_spec}/)
-                $(cp ${file_path}/output ${dir_fail_log}/${spec_name}.log)
+                cp "${file_path}/*.spec" "${dir_fail_spec}"/
+                cp "${file_path}/output" "${dir_fail_log}/${spec_name}.log"
             else 
-                echo "${spec_file_name}   ${rpm_package_name}" >> ${rpm_build_fail_no_spec_file}
-                echo "file not exist: .spec file not exist [${rpm_name}]-[${file_path}]" >> ${res_err}
+                echo "${spec_file_name}   ${rpm_package_name}" >> "${rpm_build_fail_no_spec_file}"
+                echo "file not exist: .spec file not exist [${rpm_name}]-[${file_path}]" >> "${res_err}"
             fi
         fi
     fi
-done < ${res_job_rpm}
-
-total_num=$(sed -n '$=' ${LIST_PATH})
-proc_ed_num=$(sed -n '$=' ${res_proc_ed})
-proc_ing_num=$(sed -n '$=' ${res_proc_ing})
-proc_succ_num=$(ls -lR ${RPM_BUILD_SRC_PATH} | grep "^-" | wc -l)
-proc_succ_num_1=$(sed -n '$=' ${rpm_build_succ_file})
-proc_install_num=$(sed -n '$=' ${rpm_build_installed_file})
-proc_fail_num=$((${proc_ed_num} - ${proc_succ_num_1}))
+done < "${res_job_rpm}"
+
+total_num=$(sed -n '$=' "${LIST_PATH}")
+proc_ed_num=$(sed -n '$=' "${res_proc_ed}")
+proc_ing_num=$(sed -n '$=' "${res_proc_ing}")
+proc_succ_num=$(find ${RPM_BUILD_SRC_PATH} -maxdepth 1 -type f -name '.src.rpm' | wc -l)
+proc_succ_num_1=$(sed -n '$=' "${rpm_build_succ_file}")
+proc_install_num=$(sed -n '$=' "${rpm_build_installed_file}")
+proc_fail_num=$((proc_ed_num - proc_succ_num_1))
 
 log_msg ""
 log_msg ""

diff --git a/utils/submit-init.sh b/utils/submit-init.sh
@@ -1,4 +1,4 @@
-#! /bin/zsh
+#! /bin/bash
 
 #----------------------------------------------------------------------------------
 # 参数介绍:
@@ -11,20 +11,25 @@
 
 # 提交job时, 参数
 # 架构： aarch64   x86_64
-submit_arch="x86_64"
+submit_arch="aarch64"
 # 虚拟机规格：vm-2p16g  vm-2p8g  vm-4p32g  2288hv3-2s24p-768g--b26
 submit_testbox="vm-2p16g"
 # 提交job其他参数: 
 #   check_rpm_install：(yes/no) 是否检查rpm已经安装
 #   res_file_exten：构建成功后，源文件添加.fc40后缀
-submit_other_app_param="check_rpm_install=yes res_file_exten=.el9"
-# job参数   "-m -c job.yaml -i ssh.yaml"
-submit_job_inf="job-centos9.yaml"
+submit_other_app_param="check_rpm_install=yes res_file_exten=.fc40"
+# job参数   "-m -c job.yaml -i ssh.yaml" 
+#   job-openSUSE15.6-aarch64.yaml  ob-centos9-aarch64.yaml
+submit_job_inf="./job-yaml/job-fedora40-aarch64.yaml"
 # 软件包源码仓库基地址
 #repo_base_addr="https://mirrors.huaweicloud.com/fedora/releases/40/Everything/source/tree/"
 repo_base_addr=""
 
+# submit最大提交次数
+submit_max_times=5
+
 time=$(date +"%Y%m%d-%H%M%S")
+submit_log_file="log/submit-log-${submit_arch}-${time}"
 
 # 输入参数解析
 param_list=""
@@ -67,9 +72,10 @@ param_check_other()
     [ -z "${param_other}" ] && return 0
 
     # 解析其他参数字段
-    param_other_arr=($(awk -F '+' '{for (i=1; i<=NF; i++) print $i}' <<< "${param_other}"))
+    param_other_arr=()
+    read -ra param_other_arr <<< "${param_other//+/ }"
 
-    echo "其他参数为：${param_other_arr[@]}"
+    echo "其他参数为：${param_other_arr[*]}"
 }
 
 param_check()
@@ -80,19 +86,20 @@ param_check()
 
 log_msg()
 {
-    echo "$@" | tee -a log/submit-log-${submit_arch}-${time}
+    echo "$@" | tee -a "${submit_log_file}"
 }
 
 # 参数检查
 param_check
 
 submit_num=0
-while read line; 
+while read -r line; 
 do
     [ -z "${line}" ] && continue
 
     # 初始化
-    line_array=(${line})
+    line_array=()
+    read -ra line_array <<< "${line}"
     [ ${#line_array[@]} -lt 1 ] && echo "[error] ${line}记录" 
 
     if [ $((${#line_array[@]} - 1)) -lt ${#param_other_arr[@]} ] ; then 
@@ -117,17 +124,43 @@ do
 
     commond="submit ${submit_job_inf} repo_addr=${submit_repo_addr} os_arch=${submit_arch} testbox=${submit_testbox} ${submit_other} ${submit_other_app_param} | tee -a log/submit-log-${submit_arch}-${time}"
     log_msg "${commond}"
-    submit ${submit_job_inf} repo_addr=${submit_repo_addr} os_arch=${submit_arch} testbox=${submit_testbox} ${submit_other} ${submit_other_app_param} | tee -a log/submit-log-${submit_arch}-${time}
+
+    repeat_flag="true"
+    submit_times=1
+    while [ ${submit_times} -le ${submit_max_times} ] && [ "${repeat_flag}" = "true" ]; do
+        log_msg "submit times [${submit_times}]"
+        submit_res=$(submit ${submit_job_inf} repo_addr="${submit_repo_addr}" os_arch=${submit_arch} testbox=${submit_testbox} "${submit_other}" "${submit_other_app_param}" | tee -a "${submit_log_file}")
+        echo "${submit_res}"
+        job_id=$(echo "${submit_res[@]}" | grep -o "got job id=.*" | awk -F'=' '{print  $NF}' | awk -F',' '{print $1}')
+        log_msg "job id[${job_id}]"
+        if [ -n "${job_id}" ] && [ "${job_id}" != "0" ]; then 
+            log_msg "[${submit_repo_addr}] submit success.." 
+            repeat_flag=false
+            break
+        fi
+
+        # 如果提交失败，则休眠10秒
+        log_msg "submit fail, will repeat submit. "
+        log_msg "sleep 10 seconds ..."
+        sleep 10
+
+        ((submit_times++))
+    done
+
+    if [ ${submit_times} -gt ${submit_max_times} ]; then 
+        log_msg "[${submit_repo_addr}] submit fail...[${submit_times}]" 
+    fi
 
     log_msg "" 
 
     ((submit_num++))
-    if [[ ${submit_num} != 0 && $((${submit_num} % 500)) == 0 ]]; then 
+    if [[ ${submit_num} != 0 && $((submit_num % 500)) == 0 ]]; then 
         # 睡眠20分钟，防止当前提交任务抢占机器
-        log_msg "sleep 1200 seconds ..."
-        sleep 1200
+        log_msg "sleep 3600 seconds ..."
+        sleep 3600
     fi
-done < ${param_list}
+
+done < "${param_list}"
 
 log_msg ""
-log_msg "总共提交job数量：[${submit_num}]" 
+log_msg "总共提交job数量：[${submit_num}]"