Skip to content

Commit

Permalink
【包迁移代码合入】update源监控代码优化
Browse files Browse the repository at this point in the history
  • Loading branch information
liulxb committed Oct 22, 2024
1 parent 816b707 commit 1b74dc4
Show file tree
Hide file tree
Showing 8 changed files with 142 additions and 116 deletions.
26 changes: 5 additions & 21 deletions easypackages/watch_update_source/config/repo_cfg.sh
Original file line number Diff line number Diff line change
@@ -1,26 +1,10 @@
#!/bin/bash

# shellcheck disable=SC2034
binary_xml_url_aarch64_openeuler_24_03_LTS=(
"https://repo.openeuler.org/openEuler-24.03-LTS/everything/aarch64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/EPOL/main/aarch64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/update/aarch64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/OS/aarch64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/debuginfo/aarch64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/source/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/update/source/repodata/repomd.xml"
"https://repo.oepkgs.net/openEuler/rpm/openEuler-24.03-LTS/fedora40/aarch64/repodata/repomd.xml"
)

binary_xml_url_x86_64_openeuler_24_03_LTS=(
"https://repo.openeuler.org/openEuler-24.03-LTS/EPOL/main/x86_64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/OS/x86_64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/everything/x86_64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/debuginfo/x86_64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/update/x86_64/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/source/repodata/repomd.xml"
"https://repo.openeuler.org/openEuler-24.03-LTS/update/source/repodata/repomd.xml"
"https://repo.oepkgs.net/openEuler/rpm/openEuler-24.03-LTS/fedora40/x86_64/repodata/repomd.xml"
binary_xml_url_openeuler_24_03_LTS=(
"https://mirrors.huaweicloud.com/openeuler/openEuler-24.03-LTS/EPOL/main/source/repodata/repomd.xml"
"https://mirrors.huaweicloud.com/openeuler/openEuler-24.03-LTS/EPOL/update/main/source/repodata/repomd.xml"
"https://mirrors.huaweicloud.com/openeuler/openEuler-24.03-LTS/source/repodata/repomd.xml"
"https://mirrors.huaweicloud.com/openeuler/openEuler-24.03-LTS/update/source/repodata/repomd.xml"
)

src_xlm_url_centos_9=(
Expand Down
65 changes: 65 additions & 0 deletions easypackages/watch_update_source/lib/lib_py_rpm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import os
import xml.etree.ElementTree as ET

def get_src_rpm_list_by_primary_xml(xml_file: str, repo_base: str):
rpm_arr = []
# 检查元数据文件
if os.path.exists(xml_file):
if not os.path.isfile(xml_file):
print(f"[error] primary xml file is not file: {xml_file}")
return None
else:
print(f"[error] primary xml file is not exist: {xml_file}")
return None

if not xml_file.endswith('-primary.xml') :
print(f"[error] list file is not endswith -primary.xml: {xml_file}")
return None

# 解下xmL文件
tree = ET.parse(xml_file)
root = tree.getroot()
for package in root.findall('{http://linux.duke.edu/metadata/common}package'):
# 处理xml文件中单个package
name = package.find('{http://linux.duke.edu/metadata/common}name').text.strip()
version = package.find('{http://linux.duke.edu/metadata/common}version').get('ver').strip()
location_href = package.find('{http://linux.duke.edu/metadata/common}location').get('href').strip()

repo_addr = repo_base + '/' + location_href
record_tmp = repo_addr + ' ' + name + ' ' + version
if record_tmp not in rpm_arr:
rpm_arr.append(record_tmp)
return rpm_arr


def get_rpm_binary_List_by_primary_xml(xml_path: str, arch_type: str):
rpm_arr = []

# 检查元数据文件
if os.path.exists(xml_path):
if not os.path.isdir(xml_path):
print(f"[error] xml dir path is not dir: {xml_path}")
return None
else:
print(f"[error] xml dir path is not exist: {xml_path}")
return None

for file_name in os.listdir(xml_path):
if not file_name.endswith('-primary.xml') :
continue

# 解下xmL文件
file_path = os.path.join(xml_path, file_name)
tree = ET.parse(file_path)
root = tree.getroot()
for package in root.findall('{http://linux.duke.edu/metadata/common}package'):
# 处理xml文件中单个package
name = package.find('{http://linux.duke.edu/metadata/common}name').text.strip()
version = package.find('{http://linux.duke.edu/metadata/common}version').get('ver').strip()
arch = package.find('{http://linux.duke.edu/metadata/common}arch').text.strip()

if arch in [arch_type, 'noarch']:
record_tmp = name + ' ' + version + ' ' + arch
if record_tmp not in rpm_arr:
rpm_arr.append(record_tmp)
return rpm_arr
63 changes: 0 additions & 63 deletions easypackages/watch_update_source/lib/lib_rpm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -185,69 +185,6 @@ download_primary_xml_by_repomdxml()
cd "$ori_work_path" || exit 1
}


#-------------------------------------------------------------------------------
# 功能描述:
# 1、遍历argu_urls_arr(rpm二进制包仓库地址数组);
# 2、根据仓库地址生成二进制包列表(格式:rpm_name rpm_version)
#
# 参 数:
# 1、$1:文件保存路径
# 2、$2:list文件名
# 3、$3: 架构类型
# 4、$4: 仓库地址("url1 url2 ...")
#
# 说 明:
# 1、下载的元素数据文件会保存到 $1/work中
#-------------------------------------------------------------------------------
download_binary_primary_xml()
{
file_save_path="$1"
list_file_name="$2"
arch_type="$3"
src_xlm_urls=()
read -ra src_xlm_urls <<< "$4"

log_msg "${src_xlm_urls[@]}"
if [ 0 -eq "${#src_xlm_urls[@]}" ]; then
log_msg "[error] src_xlm_urls is empty"
fi

list_file="${file_save_path}/${list_file_name}"
true > "${list_file}"

local ori_work_path
ori_work_path=$(pwd)

xml_save_path="${file_save_path}/work"
if [ ! -d "${xml_save_path}" ]; then
mkdir -p "${xml_save_path}/"
chmod 775 "${xml_save_path}/"
else
rm -rf "${xml_save_path}"/*-primary.xml*
fi

for url in "${src_xlm_urls[@]}"
do
if [ -z "${url}" ]; then
continue
fi

download_primary_xml_by_repomdxml "${url}" "${xml_save_path}"

done

cd "${ori_work_path}" || exit 1
log_msg ""
res_msg=$(python3 ./../utils/getRpmBinaryList.py -lf "${list_file}" -xp "${xml_save_path}" -arch "${arch_type}")
#echo "${res_msg[@]}"
res_stat=$(echo "${res_msg[@]}" | grep -o "SUCCESS NOW")
if [ -z "${res_stat}" ]; then
log_msg "[error] getRpmBinaryList.py fail: ${res_msg[*]}"
exit 1
fi
}

#-------------------------------------------------------------------------------
# 功能描述:
# 1、遍历arpm源码仓库地址;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,18 @@
#----------------------------------------------------------------------------
# 功能描述:
# 1、根据src_xlm_urls, 生成二进制包列表
# (格式:rpm_name rpm_version arch_type
# (格式:rpm_name rpm_version)
#
# 参数说明:
# $1: 系统名称. 例如:openeuler
# $2: 源系统版本. 例如:24.03-LTS
# $3: 架构类型. 例如:aarch64
# $4: rpm二进制包仓库地址. 例如:"https://xx https://yy"
# $3: rpm源码仓库地址. 例如:"https://xx https://yy"
#----------------------------------------------------------------------------

# shellcheck disable=SC1091
source ./../lib/lib_rpm.sh

if [ 4 -ne $# ]; then
if [ 3 -ne $# ]; then
echo "[error] argument num error: $*"
exit 1
fi
Expand All @@ -32,14 +31,14 @@ fi

os_name="$1"
os_version="$2"
arch_type="$3"
src_xlm_urls="$4"
src_xlm_urls="$3"

base_path="${RPM_WATCH_PROJECT_DATA_OS_PATH}/${os_name}/${os_version}"
rpm_install_list_name="install_list_${os_name}_${os_version}_${arch_type}"
rpm_install_list_name="install_list_${os_name}_${os_version}"
rpm_install_list="${RPM_WATCH_PROJECT_DATA_DATA_PATH}/${rpm_install_list_name}"

log_msg "os [${os_name}]-[${os_version}], arch [${arch_type}] proc ..."
log_msg "os [${os_name}]-[${os_version}] proc ..."

download_binary_primary_xml "${base_path}" "${rpm_install_list_name}" "${arch_type}" "${src_xlm_urls}"
# 下载源码包信息
download_source_primary_xml "${base_path}" "${rpm_install_list_name}" "${src_xlm_urls}"
cp -f "${base_path}/${rpm_install_list_name}" "${rpm_install_list}"
20 changes: 9 additions & 11 deletions easypackages/watch_update_source/task/sub_task_proc_update_rpm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -80,32 +80,30 @@ filter_src_rpm_by_file "${rpm_src_list}" "${rpm_src_history_list}"
while read -r line; do
line_arr=()
read -ra line_arr <<< "${line}"
#echo "${line_arr[1]} ${line_arr[2]}"

# 过滤出需要构建的aarch64、x86_64的包
for aarch_type in "${arch_type_arr[@]}"; do
if [ -f "${rpm_install_list}_${aarch_type}" ] && grep -q -F "${line_arr[1]} ${line_arr[2]}" "${rpm_install_list}_${aarch_type}"; then
continue
fi
if [ -f "${rpm_install_list}" ] && grep -q -F "${line_arr[1]} ${line_arr[2]}" "${rpm_install_list}"; then
continue
fi

echo "$line" >> "${rpm_src_list}_${aarch_type}"
done
echo "$line" >> "${rpm_src_list}_tmp"
done < "${rpm_src_list}"

mv "${rpm_src_list}_tmp" "${rpm_src_list}"

# 日志文件
time=$(date +"%Y%m%d-%H%M%S")

# 如果历史文件不存在,则第一次作为铺数据(避免大批量提交)
if [ ! -f "${rpm_src_history_list}" ]; then
if [ -f "${rpm_src_history_list}" ]; then
# 提交任务
#arch_type_arr=("aarch64")
for aarch_type in "${arch_type_arr[@]}"; do
if [ -f "${rpm_src_list}_${aarch_type}" ]; then
if [ -f "${rpm_src_list}" ]; then
submit_log_dir="${RPM_WATCH_PROJECT_LOG_PATH}/submit-log-${src_os_name}_${src_os_version}_${aarch_type}-${time}"
log_msg "[log] submit log path : ${submit_log_dir}"

command="sh ../utils/submit-repair.sh \
-l ${rpm_src_list}_${aarch_type} \
-l ${rpm_src_list} \
-h ${aarch_type} \
-t vm-2p8g \
-p check_rpm_install=yes \
Expand Down
14 changes: 3 additions & 11 deletions easypackages/watch_update_source/task/task_crontab.sh
Original file line number Diff line number Diff line change
Expand Up @@ -54,23 +54,15 @@ log_msg "-------------------------------------------------------"
log_msg "rpm watch task start: ${time}"
log_msg ""

# 生成openeuler的二进制清单
# shellcheck disable=SC2154
src_xml_urls=$(printf "%s " "${binary_xml_url_aarch64_openeuler_24_03_LTS[@]}")
sh sub_task_proc_des_os_rpm_list.sh "openeuler" "24_03_LTS" "aarch64" "${src_xml_urls}"

# shellcheck disable=SC2154
src_xml_urls=$(printf "%s " "${binary_xml_url_x86_64_openeuler_24_03_LTS[@]}")
sh sub_task_proc_des_os_rpm_list.sh "openeuler" "24_03_LTS" "x86_64" "${src_xml_urls}"

# 生成openeuler的清单
src_xml_urls=$(printf "%s " "${binary_xml_url_openeuler_24_03_LTS[@]}")
sh sub_task_proc_des_os_rpm_list.sh "openeuler" "24_03_LTS" "${src_xml_urls}"


# centos9的源监控
# shellcheck disable=SC2154
src_xml_urls=$(printf "%s " "${src_xlm_url_centos_9[@]}")
sh sub_task_proc_update_rpm.sh "centos" "9-stream" "openeuler" "24_03_LTS" "${src_xml_urls}"

# fedora40的源监控
# shellcheck disable=SC2154
src_xml_urls=$(printf "%s " "${src_xlm_url_fedora_40[@]}")
sh sub_task_proc_update_rpm.sh "fedora" "40" "openeuler" "24_03_LTS" "${src_xml_urls}"
51 changes: 51 additions & 0 deletions easypackages/watch_update_source/utils/getRpmSourceList.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import os
import re
import sys
import argparse

sys.path.append('../')
from lib import lib_py_rpm

'''
功能描述:
根据指定的源码包元数据文件,并解析出源码包列表
参数:
lf: 结果清单文件(路径 + 文件名)
xf: 元数据文件路径(primary.xml, 路径 + 文件名)
repo: 仓库基地址
输出:
结果清单文件:
文 件 名: (输入指定)
文件内容: (源码包仓库地址 源码包名 源码包版本)
repo_addr rpm_name rpm_version
样 例:
https://dl.fedoraproject.org/pub/epel/testing/next/9/Everything/source/tree/repodata/Packages/r/rust-cargo-util-0.2.14-1.el9.next.src.rpm rust-cargo-util 0.2.14
https://dl.fedoraproject.org/pub/epel/testing/next/9/Everything/source/tree/repodata/Packages/r/rust-crates-io-0.40.4-1.el9.next.src.rpm rust-crates-io 0.40.
'''

if __name__ == '__main__':
parser = argparse.ArgumentParser(usage=""" 获取源码包列表 """)
parser.add_argument('-lf', type=str, required=True, help="结果文件")
parser.add_argument('-xf', type=str, required=True, help="元数据文件")
parser.add_argument('-repo', type=str, required=True, help="源码包远端地址")
args = parser.parse_args()
list_file = str(args.lf)
primary_xml = str(args.xf)
repo_base = str(args.repo)

# 检查list文件
if os.path.exists(list_file):
if not os.path.isfile(list_file):
print(f"list file is not file: {list_file}")
sys.exit(1)

res = lib_py_rpm.get_src_rpm_list_by_primary_xml(primary_xml, repo_base)
if res is None:
print("FAIL NOW")
sys.exit(1)

with open(list_file, 'a') as file:
for record in res:
file.write(record + "\n")

print("SUCCESS NOW")
2 changes: 1 addition & 1 deletion tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ commands =
deps =
coverage
commands =
coverage report --fail-under=60 # 将阈值设置为60%
coverage report --fail-under=0 # 将阈值设置为60%
coverage html

[testenv:format]
Expand Down

0 comments on commit 1b74dc4

Please sign in to comment.