From 9529982d5b40cf1ce68df4824a9986107347ac22 Mon Sep 17 00:00:00 2001 From: wyyalt Date: Mon, 9 Oct 2023 10:13:13 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20=20=E6=8F=90=E4=BE=9B=20Agent=20?= =?UTF-8?q?=E5=8C=85=E7=AE=A1=E7=90=86=E5=90=8E=E5=8F=B0=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=20(closed=20#1683)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/agent/artifact_builder/base.py | 68 ++++++++++---- apps/backend/agent/tools.py | 5 +- .../components/collections/agent_new/base.py | 4 +- apps/backend/subscription/errors.py | 8 ++ .../steps/agent_adapter/adapter.py | 90 ++++++++++++++++--- apps/backend/subscription/views.py | 8 +- .../agent/artifact_builder/test_agent.py | 23 ++++- .../artifact_builder/test_manage_commands.py | 5 +- .../agent/artifact_builder/test_proxy.py | 1 + .../collections/agent_new/test_install.py | 13 ++- .../agent_new/test_push_agent_pkg_to_proxy.py | 1 + .../agent_new/test_push_upgrade_package.py | 1 + .../agent_new/test_run_upgrade_command.py | 1 + .../components/collections/agent_new/utils.py | 15 +++- .../agent_adapter/test_adapter.py | 28 ++++-- .../backend/tests/view/test_get_gse_config.py | 14 ++- apps/core/tag/constants.py | 8 -- apps/core/tag/targets/agent.py | 7 ++ apps/mock_data/common_unit/subscription.py | 2 +- apps/node_man/constants.py | 14 +++ apps/node_man/serializers/job.py | 24 +++-- 21 files changed, 265 insertions(+), 75 deletions(-) diff --git a/apps/backend/agent/artifact_builder/base.py b/apps/backend/agent/artifact_builder/base.py index 8bcb8e8c7..040e8c409 100644 --- a/apps/backend/agent/artifact_builder/base.py +++ b/apps/backend/agent/artifact_builder/base.py @@ -23,8 +23,9 @@ from apps.backend.agent.config_parser import GseConfigParser from apps.core.files import core_files_constants from apps.core.files.storage import get_storage -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.constants import TargetType from apps.core.tag.handlers import TagHandler +from apps.core.tag.targets import AgentTargetHelper from apps.node_man import constants, models from apps.utils import cache, files @@ -441,13 +442,57 @@ def _get_changelog(self, extract_dir: str) -> str: changelog: str = changelog_fs.read() return changelog - def update_or_create_record(self, artifact_meta_info: typing.Dict[str, typing.Any]): + def generate_location_path(self, upload_path: str, pkg_name: str) -> str: + if settings.STORAGE_TYPE == core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value: + location_path: str = f"{settings.BKREPO_ENDPOINT_URL}/generic/blueking/bknodeman/{upload_path}/{pkg_name}" + else: + location_path: str = f"http://{settings.BKAPP_LAN_IP}/{upload_path}/{pkg_name}" + + return location_path + + def update_or_create_package_records(self, package_infos: typing.List[typing.Dict[str, typing.Any]]): """ - 创建或更新制品记录，待 Agent 包管理完善 - :param artifact_meta_info: + 创建或更新制品记录 + :param package_infos: :return: """ - pass + for package_info in package_infos: + models.GsePackages.objects.update_or_create( + defaults={ + "pkg_size": package_info["package_upload_info"]["pkg_size"], + "pkg_path": package_info["package_upload_info"]["pkg_path"], + "md5": package_info["package_upload_info"]["md5"], + "location": self.generate_location_path( + package_info["package_upload_info"]["pkg_path"], + package_info["package_upload_info"]["pkg_name"], + ), + "version_log": package_info["artifact_meta_info"]["changelog"], + }, + pkg_name=package_info["package_upload_info"]["pkg_name"], + version=package_info["artifact_meta_info"]["version"], + project=package_info["artifact_meta_info"]["name"], + os=package_info["package_dir_info"]["os"], + cpu_arch=package_info["package_dir_info"]["cpu_arch"], + ) + logger.info( + f"[update_or_create_package_record] " + f"package name -> {package_info['package_upload_info']['pkg_name']} success" + ) + + if package_infos: + models.GsePackageDesc.objects.update_or_create( + defaults={ + "description": package_infos[0]["artifact_meta_info"]["changelog"], + }, + project=package_infos[0]["artifact_meta_info"]["name"], + category=constants.CategoryType.official, + ) + + logger.info( + f"[update_or_create_package_record] " + f"package desc -> {package_info['package_upload_info']['pkg_name']}, " + f"project -> {package_infos[0]['artifact_meta_info']['name']} success" + ) def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]): """ @@ -455,11 +500,12 @@ def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]) :param artifact_meta_info: :return: """ + agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map() for tag in self.tags: TagHandler.publish_tag_version( name=tag, target_type=TargetType.AGENT.value, - target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME], + target_id=agent_name_target_id_map[self.NAME], target_version=artifact_meta_info["version"], ) logger.info( @@ -517,14 +563,6 @@ def update_or_create_support_files(self, package_infos: typing.List[typing.Dict] agent_name=self.NAME, ) - def update_or_create_package_records(self, v): - """ - 创建或更新安装包记录，待 Agent 包管理完善 - :param package_infos: - :return: - """ - pass - def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]: """ 获取制品的基础信息、配置文件信息 @@ -591,8 +629,6 @@ def make( artifact_meta_info["operator"] = operator # Agent 包先导入文件源 -> 写配置文件 -> 创建包记录 -> 创建 Tag self.update_or_create_support_files(package_infos) - # TODO update_or_create_record & update_or_create_package_records 似乎是一样的功能？ - self.update_or_create_record(artifact_meta_info) self.update_or_create_package_records(package_infos) self.update_or_create_tag(artifact_meta_info) diff --git a/apps/backend/agent/tools.py b/apps/backend/agent/tools.py index 2ba78491c..f58591452 100644 --- a/apps/backend/agent/tools.py +++ b/apps/backend/agent/tools.py @@ -320,7 +320,7 @@ def check_run_commands(run_commands): def batch_gen_commands( - base_agent_setup_info: AgentSetupInfo, + agent_step_adapter, hosts: List[models.Host], pipeline_id: str, is_uninstall: bool, @@ -336,7 +336,6 @@ def batch_gen_commands( # 批量查出主机的属性并设置为property，避免在循环中进行ORM查询，提高效率 host_id__installation_tool_map = {} bk_host_ids = [host.bk_host_id for host in hosts] - base_agent_setup_info_dict: Dict[str, Any] = asdict(base_agent_setup_info) host_id_identity_map = { identity.bk_host_id: identity for identity in models.IdentityData.objects.filter(bk_host_id__in=bk_host_ids) } @@ -354,7 +353,7 @@ def batch_gen_commands( host_id__installation_tool_map[host.bk_host_id] = gen_commands( agent_setup_info=AgentSetupInfo( **{ - **base_agent_setup_info_dict, + **asdict(agent_step_adapter.get_host_setup_info(host)), "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False), } ), diff --git a/apps/backend/components/collections/agent_new/base.py b/apps/backend/components/collections/agent_new/base.py index 2e4931b33..19de5cd71 100644 --- a/apps/backend/components/collections/agent_new/base.py +++ b/apps/backend/components/collections/agent_new/base.py @@ -135,7 +135,7 @@ def get_agent_pkg_name( package_type = ("client", "proxy")[host.node_type == constants.NodeType.PROXY] agent_step_adapter = common_data.agent_step_adapter if not agent_step_adapter.is_legacy: - setup_info = agent_step_adapter.setup_info + setup_info = agent_step_adapter.get_host_setup_info(host) return f"{setup_info.name}-{setup_info.version}.tgz" # GSE1.0 的升级包是独立的，添加了 _upgrade 后缀 @@ -262,7 +262,7 @@ def get_host_id__installation_tool_map( host for host in hosts_need_gen_commands if host.bk_host_id in host_id__install_channel_map ] host_id__installation_tool_map = batch_gen_commands( - base_agent_setup_info=common_data.agent_step_adapter.setup_info, + agent_step_adapter=common_data.agent_step_adapter, hosts=hosts_need_gen_commands, pipeline_id=self.id, is_uninstall=is_uninstall, diff --git a/apps/backend/subscription/errors.py b/apps/backend/subscription/errors.py index 93f432e0a..b6d814673 100644 --- a/apps/backend/subscription/errors.py +++ b/apps/backend/subscription/errors.py @@ -169,3 +169,11 @@ class SubscriptionIncludeGrayBizError(AppBaseException): ERROR_CODE = 19 MESSAGE = _("订阅任务包含Gse2.0灰度业务，任务将暂缓执行无需重复点击") MESSAGE_TPL = _("订阅任务包含Gse2.0灰度业务，任务将暂缓执行无需重复点击") + + +class AgentPackageValidationError(AppBaseException): + """AgentPackage校验错误""" + + ERROR_CODE = 20 + MESSAGE = _("AgentPackage校验错误") + MESSAGE_TPL = _("{msg}") diff --git a/apps/backend/subscription/steps/agent_adapter/adapter.py b/apps/backend/subscription/steps/agent_adapter/adapter.py index aea7b186b..fb891b2d4 100644 --- a/apps/backend/subscription/steps/agent_adapter/adapter.py +++ b/apps/backend/subscription/steps/agent_adapter/adapter.py @@ -18,7 +18,8 @@ from apps.backend.agent.tools import fetch_proxies from apps.backend.constants import ProxyConfigFile -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.backend.subscription import errors +from apps.core.tag.constants import TargetType from apps.core.tag.targets import get_target_helper from apps.node_man import constants, models from apps.utils import cache @@ -34,11 +35,21 @@ LEGACY = "legacy" +class AgentVersionSerializer(serializers.Serializer): + os_cpu_arch = serializers.CharField(label="系统CPU架构", required=False) + bk_host_id = serializers.IntegerField(label="主机ID", required=False) + version = serializers.CharField(label="Agent Version") + + class AgentStepConfigSerializer(serializers.Serializer): name = serializers.CharField(required=False, label="构件名称") # LEGACY 表示旧版本 Agent，仅做兼容 version = serializers.CharField(required=False, label="构件版本", default=LEGACY) job_type = serializers.ChoiceField(required=True, choices=constants.JOB_TUPLE) + choice_version_type = serializers.ChoiceField( + required=False, choices=constants.AgentVersionType.list_choices(), label="选择Agent Version类型" + ) + version_map_list = AgentVersionSerializer(many=True) @dataclass @@ -57,6 +68,9 @@ class AgentStepAdapter: log_prefix: str = field(init=False) # 配置处理模块缓存 _config_handler_cache: typing.Dict[str, GseConfigHandler] = field(init=False) + _setup_info_cache: typing.Dict[str, base.AgentSetupInfo] = field(init=False) + _target_version_cache: typing.Dict[str, str] = field(init=False) + agent_name: str = field(init=False) def __post_init__(self): self.is_legacy = self.gse_version == GseVersion.V1.value @@ -64,6 +78,9 @@ def __post_init__(self): f"[{self.__class__.__name__}({self.subscription_step.step_id})] | {self.subscription_step} |" ) self._config_handler_cache: typing.Dict[str, GseConfigHandler] = {} + self._setup_info_cache: typing.Dict[str, base.AgentSetupInfo] = {} + self._target_version_cache: typing.Dict[str, str] = {} + self.agent_name = self.config.get("name") def get_config_handler(self, agent_name: str, target_version: str) -> GseConfigHandler: @@ -104,11 +121,12 @@ def _get_config( install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]], target_version: typing.Optional[typing.Dict[int, str]] = None, ) -> str: - agent_setup_info: base.AgentSetupInfo = self.setup_info + agent_setup_info: base.AgentSetupInfo = self.get_host_setup_info(host) # 目标版本优先使用传入版本，传入版本必不会是标签所以可直接使用 config_handler: GseConfigHandler = self.get_config_handler( agent_setup_info.name, target_version or agent_setup_info.version ) + config_tmpl_obj: base.AgentConfigTemplate = config_handler.get_matching_config_tmpl( os_type=host.os_type, cpu_arch=host.cpu_arch, @@ -168,29 +186,65 @@ def get_config( @property @cache.class_member_cache() - def setup_info(self) -> base.AgentSetupInfo: + def bk_host_id_version_map(self, host: models.Host) -> typing.Dict[int, str]: + return {versiom_map["bk_host_id"]: versiom_map["version"] for versiom_map in self.config["version_map_list"]} + + def get_host_setup_info(self, host: models.Host) -> base.AgentSetupInfo: """ 获取 Agent 设置信息 - TODO 后续如需支持多版本，该方法改造为 `get_host_setup_info`，根据维度进行缓存，参考 _config_handler_cache :return: """ # 如果版本号匹配到标签名称，取对应标签下的真实版本号，否则取原来的版本号 - agent_name: typing.Optional[str] = self.config.get("name") - if agent_name not in AGENT_NAME_TARGET_ID_MAP: - # 1.0 Install + if self.agent_name is None: + # 1.0 Install 或者 2.0统一版本 target_version = self.config.get("version") + setup_info_cache_key: str = f"agent_name_is_none:version:{target_version}" else: - target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( - target_id=AGENT_NAME_TARGET_ID_MAP[agent_name], - target_version=self.config.get("version"), - ) + if self.config["choice_version_type"] == constants.AgentVersionType.UNIFIED.value: + agent_version = self.config.get("version") + setup_info_cache_key: str = ( + f"agent_name:{self.agent_name}:" + f"type:{constants.AgentVersionType.UNIFIED.value}:version:{agent_version}" + ) + elif self.config["choice_version_type"] == constants.AgentVersionType.BY_SYSTEM_ARCH.value: + # TODO 按系统架构维度, 当前只支持按系统，后续需求完善按系统架构 + os_cpu_arch_version_list: typing.List[str] = [ + versiom_map["version"] + for versiom_map in self.config["version_map_list"] + if host.os_type.lower() in versiom_map["os_cpu_arch"] + ] + agent_version: str = os_cpu_arch_version_list[0] if os_cpu_arch_version_list else "stable" + setup_info_cache_key: str = ( + f"agent_name:{self.agent_name}:type:{constants.AgentVersionType.BY_SYSTEM_ARCH.value}:" + f"os:{host.os_type.lower()}:version:{agent_version}" + ) + else: + # 按主机维度 + agent_version: str = self.bk_host_id_version_map[host.bk_host_id] + + target_version_cache_key: str = f"agent_desc_id:{self.agent_desc.id}:agent_version:{agent_version}" + target_version: str = self._target_version_cache.get(target_version_cache_key) + if target_version is None: + target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( + target_id=self.agent_desc.id, + target_version=agent_version, + ) + self._target_version_cache[target_version_cache_key] = target_version - return base.AgentSetupInfo( + if self.config["choice_version_type"] != constants.AgentVersionType.BY_HOST.value: + agent_setup_info: typing.Optional[base.AgentSetupInfo] = self._setup_info_cache.get(setup_info_cache_key) + if agent_setup_info: + return agent_setup_info + + agent_setup_info: base.AgentSetupInfo = base.AgentSetupInfo( is_legacy=self.is_legacy, agent_tools_relative_dir=("agent_tools/agent2", "")[self.is_legacy], name=self.config.get("name"), version=target_version, ) + if self.config["choice_version_type"] != constants.AgentVersionType.BY_HOST.value: + self._setup_info_cache[setup_info_cache_key] = agent_setup_info + return agent_setup_info @staticmethod def validated_data(data, serializer) -> OrderedDict: @@ -204,3 +258,15 @@ def get_os_key(os_type: str, cpu_arch: str) -> str: os_type = os_type or constants.OsType.LINUX cpu_arch = cpu_arch or constants.CpuType.x86_64 return f"{os_type.lower()}-{cpu_arch}" + + @property + def agent_desc(self) -> models.GsePackageDesc: + if hasattr(self, "_agent_desc") and self._agent_desc: + return self._agent_desc + try: + agent_desc = models.GsePackageDesc.objects.get(project=self.agent_name) + except models.GsePackageDesc.DoesNotExist: + raise errors.AgentPackageValidationError(msg="GsePackageDesc [{name}] 不存在".format(name=self.agent_name)) + + setattr(self, "_agent_desc", agent_desc) + return self._agent_desc diff --git a/apps/backend/subscription/views.py b/apps/backend/subscription/views.py index 50ce35d67..dd4206ab8 100644 --- a/apps/backend/subscription/views.py +++ b/apps/backend/subscription/views.py @@ -15,7 +15,7 @@ from collections import defaultdict from dataclasses import asdict from functools import cmp_to_key, reduce -from typing import Any, Dict, List, Set +from typing import Dict, List, Set from django.core.cache import caches from django.db import transaction @@ -691,14 +691,14 @@ def fetch_commands(self, request): ap_id_obj_map: Dict[int, models.AccessPoint] = models.AccessPoint.ap_id_obj_map() host_ap: models.AccessPoint = ap_id_obj_map[host_ap_id] - base_agent_setup_info_dict: Dict[str, Any] = asdict( - AgentStepAdapter(subscription_step=sub_step_obj, gse_version=host_ap.gse_version).setup_info + agent_setup_adapter: AgentStepAdapter = AgentStepAdapter( + subscription_step=sub_step_obj, gse_version=host_ap.gse_version ) agent_setup_extra_info_dict = sub_inst.instance_info["host"].get("agent_setup_extra_info") or {} installation_tool = gen_commands( agent_setup_info=AgentSetupInfo( **{ - **base_agent_setup_info_dict, + **asdict(agent_setup_adapter.get_host_setup_info(host)), "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False), } ), diff --git a/apps/backend/tests/agent/artifact_builder/test_agent.py b/apps/backend/tests/agent/artifact_builder/test_agent.py index b6d2bbf51..5e108eaf9 100644 --- a/apps/backend/tests/agent/artifact_builder/test_agent.py +++ b/apps/backend/tests/agent/artifact_builder/test_agent.py @@ -16,8 +16,9 @@ from apps.backend.subscription.steps.agent_adapter.handlers import GseConfigHandler from apps.backend.tests.agent import template_env, utils -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.constants import TargetType from apps.core.tag.models import Tag +from apps.core.tag.targets.agent import AgentTargetHelper from apps.mock_data import utils as mock_data_utils from apps.node_man import constants, models @@ -49,10 +50,11 @@ def pkg_checker(self, version_str: str): ) self.assertTrue(os.path.exists(package_path)) - def tag_checker(self, target_id: int): + def tag_checker(self): + agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map() agent_target_version = Tag.objects.get( - target_id=target_id, + target_id=agent_name_target_id_map[self.NAME], name=self.OVERWRITE_VERSION, target_type=TargetType.AGENT.value, ).target_version @@ -106,12 +108,24 @@ def template_and_env_checker(self, version_str): self.assertTrue(models.GseConfigTemplate.objects.filter(**filter_kwargs).exists()) + def gse_package_and_desc_records_checker(self, version_str): + for package_os, cpu_arch in self.OS_CPU_CHOICES: + filter_kwargs: dict = { + "project": self.NAME, + "os": package_os, + "cpu_arch": cpu_arch, + "version": version_str, + } + self.assertTrue(models.GsePackages.objects.filter(**filter_kwargs).exists()) + self.assertTrue(models.GsePackageDesc.objects.filter(**{"project": filter_kwargs.pop("project")}).exists()) + def test_make(self): """测试安装包制作""" with self.ARTIFACT_BUILDER_CLASS(initial_artifact_path=self.ARCHIVE_PATH) as builder: builder.make() self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) def test_make__overwrite_version(self): """测试版本号覆盖""" @@ -129,7 +143,8 @@ def test_make__overwrite_version(self): self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) self.pkg_checker(version_str=self.OVERWRITE_VERSION) - self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.tag_checker() + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoTestCase(FileSystemTestCase): diff --git a/apps/backend/tests/agent/artifact_builder/test_manage_commands.py b/apps/backend/tests/agent/artifact_builder/test_manage_commands.py index ed2d7bb48..ee6f57429 100644 --- a/apps/backend/tests/agent/artifact_builder/test_manage_commands.py +++ b/apps/backend/tests/agent/artifact_builder/test_manage_commands.py @@ -17,7 +17,6 @@ from django.core.management import call_command from apps.backend.tests.agent import utils -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP from apps.mock_data import utils as mock_data_utils from apps.node_man import models @@ -44,6 +43,7 @@ def test_make(self): self.assertTrue(models.UploadPackage.objects.all().exists()) self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) def test_make__overwrite_version(self): """测试版本号覆盖""" @@ -51,7 +51,8 @@ def test_make__overwrite_version(self): self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) self.pkg_checker(version_str=self.OVERWRITE_VERSION) - self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.tag_checker() + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoImportAgentTestCase(FileSystemImportAgentTestCase): diff --git a/apps/backend/tests/agent/artifact_builder/test_proxy.py b/apps/backend/tests/agent/artifact_builder/test_proxy.py index 15a5468ad..d182aabe5 100644 --- a/apps/backend/tests/agent/artifact_builder/test_proxy.py +++ b/apps/backend/tests/agent/artifact_builder/test_proxy.py @@ -34,6 +34,7 @@ def test_make(self): builder.make() self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoTestCase(FileSystemTestCase): diff --git a/apps/backend/tests/components/collections/agent_new/test_install.py b/apps/backend/tests/components/collections/agent_new/test_install.py index 877d05755..ea028198e 100644 --- a/apps/backend/tests/components/collections/agent_new/test_install.py +++ b/apps/backend/tests/components/collections/agent_new/test_install.py @@ -161,6 +161,7 @@ def start_patch(self): fs.write("哈哈哈113343ddfd🐒") def setUp(self) -> None: + self.obj_factory.init_gse_package_desc() self.update_callback_url() self.init_mock_clients() self.init_hosts() @@ -389,7 +390,9 @@ async def connect(self): class InstallAgent2WindowsTestCase(InstallWindowsTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] - sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.config.update( + {"name": "gse_agent", "version": "2.0.0", "version_map_list": [], "choice_version_type": "unified"} + ) sub_step_obj.save(update_fields=["config"]) def structure_common_inputs(self): @@ -405,7 +408,7 @@ def test_batch_solution(self): gse_version=GseVersion.V2.value, ) installation_tool = gen_commands( - agent_step_adapter.setup_info, + agent_step_adapter.get_host_setup_info(host), host, mock_data_utils.JOB_TASK_PIPELINE_ID, is_uninstall=False, @@ -862,7 +865,9 @@ class LinuxAgent2InstallTestCase(InstallBaseTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] - sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.config.update( + {"name": "gse_agent", "version": "2.0.0", "version_map_list": [], "choice_version_type": "unified"} + ) sub_step_obj.save() def structure_common_inputs(self): @@ -878,7 +883,7 @@ def test_shell_solution(self): gse_version=GseVersion.V2.value, ) installation_tool = gen_commands( - agent_step_adapter.setup_info, + agent_step_adapter.get_host_setup_info(host), host, mock_data_utils.JOB_TASK_PIPELINE_ID, is_uninstall=False, diff --git a/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py b/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py index ed1922fc0..b29b4eba9 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py @@ -127,6 +127,7 @@ def component_cls(self): @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) sub_step_obj.save() diff --git a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py index 69af15a7e..9f8a56276 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py @@ -53,6 +53,7 @@ def get_default_case_name(cls) -> str: @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() models.Host.objects.filter(bk_host_id__in=cls.obj_factory.bk_host_ids).update(bk_agent_id=get_random_string()) sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) diff --git a/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py b/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py index f2cba8d4b..5233a94cd 100644 --- a/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py +++ b/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py @@ -85,6 +85,7 @@ def structure_common_inputs(self): @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) sub_step_obj.save() diff --git a/apps/backend/tests/components/collections/agent_new/utils.py b/apps/backend/tests/components/collections/agent_new/utils.py index cc6c3c9f2..ded153510 100644 --- a/apps/backend/tests/components/collections/agent_new/utils.py +++ b/apps/backend/tests/components/collections/agent_new/utils.py @@ -295,7 +295,11 @@ def structure_sub_step_data_list(self) -> List[Dict[str, Any]]: sub_step_data.update( { "subscription_id": self.sub_obj.id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": { + "job_type": constants.JobType.INSTALL_AGENT, + "version_map_list": [], + "choice_version_type": "unified", + }, } ) return [sub_step_data] @@ -371,6 +375,15 @@ def init_host_related_data_in_db(self): self.bulk_create_model(model=models.IdentityData, create_data_list=identity_data_list) self.identity_data_objs = models.IdentityData.objects.filter(bk_host_id__in=self.bk_host_ids) + @classmethod + def init_gse_package_desc(cls): + models.GsePackageDesc.objects.update_or_create( + defaults={"description": ""}, project="gse_agent", category="official" + ) + models.GsePackageDesc.objects.update_or_create( + defaults={"description": ""}, project="gse_proxy", category="official" + ) + def init_db(self): """ 初始化DB测试数据 diff --git a/apps/backend/tests/subscription/agent_adapter/test_adapter.py b/apps/backend/tests/subscription/agent_adapter/test_adapter.py index 62f4b6013..56b5aec76 100644 --- a/apps/backend/tests/subscription/agent_adapter/test_adapter.py +++ b/apps/backend/tests/subscription/agent_adapter/test_adapter.py @@ -13,6 +13,9 @@ from apps.backend.subscription.steps.agent_adapter.adapter import AgentStepAdapter from apps.backend.tests.agent.utils import VERSION, AgentBaseTestCase, ProxyBaseTestCase +from apps.backend.tests.components.collections.agent_new.utils import ( + AgentTestObjFactory, +) from apps.mock_data import common_unit from apps.node_man import constants, models from apps.utils import basic @@ -31,6 +34,7 @@ def setUpTestData(cls): super().setUpTestData() host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) cls.host = models.Host.objects.create(**host_model_data) + AgentTestObjFactory.init_gse_package_desc() # 创建订阅相关数据 sub_inst_data = basic.remove_keys_from_dict( @@ -45,7 +49,7 @@ def setUpTestData(cls): **sub_step_data, **{ "subscription_id": cls.sub_inst_record_obj.subscription_id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, }, } ) @@ -85,6 +89,8 @@ def setUpTestData(cls): "job_type": constants.JobType.INSTALL_AGENT, "name": constants.GsePackageCode.PROXY.value, "version": VERSION, + "version_map_list": [], + "choice_version_type": "unified", } cls.sub_step_obj.save() cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=GseVersion.V2.value) @@ -93,12 +99,13 @@ def test_get_config(self): self.clear_agent_data() self.host.node_type = "PROXY" self.host.bk_cloud_id = 1 + agent_setup_info = self.agent_step_adapter.get_host_setup_info(self.host) for config_name in constants.GsePackageTemplate.PROXY.value: self.get_config(config_name) self.assertEqual( self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ) .get_matching_config_tmpl(self.host.os_type, self.host.cpu_arch, config_name) .agent_name_from, @@ -108,8 +115,8 @@ def test_get_config(self): self.get_config(config_name) self.assertEqual( self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ) .get_matching_config_tmpl(self.host.os_type, self.host.cpu_arch, config_name) .agent_name_from, @@ -128,16 +135,17 @@ def clear_agent_data(cls): pass def test_get_env(self): + agent_setup_info = self.agent_step_adapter.get_host_setup_info(self.host) agent_env = self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ).get_matching_template_env(self.host.os_type, self.host.cpu_arch, constants.GsePackageCode.AGENT.value) self.assertEqual(agent_env["BK_GSE_HOME_DIR"], "/usr/local/gse/agent") proxy_env = self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ).get_matching_template_env(self.host.os_type, self.host.cpu_arch, constants.GsePackageCode.PROXY.value) self.assertEqual(proxy_env["BK_GSE_HOME_DIR"], "/usr/local/gse/proxy") @@ -155,6 +163,8 @@ def setUpTestData(cls): "job_type": constants.JobType.INSTALL_AGENT, "name": constants.GsePackageCode.AGENT.value, "version": VERSION, + "version_map_list": [], + "choice_version_type": "unified", } cls.sub_step_obj.save() cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=GseVersion.V2.value) diff --git a/apps/backend/tests/view/test_get_gse_config.py b/apps/backend/tests/view/test_get_gse_config.py index 37e29d55a..cc9e98090 100644 --- a/apps/backend/tests/view/test_get_gse_config.py +++ b/apps/backend/tests/view/test_get_gse_config.py @@ -38,7 +38,7 @@ def setUpTestData(cls): host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) cls.host = models.Host.objects.create(**host_model_data) models.AccessPoint.objects.all().update(gse_version=cls.GSE_VERSION) - + models.GsePackageDesc.objects.create(project="gse_agent", description="", category="official") # 创建订阅相关数据 sub_inst_data = basic.remove_keys_from_dict( origin_data=common_unit.subscription.SUB_INST_RECORD_MODEL_DATA, keys=["id"] @@ -52,7 +52,7 @@ def setUpTestData(cls): **sub_step_data, **{ "subscription_id": cls.sub_inst_record_obj.subscription_id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, }, } ) @@ -117,13 +117,19 @@ def setUp(self) -> None: @classmethod def setUpTestData(cls): super().setUpTestData() - cls.sub_step_obj.config = {"job_type": constants.JobType.INSTALL_AGENT, "name": "gse_agent", "version": "2.0.0"} + cls.sub_step_obj.config = { + "job_type": constants.JobType.INSTALL_AGENT, + "name": "gse_agent", + "version": "2.0.0", + "version_map_list": [], + "choice_version_type": "unified", + } cls.sub_step_obj.save() cls.redis_agent_conf_key = REDIS_AGENT_CONF_KEY_TPL.format( file_name=cls.agent_step_adapter.get_main_config_filename(), sub_inst_id=cls.sub_inst_record_obj.id ) cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=cls.GSE_VERSION) - target_version = cls.agent_step_adapter.setup_info.version + target_version = cls.agent_step_adapter.get_host_setup_info(cls.host).version models.GseConfigEnv.objects.create( agent_name="gse_agent", version=target_version, diff --git a/apps/core/tag/constants.py b/apps/core/tag/constants.py index 8b0c27614..cf922484a 100644 --- a/apps/core/tag/constants.py +++ b/apps/core/tag/constants.py @@ -13,7 +13,6 @@ from django.utils.translation import ugettext_lazy as _ -from apps.node_man.constants import GsePackageCode from apps.utils.enum import EnhanceEnum @@ -37,10 +36,3 @@ class TagChangeAction(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DELETE: _("删除标签"), cls.CREATE: _("新建标签"), cls.UPDATE: _("更新版本"), cls.OVERWRITE: _("同版本覆盖更新")} - - -# TODO: target_id 临时写死 -AGENT_NAME_TARGET_ID_MAP: Dict[str, int] = { - GsePackageCode.AGENT.value: 1, - GsePackageCode.PROXY.value: 2, -} diff --git a/apps/core/tag/targets/agent.py b/apps/core/tag/targets/agent.py index 400acd7ac..9f51fcae0 100644 --- a/apps/core/tag/targets/agent.py +++ b/apps/core/tag/targets/agent.py @@ -11,8 +11,10 @@ import logging +import typing from apps.core.tag.models import Tag +from apps.node_man.models import GsePackageDesc from .. import constants from . import base @@ -35,3 +37,8 @@ def _publish_tag_version(self): def _delete_tag_version(self): return super()._delete_tag_version() + + @classmethod + def get_agent_name_target_id_map(cls) -> typing.Dict[str, int]: + package_descs = GsePackageDesc.objects.values("project", "id") + return {package_desc["project"]: package_desc["id"] for package_desc in package_descs} diff --git a/apps/mock_data/common_unit/subscription.py b/apps/mock_data/common_unit/subscription.py index 39d003403..3b2725f54 100644 --- a/apps/mock_data/common_unit/subscription.py +++ b/apps/mock_data/common_unit/subscription.py @@ -92,7 +92,7 @@ "subscription_id": DEFAULT_SUBSCRIPTION_ID, "step_id": constants.SubStepType.AGENT.lower(), "type": constants.SubStepType.AGENT, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, "params": {"context": {}, "blueking_language": "zh-hans"}, } diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index 4905c7972..b3cf89689 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -1136,3 +1136,17 @@ class CommonExecutionSolutionStepType(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DEPENDENCIES: _("依赖文件"), cls.COMMANDS: _("命令")} + + +class AgentVersionType(EnhanceEnum): + UNIFIED = "unified" + BY_HOST = "by_host" + BY_SYSTEM_ARCH = "by_system_arch" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.UNIFIED: _("统一的版本"), + cls.BY_HOST: _("按主机的"), + cls.BY_SYSTEM_ARCH: _("按系统架构"), + } diff --git a/apps/node_man/serializers/job.py b/apps/node_man/serializers/job.py index 1985fe825..700fa4f0e 100644 --- a/apps/node_man/serializers/job.py +++ b/apps/node_man/serializers/job.py @@ -16,7 +16,10 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from apps.backend.subscription.steps.agent_adapter.adapter import LEGACY +from apps.backend.subscription.steps.agent_adapter.adapter import ( + LEGACY, + AgentVersionSerializer, +) from apps.core.gray.constants import INSTALL_OTHER_AGENT_AP_ID_OFFSET from apps.core.gray.handlers import GrayHandler from apps.core.gray.tools import GrayTools @@ -51,11 +54,18 @@ def set_agent_setup_info_to_attrs(attrs): # 如果开启 DHCP，安装 2.0 Agent，开启 AgentID 特性 # 在执行模块根据主机接入点所属的 GSE 版本决定是否采用下列的 agent_setup_info name = ("gse_agent", "gse_proxy")[attrs["node_type"] == "PROXY"] + # attrs["agent_setup_info"]["name"] = name + # 处理重装类型setup_info结构 + agent_setup_info = attrs.get("agent_setup_info", {}) + global_settings_agent_version = models.GlobalSettings.get_config( + models.GlobalSettings.KeyEnum.GSE_AGENT2_VERSION.value, default="stable" + ) + attrs["agent_setup_info"] = { "name": name, - "version": models.GlobalSettings.get_config( - models.GlobalSettings.KeyEnum.GSE_AGENT2_VERSION.value, default="stable" - ), + "version": agent_setup_info.get("version") or global_settings_agent_version, + "choice_version_type": agent_setup_info.get("choice_version_type") or constants.AgentVersionType.UNIFIED.value, + "version_map_list": agent_setup_info.get("version_map_list", []), } @@ -112,7 +122,6 @@ def backfill_bk_host_id(self, hosts): else: sub_query.children.append(("inner_ipv6", _host["inner_ipv6"])) ip_key = _host["inner_ipv6"] - cloud_ip_host_info_map[f"{_host['bk_cloud_id']}:{ip_key}"] = _host query_params.children.append(sub_query) @@ -257,6 +266,11 @@ class AgentSetupInfoSerializer(serializers.Serializer): # LEGACY 表示旧版本 Agent，仅做兼容 version = serializers.CharField(required=False, label="构件版本", default=LEGACY) + choice_version_type = serializers.ChoiceField( + required=False, choices=constants.AgentVersionType.list_choices(), label=_("选择Agent Version类型") + ) + version_map_list = AgentVersionSerializer(required=False, many=True) + class ScriptHook(serializers.Serializer): name = serializers.CharField(label=_("脚本名称"), min_length=1)