diff --git a/apps/backend/agent/artifact_builder/base.py b/apps/backend/agent/artifact_builder/base.py index 8bcb8e8c7..c569e5eee 100644 --- a/apps/backend/agent/artifact_builder/base.py +++ b/apps/backend/agent/artifact_builder/base.py @@ -23,8 +23,9 @@ from apps.backend.agent.config_parser import GseConfigParser from apps.core.files import core_files_constants from apps.core.files.storage import get_storage -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.constants import TargetType from apps.core.tag.handlers import TagHandler +from apps.core.tag.targets import AgentTargetHelper from apps.node_man import constants, models from apps.utils import cache, files @@ -65,6 +66,7 @@ def __init__( overwrite_version: typing.Optional[str] = None, tags: typing.Optional[typing.List[str]] = None, enable_agent_pkg_manage: bool = False, + username: str = "", ): """ :param initial_artifact_path: 原始制品所在路径 @@ -84,6 +86,7 @@ def __init__( self.applied_tmp_dirs = set() # 文件源 self.storage = get_storage(file_overwrite=True) + self.username = username @staticmethod def download_file(file_path: str, target_path: str): @@ -428,26 +431,76 @@ def _get_version(self, extract_dir: str) -> str: raise exceptions.NotSemanticVersionError({"version": version_match}) @cache.class_member_cache() - def _get_changelog(self, extract_dir: str) -> str: + def _get_description(self, extract_dir: str) -> typing.Tuple[str, str]: """ 获取版本日志 :param extract_dir: 解压目录 :return: """ - changelog_file_path: str = os.path.join(extract_dir, "CHANGELOG.md") - if not os.path.exists(changelog_file_path): - raise exceptions.FileNotExistError(_("版本日志文件不存在")) - with open(changelog_file_path, "r", encoding="utf-8") as changelog_fs: - changelog: str = changelog_fs.read() - return changelog - - def update_or_create_record(self, artifact_meta_info: typing.Dict[str, typing.Any]): + description_file_path: str = os.path.join(extract_dir, "DESCRIPTION") + description_en_file_path: str = os.path.join(extract_dir, "DESCRIPTION_EN") + description, description_en = "", "" + if os.path.exists(description_file_path): + with open(description_file_path, "r", encoding="utf-8") as description_fs: + description: str = description_fs.read() + + if os.path.exists(description_en_file_path): + with open(description_en_file_path, "r", encoding="utf-8") as description_en_fs: + description_en: str = description_en_fs.read() + return description, description_en + + def generate_location_path(self, upload_path: str, pkg_name: str) -> str: + if settings.STORAGE_TYPE == core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value: + location_path: str = f"{settings.BKREPO_ENDPOINT_URL}/generic/blueking/bknodeman/{upload_path}/{pkg_name}" + else: + location_path: str = f"http://{settings.BKAPP_LAN_IP}/{upload_path}/{pkg_name}" + + return location_path + + def update_or_create_package_records(self, package_infos: typing.List[typing.Dict[str, typing.Any]]): """ - 创建或更新制品记录，待 Agent 包管理完善 - :param artifact_meta_info: + 创建或更新制品记录 + :param package_infos: :return: """ - pass + for package_info in package_infos: + models.GsePackages.objects.update_or_create( + defaults={ + "pkg_size": package_info["package_upload_info"]["pkg_size"], + "pkg_path": package_info["package_upload_info"]["pkg_path"], + "md5": package_info["package_upload_info"]["md5"], + "location": self.generate_location_path( + package_info["package_upload_info"]["pkg_path"], + package_info["package_upload_info"]["pkg_name"], + ), + "version_log": package_info["artifact_meta_info"]["description"], + "version_log_en": package_info["artifact_meta_info"]["description_en"], + }, + pkg_name=package_info["package_upload_info"]["pkg_name"], + version=package_info["artifact_meta_info"]["version"], + project=package_info["artifact_meta_info"]["name"], + os=package_info["package_dir_info"]["os"], + cpu_arch=package_info["package_dir_info"]["cpu_arch"], + ) + logger.info( + f"[update_or_create_package_record] " + f"package name -> {package_info['package_upload_info']['pkg_name']} success" + ) + + if package_infos: + models.GsePackageDesc.objects.update_or_create( + defaults={ + "description": package_infos[0]["artifact_meta_info"]["description"], + }, + project=package_infos[0]["artifact_meta_info"]["name"], + category=constants.CategoryType.official, + ) + + logger.info( + f"[update_or_create_package_record] " + f"package desc -> {package_info['package_upload_info']['pkg_name']}, " + f"project -> {package_infos[0]['artifact_meta_info']['name']} success" + ) def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]): """ @@ -455,11 +508,12 @@ def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]) :param artifact_meta_info: :return: """ + agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map() for tag in self.tags: TagHandler.publish_tag_version( name=tag, target_type=TargetType.AGENT.value, - target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME], + target_id=agent_name_target_id_map[self.NAME], target_version=artifact_meta_info["version"], ) logger.info( @@ -517,14 +571,6 @@ def update_or_create_support_files(self, package_infos: typing.List[typing.Dict] agent_name=self.NAME, ) - def update_or_create_package_records(self, v): - """ - 创建或更新安装包记录，待 Agent 包管理完善 - :param package_infos: - :return: - """ - pass - def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]: """ 获取制品的基础信息、配置文件信息 @@ -535,13 +581,14 @@ def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.An version_str: str = self._get_version(extract_dir) # 配置文件 support_files_info = self._get_support_files_info(extract_dir) - # changelog - changelog: str = self._get_changelog(extract_dir) + # description + description, description_en = self._get_description(extract_dir) return { "name": self.NAME, "version": version_str, - "changelog": changelog, + "description": description, + "description_en": description_en, "support_files_info": support_files_info, } @@ -591,8 +638,6 @@ def make( artifact_meta_info["operator"] = operator # Agent 包先导入文件源 -> 写配置文件 -> 创建包记录 -> 创建 Tag self.update_or_create_support_files(package_infos) - # TODO update_or_create_record & update_or_create_package_records 似乎是一样的功能？ - self.update_or_create_record(artifact_meta_info) self.update_or_create_package_records(package_infos) self.update_or_create_tag(artifact_meta_info) diff --git a/apps/backend/agent/artifact_builder/proxy.py b/apps/backend/agent/artifact_builder/proxy.py index 5223ca675..3d9b94d39 100644 --- a/apps/backend/agent/artifact_builder/proxy.py +++ b/apps/backend/agent/artifact_builder/proxy.py @@ -36,6 +36,7 @@ class ProxyArtifactBuilder(base.BaseArtifactBuilder): PROXY_SVR_EXES: typing.List[str] = ["gse_data", "gse_file"] def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir: str): + # todo: 是否使用Archive(initial_artifact_local_path).extractall(extract_dir, auto_create_dir=True) with tarfile.open(name=initial_artifact_local_path) as tf: tf.extractall(path=extract_dir) diff --git a/apps/backend/agent/tools.py b/apps/backend/agent/tools.py index 6887b1776..a10a18447 100644 --- a/apps/backend/agent/tools.py +++ b/apps/backend/agent/tools.py @@ -334,7 +334,7 @@ def check_run_commands(run_commands): def batch_gen_commands( - base_agent_setup_info: AgentSetupInfo, + agent_step_adapter, hosts: List[models.Host], pipeline_id: str, is_uninstall: bool, @@ -350,7 +350,6 @@ def batch_gen_commands( # 批量查出主机的属性并设置为property，避免在循环中进行ORM查询，提高效率 host_id__installation_tool_map = {} bk_host_ids = [host.bk_host_id for host in hosts] - base_agent_setup_info_dict: Dict[str, Any] = asdict(base_agent_setup_info) host_id_identity_map = { identity.bk_host_id: identity for identity in models.IdentityData.objects.filter(bk_host_id__in=bk_host_ids) } @@ -368,7 +367,7 @@ def batch_gen_commands( host_id__installation_tool_map[host.bk_host_id] = gen_commands( agent_setup_info=AgentSetupInfo( **{ - **base_agent_setup_info_dict, + **asdict(agent_step_adapter.get_host_setup_info(host)), "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False), } ), diff --git a/apps/backend/agent/views.py b/apps/backend/agent/views.py new file mode 100644 index 000000000..a4a6196e9 --- /dev/null +++ b/apps/backend/agent/views.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import typing + +from django.utils.translation import get_language +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.status import HTTP_200_OK + +from apps.generic import ApiMixinModelViewSet as ModelViewSet +from apps.generic import ValidationMixin +from apps.node_man.serializers import package_manage as pkg_manage +from apps.node_man.tools.gse_package import GsePackageTools +from apps.node_man.views.package_manage import PACKAGE_MANAGE_VIEW_TAGS +from common.utils.drf_utils import swagger_auto_schema + + +class AgentViewSet(ModelViewSet, ValidationMixin): + """ + agent相关API + """ + + @swagger_auto_schema( + operation_summary="解析Agent包", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.ParseResponseSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.ParseSerializer) + def parse(self, request): + """ + return: { + "description": "test", + "packages": [ + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86_64.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86_64", + }, + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86", + }, + ], + } + """ + validated_data = self.validated_data + + # 获取最新的agent上传包记录 + upload_package_obj = GsePackageTools.get_latest_upload_record(file_name=validated_data["file_name"]) + + # 区分agent和proxy包 + project, artifact_builder_class = GsePackageTools.distinguish_gse_package( + file_path=upload_package_obj.file_path + ) + + # 解析包 + with artifact_builder_class(initial_artifact_path=upload_package_obj.file_path) as builder: + extract_dir, package_dir_infos = builder.list_package_dir_infos() + artifact_meta_info: typing.Dict[str, typing.Any] = builder.get_artifact_meta_info(extract_dir) + + language = get_language() + res = { + "description": artifact_meta_info["description"] + if language == "zh-hans" + else artifact_meta_info["description_en"], + "packages": package_dir_infos, + } + + context = { + "project": project, + "version": artifact_meta_info["version"], + } + + return Response(pkg_manage.ParseResponseSerializer(res, context=context).data) diff --git a/apps/backend/components/collections/agent_new/base.py b/apps/backend/components/collections/agent_new/base.py index 3c8fad0cd..9679ab59c 100644 --- a/apps/backend/components/collections/agent_new/base.py +++ b/apps/backend/components/collections/agent_new/base.py @@ -141,7 +141,7 @@ def get_agent_pkg_name( package_type = ("client", "proxy")[host.node_type == constants.NodeType.PROXY] agent_step_adapter = common_data.agent_step_adapter if not agent_step_adapter.is_legacy: - setup_info = agent_step_adapter.setup_info + setup_info = agent_step_adapter.get_host_setup_info(host) return f"{setup_info.name}-{setup_info.version}.tgz" # GSE1.0 的升级包是独立的，添加了 _upgrade 后缀 @@ -297,7 +297,7 @@ def get_host_id__installation_tool_map( host for host in hosts_need_gen_commands if host.bk_host_id in host_id__install_channel_map ] host_id__installation_tool_map = batch_gen_commands( - base_agent_setup_info=common_data.agent_step_adapter.setup_info, + agent_step_adapter=common_data.agent_step_adapter, hosts=hosts_need_gen_commands, pipeline_id=self.id, is_uninstall=is_uninstall, diff --git a/apps/backend/constants.py b/apps/backend/constants.py index e024154c6..68f7e98f4 100644 --- a/apps/backend/constants.py +++ b/apps/backend/constants.py @@ -64,6 +64,7 @@ class InstNodeType(object): "INSTALL_AGENT", "REINSTALL_AGENT", "UPGRADE_AGENT", + "DOWNGRADE_AGENT", "RESTART_AGENT", "UNINSTALL_AGENT", "RELOAD_AGENT", diff --git a/apps/backend/exceptions.py b/apps/backend/exceptions.py index a2e810d9f..cbd73129d 100644 --- a/apps/backend/exceptions.py +++ b/apps/backend/exceptions.py @@ -94,3 +94,9 @@ class AgentConfigTemplateEnvNotExistError(BackendBaseException): MESSAGE = _("配置模板Env不存在") MESSAGE_TPL = _("配置模板Env不存在[{name}-{version}-{os_type}-{cpu_arch}]不存在") ERROR_CODE = 15 + + +class ModelInstanceNotFoundError(BackendBaseException): + MESSAGE = _("模型对象不存在") + MESSAGE_TPL = _("模型对象 -> [{model_name}] 不存在") + ERROR_CODE = 16 diff --git a/apps/backend/subscription/errors.py b/apps/backend/subscription/errors.py index 93f432e0a..b6d814673 100644 --- a/apps/backend/subscription/errors.py +++ b/apps/backend/subscription/errors.py @@ -169,3 +169,11 @@ class SubscriptionIncludeGrayBizError(AppBaseException): ERROR_CODE = 19 MESSAGE = _("订阅任务包含Gse2.0灰度业务，任务将暂缓执行无需重复点击") MESSAGE_TPL = _("订阅任务包含Gse2.0灰度业务，任务将暂缓执行无需重复点击") + + +class AgentPackageValidationError(AppBaseException): + """AgentPackage校验错误""" + + ERROR_CODE = 20 + MESSAGE = _("AgentPackage校验错误") + MESSAGE_TPL = _("{msg}") diff --git a/apps/backend/subscription/steps/agent.py b/apps/backend/subscription/steps/agent.py index ec6bb466b..bb627dff2 100644 --- a/apps/backend/subscription/steps/agent.py +++ b/apps/backend/subscription/steps/agent.py @@ -20,6 +20,7 @@ from apps.node_man import constants, models from apps.node_man.constants import DEFAULT_CLOUD from apps.node_man.models import GsePluginDesc, SubscriptionStep +from apps.node_man.tools.gse_package import GsePackageTools from env.constants import GseVersion from pipeline import builder from pipeline.builder import Var @@ -49,6 +50,7 @@ def get_supported_actions(self): ReinstallAgent, UninstallAgent, UpgradeAgent, + DowngradeAgent, RestartAgent, InstallProxy, ReinstallProxy, @@ -115,6 +117,24 @@ def make_instances_migrate_actions(self, instances, auto_trigger=False, preview_ continue instance_actions[instance_id] = job_type_map[self.subscription_step.config["job_type"]] + if instance_actions[instance_id] == backend_const.ActionNameType.UPGRADE_AGENT: + version_map: Dict[int, str] = { + version_map["bk_host_id"]: version_map["version"] + for version_map in self.subscription_step.config.get("version_map_list", []) + } + + bk_host_id: int = instance["host"]["bk_host_id"] + agent_version: str = models.ProcessStatus.objects.get( + bk_host_id=bk_host_id, name=models.ProcessStatus.GSE_AGENT_PROCESS_NAME + ).version + if all( + [ + version_map.get(bk_host_id), + not GsePackageTools.compare_version(version_map[bk_host_id], agent_version), + ] + ): + instance_actions[instance_id] = backend_const.ActionNameType.DOWNGRADE_AGENT + return {"instance_actions": instance_actions, "migrate_reasons": migrate_reasons} @@ -325,6 +345,13 @@ def _generate_activities(self, agent_manager: AgentManager): return activities, None +class DowngradeAgent(UpgradeAgent): + """回退Agent""" + + ACTION_NAME = backend_const.ActionNameType.DOWNGRADE_AGENT + ACTION_DESCRIPTION = _("回退") + + class RestartAgent(AgentAction): """ 重启Agent diff --git a/apps/backend/subscription/steps/agent_adapter/adapter.py b/apps/backend/subscription/steps/agent_adapter/adapter.py index aea7b186b..dfdafe305 100644 --- a/apps/backend/subscription/steps/agent_adapter/adapter.py +++ b/apps/backend/subscription/steps/agent_adapter/adapter.py @@ -18,7 +18,8 @@ from apps.backend.agent.tools import fetch_proxies from apps.backend.constants import ProxyConfigFile -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.backend.subscription import errors +from apps.core.tag.constants import TargetType from apps.core.tag.targets import get_target_helper from apps.node_man import constants, models from apps.utils import cache @@ -34,11 +35,21 @@ LEGACY = "legacy" +class AgentVersionSerializer(serializers.Serializer): + os_cpu_arch = serializers.CharField(label="系统CPU架构", required=False) + bk_host_id = serializers.IntegerField(label="主机ID", required=False) + version = serializers.CharField(label="Agent Version") + + class AgentStepConfigSerializer(serializers.Serializer): name = serializers.CharField(required=False, label="构件名称") # LEGACY 表示旧版本 Agent，仅做兼容 version = serializers.CharField(required=False, label="构件版本", default=LEGACY) job_type = serializers.ChoiceField(required=True, choices=constants.JOB_TUPLE) + choice_version_type = serializers.ChoiceField( + required=False, choices=constants.AgentVersionType.list_choices(), label="选择Agent Version类型" + ) + version_map_list = AgentVersionSerializer(many=True) @dataclass @@ -57,6 +68,9 @@ class AgentStepAdapter: log_prefix: str = field(init=False) # 配置处理模块缓存 _config_handler_cache: typing.Dict[str, GseConfigHandler] = field(init=False) + _setup_info_cache: typing.Dict[str, base.AgentSetupInfo] = field(init=False) + _target_version_cache: typing.Dict[str, str] = field(init=False) + agent_name: str = field(init=False) def __post_init__(self): self.is_legacy = self.gse_version == GseVersion.V1.value @@ -64,6 +78,9 @@ def __post_init__(self): f"[{self.__class__.__name__}({self.subscription_step.step_id})] | {self.subscription_step} |" ) self._config_handler_cache: typing.Dict[str, GseConfigHandler] = {} + self._setup_info_cache: typing.Dict[str, base.AgentSetupInfo] = {} + self._target_version_cache: typing.Dict[str, str] = {} + self.agent_name = self.config.get("name") def get_config_handler(self, agent_name: str, target_version: str) -> GseConfigHandler: @@ -104,11 +121,12 @@ def _get_config( install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]], target_version: typing.Optional[typing.Dict[int, str]] = None, ) -> str: - agent_setup_info: base.AgentSetupInfo = self.setup_info + agent_setup_info: base.AgentSetupInfo = self.get_host_setup_info(host) # 目标版本优先使用传入版本，传入版本必不会是标签所以可直接使用 config_handler: GseConfigHandler = self.get_config_handler( agent_setup_info.name, target_version or agent_setup_info.version ) + config_tmpl_obj: base.AgentConfigTemplate = config_handler.get_matching_config_tmpl( os_type=host.os_type, cpu_arch=host.cpu_arch, @@ -168,29 +186,65 @@ def get_config( @property @cache.class_member_cache() - def setup_info(self) -> base.AgentSetupInfo: + def bk_host_id_version_map(self) -> typing.Dict[int, str]: + return {versiom_map["bk_host_id"]: versiom_map["version"] for versiom_map in self.config["version_map_list"]} + + def get_host_setup_info(self, host: models.Host) -> base.AgentSetupInfo: """ 获取 Agent 设置信息 - TODO 后续如需支持多版本，该方法改造为 `get_host_setup_info`，根据维度进行缓存，参考 _config_handler_cache :return: """ # 如果版本号匹配到标签名称，取对应标签下的真实版本号，否则取原来的版本号 - agent_name: typing.Optional[str] = self.config.get("name") - if agent_name not in AGENT_NAME_TARGET_ID_MAP: - # 1.0 Install + if self.agent_name is None: + # 1.0 Install 或者 2.0统一版本 target_version = self.config.get("version") + setup_info_cache_key: str = f"agent_name_is_none:version:{target_version}" else: - target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( - target_id=AGENT_NAME_TARGET_ID_MAP[agent_name], - target_version=self.config.get("version"), - ) + if self.config["choice_version_type"] == constants.AgentVersionType.UNIFIED.value: + agent_version = self.config.get("version") + setup_info_cache_key: str = ( + f"agent_name:{self.agent_name}:" + f"type:{constants.AgentVersionType.UNIFIED.value}:version:{agent_version}" + ) + elif self.config["choice_version_type"] == constants.AgentVersionType.BY_SYSTEM_ARCH.value: + # TODO 按系统架构维度, 当前只支持按系统，后续需求完善按系统架构 + os_cpu_arch_version_list: typing.List[str] = [ + versiom_map["version"] + for versiom_map in self.config["version_map_list"] + if host.os_type.lower() in versiom_map["os_cpu_arch"] + ] + agent_version: str = os_cpu_arch_version_list[0] if os_cpu_arch_version_list else "stable" + setup_info_cache_key: str = ( + f"agent_name:{self.agent_name}:type:{constants.AgentVersionType.BY_SYSTEM_ARCH.value}:" + f"os:{host.os_type.lower()}:version:{agent_version}" + ) + else: + # 按主机维度 + agent_version: str = self.bk_host_id_version_map[host.bk_host_id] + + target_version_cache_key: str = f"agent_desc_id:{self.agent_desc.id}:agent_version:{agent_version}" + target_version: str = self._target_version_cache.get(target_version_cache_key) + if target_version is None: + target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( + target_id=self.agent_desc.id, + target_version=agent_version, + ) + self._target_version_cache[target_version_cache_key] = target_version - return base.AgentSetupInfo( + if self.config["choice_version_type"] != constants.AgentVersionType.BY_HOST.value: + agent_setup_info: typing.Optional[base.AgentSetupInfo] = self._setup_info_cache.get(setup_info_cache_key) + if agent_setup_info: + return agent_setup_info + + agent_setup_info: base.AgentSetupInfo = base.AgentSetupInfo( is_legacy=self.is_legacy, agent_tools_relative_dir=("agent_tools/agent2", "")[self.is_legacy], name=self.config.get("name"), version=target_version, ) + if self.config["choice_version_type"] != constants.AgentVersionType.BY_HOST.value: + self._setup_info_cache[setup_info_cache_key] = agent_setup_info + return agent_setup_info @staticmethod def validated_data(data, serializer) -> OrderedDict: @@ -204,3 +258,15 @@ def get_os_key(os_type: str, cpu_arch: str) -> str: os_type = os_type or constants.OsType.LINUX cpu_arch = cpu_arch or constants.CpuType.x86_64 return f"{os_type.lower()}-{cpu_arch}" + + @property + def agent_desc(self) -> models.GsePackageDesc: + if hasattr(self, "_agent_desc") and self._agent_desc: + return self._agent_desc + try: + agent_desc = models.GsePackageDesc.objects.get(project=self.agent_name) + except models.GsePackageDesc.DoesNotExist: + raise errors.AgentPackageValidationError(msg="GsePackageDesc [{name}] 不存在".format(name=self.agent_name)) + + setattr(self, "_agent_desc", agent_desc) + return self._agent_desc diff --git a/apps/backend/subscription/views.py b/apps/backend/subscription/views.py index 2b8ac6d74..47a18f6fb 100644 --- a/apps/backend/subscription/views.py +++ b/apps/backend/subscription/views.py @@ -14,7 +14,7 @@ import operator from dataclasses import asdict from functools import cmp_to_key, reduce -from typing import Any, Dict, List, Set +from typing import Dict, List, Set from django.core.cache import caches from django.db import transaction @@ -609,14 +609,14 @@ def fetch_commands(self, request): ap_id_obj_map: Dict[int, models.AccessPoint] = models.AccessPoint.ap_id_obj_map() host_ap: models.AccessPoint = ap_id_obj_map[host_ap_id] - base_agent_setup_info_dict: Dict[str, Any] = asdict( - AgentStepAdapter(subscription_step=sub_step_obj, gse_version=host_ap.gse_version).setup_info + agent_setup_adapter: AgentStepAdapter = AgentStepAdapter( + subscription_step=sub_step_obj, gse_version=host_ap.gse_version ) agent_setup_extra_info_dict = sub_inst.instance_info["host"].get("agent_setup_extra_info") or {} installation_tool = gen_commands( agent_setup_info=AgentSetupInfo( **{ - **base_agent_setup_info_dict, + **asdict(agent_setup_adapter.get_host_setup_info(host)), "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False), } ), diff --git a/apps/backend/sync_task/constants.py b/apps/backend/sync_task/constants.py index f9be617d6..c6090585b 100644 --- a/apps/backend/sync_task/constants.py +++ b/apps/backend/sync_task/constants.py @@ -18,22 +18,29 @@ class SyncTaskType(EnhanceEnum): """同步任务类型""" SYNC_CMDB_HOST = "sync_cmdb_host" + REGISTER_GSE_PACKAGE = "register_gse_package" @classmethod def _get_member__alias_map(cls): - return {cls.SYNC_CMDB_HOST: _("同步 CMDB 主机数据")} + return { + cls.SYNC_CMDB_HOST: _("同步 CMDB 主机数据"), + cls.REGISTER_GSE_PACKAGE: _("注册gse package任务"), + } @classmethod def get_member__cache_key_map(cls): """获取缓存键名""" cache_key_map = { - cls.SYNC_CMDB_HOST: f"{settings.APP_CODE}:backend:sync_task:sync_cmdb_host:" + "biz:{bk_biz_id}" + cls.SYNC_CMDB_HOST: f"{settings.APP_CODE}:backend:sync_task:sync_cmdb_host:" + "biz:{bk_biz_id}", + cls.REGISTER_GSE_PACKAGE: f"{settings.APP_CODE}:backend:sync_task:register_gse_package:" + + "file_name={file_name}:tags={tags}", } return cache_key_map @classmethod def get_member__import_path_map(cls): import_path_map = { - cls.SYNC_CMDB_HOST: "apps.node_man.periodic_tasks.sync_cmdb_host.sync_cmdb_host_task" + cls.SYNC_CMDB_HOST: "apps.node_man.periodic_tasks.sync_cmdb_host.sync_cmdb_host_task", + cls.REGISTER_GSE_PACKAGE: "apps.node_man.periodic_tasks.register_gse_package.register_gse_package_task", } return import_path_map diff --git a/apps/backend/sync_task/handler.py b/apps/backend/sync_task/handler.py index b527f4a29..415dac207 100644 --- a/apps/backend/sync_task/handler.py +++ b/apps/backend/sync_task/handler.py @@ -15,6 +15,7 @@ class AsyncTaskHandler: """写入同步任务的公共逻辑，提供给多方调用""" + @staticmethod def sync_cmdb_host(bk_biz_id=None): # 获取缓存中该业务主机是否正在同步 @@ -36,3 +37,21 @@ def sync_cmdb_host(bk_biz_id=None): REDIS_INST.set(task_key_tpl.format(bk_biz_id=bk_biz_id if bk_biz_id else "all"), task_id, 10) return task_id + + @staticmethod + def register_gse_package(*args, **kwargs): + task_key_tpl_map = constants.SyncTaskType.get_member__cache_key_map() + task_key_tpl = task_key_tpl_map[constants.SyncTaskType.REGISTER_GSE_PACKAGE] + task_key = task_key_tpl.format(*args, **kwargs) + + task_id = REDIS_INST.get(task_key) + if task_id: + return task_id + + async_task = AsyncTaskManager() + async_task.as_task(constants.SyncTaskType.REGISTER_GSE_PACKAGE) + task_id = async_task.delay(*args, **kwargs) + + REDIS_INST.set(task_key, task_id, 10) + + return task_id diff --git a/apps/backend/tests/agent/artifact_builder/test_agent.py b/apps/backend/tests/agent/artifact_builder/test_agent.py index b6d2bbf51..5e108eaf9 100644 --- a/apps/backend/tests/agent/artifact_builder/test_agent.py +++ b/apps/backend/tests/agent/artifact_builder/test_agent.py @@ -16,8 +16,9 @@ from apps.backend.subscription.steps.agent_adapter.handlers import GseConfigHandler from apps.backend.tests.agent import template_env, utils -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.constants import TargetType from apps.core.tag.models import Tag +from apps.core.tag.targets.agent import AgentTargetHelper from apps.mock_data import utils as mock_data_utils from apps.node_man import constants, models @@ -49,10 +50,11 @@ def pkg_checker(self, version_str: str): ) self.assertTrue(os.path.exists(package_path)) - def tag_checker(self, target_id: int): + def tag_checker(self): + agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map() agent_target_version = Tag.objects.get( - target_id=target_id, + target_id=agent_name_target_id_map[self.NAME], name=self.OVERWRITE_VERSION, target_type=TargetType.AGENT.value, ).target_version @@ -106,12 +108,24 @@ def template_and_env_checker(self, version_str): self.assertTrue(models.GseConfigTemplate.objects.filter(**filter_kwargs).exists()) + def gse_package_and_desc_records_checker(self, version_str): + for package_os, cpu_arch in self.OS_CPU_CHOICES: + filter_kwargs: dict = { + "project": self.NAME, + "os": package_os, + "cpu_arch": cpu_arch, + "version": version_str, + } + self.assertTrue(models.GsePackages.objects.filter(**filter_kwargs).exists()) + self.assertTrue(models.GsePackageDesc.objects.filter(**{"project": filter_kwargs.pop("project")}).exists()) + def test_make(self): """测试安装包制作""" with self.ARTIFACT_BUILDER_CLASS(initial_artifact_path=self.ARCHIVE_PATH) as builder: builder.make() self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) def test_make__overwrite_version(self): """测试版本号覆盖""" @@ -129,7 +143,8 @@ def test_make__overwrite_version(self): self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) self.pkg_checker(version_str=self.OVERWRITE_VERSION) - self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.tag_checker() + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoTestCase(FileSystemTestCase): diff --git a/apps/backend/tests/agent/artifact_builder/test_manage_commands.py b/apps/backend/tests/agent/artifact_builder/test_manage_commands.py index ed2d7bb48..ee6f57429 100644 --- a/apps/backend/tests/agent/artifact_builder/test_manage_commands.py +++ b/apps/backend/tests/agent/artifact_builder/test_manage_commands.py @@ -17,7 +17,6 @@ from django.core.management import call_command from apps.backend.tests.agent import utils -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP from apps.mock_data import utils as mock_data_utils from apps.node_man import models @@ -44,6 +43,7 @@ def test_make(self): self.assertTrue(models.UploadPackage.objects.all().exists()) self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) def test_make__overwrite_version(self): """测试版本号覆盖""" @@ -51,7 +51,8 @@ def test_make__overwrite_version(self): self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) self.pkg_checker(version_str=self.OVERWRITE_VERSION) - self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.tag_checker() + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoImportAgentTestCase(FileSystemImportAgentTestCase): diff --git a/apps/backend/tests/agent/artifact_builder/test_proxy.py b/apps/backend/tests/agent/artifact_builder/test_proxy.py index 15a5468ad..d182aabe5 100644 --- a/apps/backend/tests/agent/artifact_builder/test_proxy.py +++ b/apps/backend/tests/agent/artifact_builder/test_proxy.py @@ -34,6 +34,7 @@ def test_make(self): builder.make() self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoTestCase(FileSystemTestCase): diff --git a/apps/backend/tests/components/collections/agent_new/test_install.py b/apps/backend/tests/components/collections/agent_new/test_install.py index bfd7b131b..1ed5f26fc 100644 --- a/apps/backend/tests/components/collections/agent_new/test_install.py +++ b/apps/backend/tests/components/collections/agent_new/test_install.py @@ -184,6 +184,7 @@ def start_patch(self): fs.write("哈哈哈113343ddfd") def setUp(self) -> None: + self.obj_factory.init_gse_package_desc() self.update_callback_url() self.init_mock_clients() self.init_hosts() @@ -412,7 +413,9 @@ async def connect(self): class InstallAgent2WindowsTestCase(InstallWindowsTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] - sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.config.update( + {"name": "gse_agent", "version": "2.0.0", "version_map_list": [], "choice_version_type": "unified"} + ) sub_step_obj.save(update_fields=["config"]) def structure_common_inputs(self): @@ -428,7 +431,7 @@ def test_batch_solution(self): gse_version=GseVersion.V2.value, ) installation_tool = gen_commands( - agent_step_adapter.setup_info, + agent_step_adapter.get_host_setup_info(host), host, mock_data_utils.JOB_TASK_PIPELINE_ID, is_uninstall=False, @@ -885,7 +888,9 @@ class LinuxAgent2InstallTestCase(InstallBaseTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] - sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.config.update( + {"name": "gse_agent", "version": "2.0.0", "version_map_list": [], "choice_version_type": "unified"} + ) sub_step_obj.save() def structure_common_inputs(self): @@ -901,7 +906,7 @@ def test_shell_solution(self): gse_version=GseVersion.V2.value, ) installation_tool = gen_commands( - agent_step_adapter.setup_info, + agent_step_adapter.get_host_setup_info(host), host, mock_data_utils.JOB_TASK_PIPELINE_ID, is_uninstall=False, diff --git a/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py b/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py index ed1922fc0..b29b4eba9 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py @@ -127,6 +127,7 @@ def component_cls(self): @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) sub_step_obj.save() diff --git a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py index 69af15a7e..9f8a56276 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py @@ -53,6 +53,7 @@ def get_default_case_name(cls) -> str: @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() models.Host.objects.filter(bk_host_id__in=cls.obj_factory.bk_host_ids).update(bk_agent_id=get_random_string()) sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) diff --git a/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py b/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py index f2cba8d4b..5233a94cd 100644 --- a/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py +++ b/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py @@ -85,6 +85,7 @@ def structure_common_inputs(self): @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) sub_step_obj.save() diff --git a/apps/backend/tests/components/collections/agent_new/utils.py b/apps/backend/tests/components/collections/agent_new/utils.py index cc6c3c9f2..ded153510 100644 --- a/apps/backend/tests/components/collections/agent_new/utils.py +++ b/apps/backend/tests/components/collections/agent_new/utils.py @@ -295,7 +295,11 @@ def structure_sub_step_data_list(self) -> List[Dict[str, Any]]: sub_step_data.update( { "subscription_id": self.sub_obj.id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": { + "job_type": constants.JobType.INSTALL_AGENT, + "version_map_list": [], + "choice_version_type": "unified", + }, } ) return [sub_step_data] @@ -371,6 +375,15 @@ def init_host_related_data_in_db(self): self.bulk_create_model(model=models.IdentityData, create_data_list=identity_data_list) self.identity_data_objs = models.IdentityData.objects.filter(bk_host_id__in=self.bk_host_ids) + @classmethod + def init_gse_package_desc(cls): + models.GsePackageDesc.objects.update_or_create( + defaults={"description": ""}, project="gse_agent", category="official" + ) + models.GsePackageDesc.objects.update_or_create( + defaults={"description": ""}, project="gse_proxy", category="official" + ) + def init_db(self): """ 初始化DB测试数据 diff --git a/apps/backend/tests/subscription/agent_adapter/test_adapter.py b/apps/backend/tests/subscription/agent_adapter/test_adapter.py index 62f4b6013..56b5aec76 100644 --- a/apps/backend/tests/subscription/agent_adapter/test_adapter.py +++ b/apps/backend/tests/subscription/agent_adapter/test_adapter.py @@ -13,6 +13,9 @@ from apps.backend.subscription.steps.agent_adapter.adapter import AgentStepAdapter from apps.backend.tests.agent.utils import VERSION, AgentBaseTestCase, ProxyBaseTestCase +from apps.backend.tests.components.collections.agent_new.utils import ( + AgentTestObjFactory, +) from apps.mock_data import common_unit from apps.node_man import constants, models from apps.utils import basic @@ -31,6 +34,7 @@ def setUpTestData(cls): super().setUpTestData() host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) cls.host = models.Host.objects.create(**host_model_data) + AgentTestObjFactory.init_gse_package_desc() # 创建订阅相关数据 sub_inst_data = basic.remove_keys_from_dict( @@ -45,7 +49,7 @@ def setUpTestData(cls): **sub_step_data, **{ "subscription_id": cls.sub_inst_record_obj.subscription_id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, }, } ) @@ -85,6 +89,8 @@ def setUpTestData(cls): "job_type": constants.JobType.INSTALL_AGENT, "name": constants.GsePackageCode.PROXY.value, "version": VERSION, + "version_map_list": [], + "choice_version_type": "unified", } cls.sub_step_obj.save() cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=GseVersion.V2.value) @@ -93,12 +99,13 @@ def test_get_config(self): self.clear_agent_data() self.host.node_type = "PROXY" self.host.bk_cloud_id = 1 + agent_setup_info = self.agent_step_adapter.get_host_setup_info(self.host) for config_name in constants.GsePackageTemplate.PROXY.value: self.get_config(config_name) self.assertEqual( self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ) .get_matching_config_tmpl(self.host.os_type, self.host.cpu_arch, config_name) .agent_name_from, @@ -108,8 +115,8 @@ def test_get_config(self): self.get_config(config_name) self.assertEqual( self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ) .get_matching_config_tmpl(self.host.os_type, self.host.cpu_arch, config_name) .agent_name_from, @@ -128,16 +135,17 @@ def clear_agent_data(cls): pass def test_get_env(self): + agent_setup_info = self.agent_step_adapter.get_host_setup_info(self.host) agent_env = self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ).get_matching_template_env(self.host.os_type, self.host.cpu_arch, constants.GsePackageCode.AGENT.value) self.assertEqual(agent_env["BK_GSE_HOME_DIR"], "/usr/local/gse/agent") proxy_env = self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ).get_matching_template_env(self.host.os_type, self.host.cpu_arch, constants.GsePackageCode.PROXY.value) self.assertEqual(proxy_env["BK_GSE_HOME_DIR"], "/usr/local/gse/proxy") @@ -155,6 +163,8 @@ def setUpTestData(cls): "job_type": constants.JobType.INSTALL_AGENT, "name": constants.GsePackageCode.AGENT.value, "version": VERSION, + "version_map_list": [], + "choice_version_type": "unified", } cls.sub_step_obj.save() cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=GseVersion.V2.value) diff --git a/apps/backend/tests/view/test_get_gse_config.py b/apps/backend/tests/view/test_get_gse_config.py index 37e29d55a..cc9e98090 100644 --- a/apps/backend/tests/view/test_get_gse_config.py +++ b/apps/backend/tests/view/test_get_gse_config.py @@ -38,7 +38,7 @@ def setUpTestData(cls): host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) cls.host = models.Host.objects.create(**host_model_data) models.AccessPoint.objects.all().update(gse_version=cls.GSE_VERSION) - + models.GsePackageDesc.objects.create(project="gse_agent", description="", category="official") # 创建订阅相关数据 sub_inst_data = basic.remove_keys_from_dict( origin_data=common_unit.subscription.SUB_INST_RECORD_MODEL_DATA, keys=["id"] @@ -52,7 +52,7 @@ def setUpTestData(cls): **sub_step_data, **{ "subscription_id": cls.sub_inst_record_obj.subscription_id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, }, } ) @@ -117,13 +117,19 @@ def setUp(self) -> None: @classmethod def setUpTestData(cls): super().setUpTestData() - cls.sub_step_obj.config = {"job_type": constants.JobType.INSTALL_AGENT, "name": "gse_agent", "version": "2.0.0"} + cls.sub_step_obj.config = { + "job_type": constants.JobType.INSTALL_AGENT, + "name": "gse_agent", + "version": "2.0.0", + "version_map_list": [], + "choice_version_type": "unified", + } cls.sub_step_obj.save() cls.redis_agent_conf_key = REDIS_AGENT_CONF_KEY_TPL.format( file_name=cls.agent_step_adapter.get_main_config_filename(), sub_inst_id=cls.sub_inst_record_obj.id ) cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=cls.GSE_VERSION) - target_version = cls.agent_step_adapter.setup_info.version + target_version = cls.agent_step_adapter.get_host_setup_info(cls.host).version models.GseConfigEnv.objects.create( agent_name="gse_agent", version=target_version, diff --git a/apps/backend/urls.py b/apps/backend/urls.py index 997d9e80e..6085b2d10 100644 --- a/apps/backend/urls.py +++ b/apps/backend/urls.py @@ -13,6 +13,7 @@ from rest_framework import routers as drf_routers from apps.backend import views +from apps.backend.agent.views import AgentViewSet from apps.backend.healthz.views import HealthzViewSet from apps.backend.plugin.views import PluginViewSet, export_download, upload_package from apps.backend.subscription.views import SubscriptionViewSet @@ -28,6 +29,7 @@ routers.register("subscription", SubscriptionViewSet, basename="subscription") routers.register("healthz", HealthzViewSet, basename="healthz") routers.register("sync_task", SyncTaskViewSet, basename="sync_task") + routers.register("agent", AgentViewSet, basename="agent") export_routers = drf_routers.DefaultRouter(trailing_slash=True) urlpatterns.extend( [ diff --git a/apps/core/ipchooser/tools/base.py b/apps/core/ipchooser/tools/base.py index fc39c6b81..a6b767ce7 100644 --- a/apps/core/ipchooser/tools/base.py +++ b/apps/core/ipchooser/tools/base.py @@ -175,6 +175,31 @@ def extract_bools(cls, values: typing.List[typing.Union[str, bool, int]]) -> typ pass return list(bool_set) + @classmethod + def _handle_topo_conditions(cls, bk_module_ids, bk_set_ids, topo_biz_id, topo_host_ids): + if bk_module_ids: + extra_kwargs = { + "filter_obj_id": constants.ObjectType.MODULE.value, + "filter_inst_ids": list(set(bk_module_ids)), + } + else: + extra_kwargs = { + "filter_obj_id": constants.ObjectType.SET.value, + "filter_inst_ids": list(set(bk_set_ids)), + } + + host_infos: typing.List[types.HostInfo] = resource.ResourceQueryHelper.fetch_biz_hosts( + bk_biz_id=topo_biz_id, fields=["bk_host_id"], **extra_kwargs + ) + host_ids: typing.Set[int] = {host_info["bk_host_id"] for host_info in host_infos} + + if topo_host_ids is None: + topo_host_ids = host_ids + else: + topo_host_ids = topo_host_ids | host_ids + + return topo_host_ids + @classmethod def multiple_cond_sql( cls, @@ -183,6 +208,7 @@ def multiple_cond_sql( is_proxy: bool = False, return_all_node_type: bool = False, extra_wheres: typing.List[str] = None, + need_biz_scope: bool = True, ) -> QuerySet: """ 用于生成多条件sql查询 @@ -191,6 +217,7 @@ def multiple_cond_sql( :param biz_scope: 业务范围限制 :param is_proxy: 是否为代理 :param extra_wheres: 额外的查询条件 + :param need_biz_scope: 是否需要业务范围限制 :return: 根据条件查询的所有结果 """ select: typing.Dict[str, str] = { @@ -210,14 +237,17 @@ def multiple_cond_sql( ] wheres: typing.List[str] = extra_wheres or [] + filter_q = Q() + if params.get("bk_host_id") is not None: + filter_q &= Q(bk_host_id__in=params.get("bk_host_id")) + final_biz_scope: typing.Set[int] = set(biz_scope) # 带有业务筛选条件，需要确保落在指定业务范围内 if params.get("bk_biz_id"): final_biz_scope = final_biz_scope & set(params["bk_biz_id"]) - filter_q: Q = Q(bk_biz_id__in=final_biz_scope) - if params.get("bk_host_id") is not None: - filter_q &= Q(bk_host_id__in=params.get("bk_host_id")) + if need_biz_scope: + filter_q &= Q(bk_biz_id__in=final_biz_scope) # 条件搜索 where_or = [] @@ -278,26 +308,12 @@ def multiple_cond_sql( topo_biz_scope.add(topo_biz_id) continue - if bk_module_ids: - extra_kwargs = { - "filter_obj_id": constants.ObjectType.MODULE.value, - "filter_inst_ids": list(set(bk_module_ids)), - } - else: - extra_kwargs = { - "filter_obj_id": constants.ObjectType.SET.value, - "filter_inst_ids": list(set(bk_set_ids)), - } - - host_infos: typing.List[types.HostInfo] = resource.ResourceQueryHelper.fetch_biz_hosts( - bk_biz_id=topo_biz_id, fields=["bk_host_id"], **extra_kwargs + topo_host_ids = cls._handle_topo_conditions( + bk_module_ids=bk_module_ids, + bk_set_ids=bk_set_ids, + topo_biz_id=topo_biz_id, + topo_host_ids=topo_host_ids, ) - host_ids: typing.Set[int] = {host_info["bk_host_id"] for host_info in host_infos} - - if topo_host_ids is None: - topo_host_ids = host_ids - else: - topo_host_ids = topo_host_ids | host_ids elif condition["key"] == "query" and isinstance(condition["value"], str): fuzzy_search_fields: typing.List[str] = ( @@ -346,17 +362,49 @@ def multiple_cond_sql( if topo_host_ids is not None: topo_query = topo_query | Q(bk_host_id__in=topo_host_ids) - host_queryset: QuerySet = ( - node_man_models.Host.objects.filter( + host_queryset = cls.get_filtered_host_queryset( + is_proxy=is_proxy, + return_all_node_type=return_all_node_type, + final_biz_scope=final_biz_scope, + wheres=wheres, + sql_params=sql_params, + select=select, + topo_query=topo_query, + is_enable_cloud_area_ip_filter=is_enable_cloud_area_ip_filter, + filter_q=filter_q, + need_biz_scope=need_biz_scope, + ) + + return host_queryset + + @classmethod + def get_filtered_host_queryset( + cls, + is_proxy, + return_all_node_type, + final_biz_scope, + wheres, + sql_params, + select, + topo_query, + is_enable_cloud_area_ip_filter, + filter_q, + need_biz_scope=False, + ): + if need_biz_scope: + host_queryset: QuerySet = node_man_models.Host.objects.filter( node_type__in=cls.fetch_match_node_types(is_proxy, return_all_node_type), bk_biz_id__in=final_biz_scope ) - .extra( - select=select, tables=[node_man_models.ProcessStatus._meta.db_table], where=wheres, params=sql_params + else: + host_queryset: QuerySet = node_man_models.Host.objects.filter( + node_type__in=cls.fetch_match_node_types(is_proxy, return_all_node_type), ) - .filter(topo_query) - ) - host_queryset = handle_filter_queryset_by_flag_value(is_enable_cloud_area_ip_filter, host_queryset, filter_q) + host_queryset = host_queryset.extra( + select=select, tables=[node_man_models.ProcessStatus._meta.db_table], where=wheres, params=sql_params + ).filter(topo_query) + + host_queryset = handle_filter_queryset_by_flag_value(is_enable_cloud_area_ip_filter, host_queryset, filter_q) return host_queryset @classmethod diff --git a/apps/core/tag/constants.py b/apps/core/tag/constants.py index 8b0c27614..cf922484a 100644 --- a/apps/core/tag/constants.py +++ b/apps/core/tag/constants.py @@ -13,7 +13,6 @@ from django.utils.translation import ugettext_lazy as _ -from apps.node_man.constants import GsePackageCode from apps.utils.enum import EnhanceEnum @@ -37,10 +36,3 @@ class TagChangeAction(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DELETE: _("删除标签"), cls.CREATE: _("新建标签"), cls.UPDATE: _("更新版本"), cls.OVERWRITE: _("同版本覆盖更新")} - - -# TODO: target_id 临时写死 -AGENT_NAME_TARGET_ID_MAP: Dict[str, int] = { - GsePackageCode.AGENT.value: 1, - GsePackageCode.PROXY.value: 2, -} diff --git a/apps/core/tag/exceptions.py b/apps/core/tag/exceptions.py index 26227c56f..e505c7545 100644 --- a/apps/core/tag/exceptions.py +++ b/apps/core/tag/exceptions.py @@ -52,3 +52,8 @@ class TagInvalidNameError(TagBaseException): MESSAGE_TPL = _("标签名非法：{err_msg}") MESSAGE = _("标签名非法") ERROR_CODE = 6 + + +class ValidationError(TagBaseException): + MESSAGE = _("参数验证失败") + ERROR_CODE = 7 diff --git a/apps/core/tag/migrations/0004_alter_tag_unique_together.py b/apps/core/tag/migrations/0004_alter_tag_unique_together.py new file mode 100644 index 000000000..8c0178249 --- /dev/null +++ b/apps/core/tag/migrations/0004_alter_tag_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2024-02-29 09:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0003_auto_20231029_1336'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='tag', + unique_together={('target_type', 'target_id', 'name', 'target_version')}, + ), + ] diff --git a/apps/core/tag/migrations/0005_create_initial_tags.py b/apps/core/tag/migrations/0005_create_initial_tags.py new file mode 100644 index 000000000..2abdfaf5b --- /dev/null +++ b/apps/core/tag/migrations/0005_create_initial_tags.py @@ -0,0 +1,56 @@ +# Generated by Django 3.2.4 on 2024-02-29 09:40 + +from django.db import IntegrityError, migrations + +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.node_man.constants import CategoryType, GsePackageCode +from apps.node_man.models import GsePackageDesc + + +def create_initial_tags(apps, schema_editor): + try: + agent_target_id = GsePackageDesc.objects.get( + project=GsePackageCode.AGENT.value, category=CategoryType.official + ).id + + proxy_target_id = GsePackageDesc.objects.get( + project=GsePackageCode.PROXY.value, category=CategoryType.official + ).id + except GsePackageDesc.DoesNotExist: + agent_target_id = GsePackageDesc.objects.create( + project=GsePackageCode.AGENT.value, category=CategoryType.official + ).id + + proxy_target_id = GsePackageDesc.objects.create( + project=GsePackageCode.PROXY.value, category=CategoryType.official + ).id + + tag_name__tag_description__map = { + "stable": "稳定版本", + "latest": "最新版本", + "test": "测试版本", + } + for target_id in [proxy_target_id, agent_target_id]: + for tag_name, tag_description in tag_name__tag_description__map.items(): + try: + # 添加Tag记录 + Tag.objects.create( + name=tag_name, + description=tag_description, + target_id=target_id, + target_type=TargetType.AGENT.value, + ) + except IntegrityError: + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("tag", "0004_alter_tag_unique_together"), + ] + + operations = [ + migrations.RunPython(create_initial_tags), + ] diff --git a/apps/core/tag/migrations/0006_create_initial_gsepackages.py b/apps/core/tag/migrations/0006_create_initial_gsepackages.py new file mode 100644 index 000000000..8242b02a9 --- /dev/null +++ b/apps/core/tag/migrations/0006_create_initial_gsepackages.py @@ -0,0 +1,42 @@ +# Generated by Django 3.2.4 on 2024-02-29 09:40 +from typing import List + +from django.db import IntegrityError, migrations + +from apps.node_man.models import GseConfigEnv, GsePackages + + +def create_initial_gse_packages(apps, schema_editor): + extra_env_infos = GseConfigEnv.objects.all() + + gse_package_list: List[GsePackages] = [] + for extra_env_obj in extra_env_infos: + gse_package_list.append( + GsePackages( + pkg_name=str(extra_env_obj), + version=extra_env_obj.version, + project=extra_env_obj.agent_name, + pkg_size=0, + pkg_path="", + md5="", + location="", + os=extra_env_obj.os, + cpu_arch=extra_env_obj.cpu_arch, + ) + ) + + try: + GsePackages.objects.bulk_create(gse_package_list) + except IntegrityError: + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("tag", "0005_create_initial_tags"), + ] + + operations = [ + migrations.RunPython(create_initial_gse_packages), + ] diff --git a/apps/core/tag/models.py b/apps/core/tag/models.py index 1e61e7b42..41e530119 100644 --- a/apps/core/tag/models.py +++ b/apps/core/tag/models.py @@ -29,7 +29,7 @@ class Meta: verbose_name = _("标签") verbose_name_plural = _("标签") # 唯一性校验 - unique_together = (("target_type", "target_id", "name"),) + unique_together = (("target_type", "target_id", "name", "target_version"),) index_together = [ ["target_id", "target_type"], ] diff --git a/apps/core/tag/targets/agent.py b/apps/core/tag/targets/agent.py index 400acd7ac..64f75a36f 100644 --- a/apps/core/tag/targets/agent.py +++ b/apps/core/tag/targets/agent.py @@ -11,8 +11,10 @@ import logging +import typing from apps.core.tag.models import Tag +from apps.node_man.models import GsePackageDesc from .. import constants from . import base @@ -26,12 +28,42 @@ class AgentTargetHelper(base.BaseTargetHelper): TARGET_TYPE = constants.TargetType.AGENT.value def _publish_tag_version(self): - Tag.objects.update_or_create( - defaults={"target_version": self.target_version}, + if self.tag_name in ["stable", "latest", "test"]: + # 内置标签相互覆盖 + Tag.objects.update_or_create( + defaults={"target_version": self.target_version}, + name=self.tag_name, + target_id=self.target_id, + target_type=self.TARGET_TYPE, + ) + return + + tag: typing.Optional[Tag] = Tag.objects.filter( name=self.tag_name, target_id=self.target_id, target_type=self.TARGET_TYPE, - ) + ).first() + + if not tag: + return + + if not tag.target_version: + # 刚创建的未指定版本的标签 + tag.target_version = self.target_version + tag.save() + else: + Tag.objects.update_or_create( + name=self.tag_name, + target_id=self.target_id, + target_type=self.TARGET_TYPE, + target_version=self.target_version, + description=tag.description, + ) def _delete_tag_version(self): - return super()._delete_tag_version() + pass + + @classmethod + def get_agent_name_target_id_map(cls) -> typing.Dict[str, int]: + package_descs = GsePackageDesc.objects.values("project", "id") + return {package_desc["project"]: package_desc["id"] for package_desc in package_descs} diff --git a/apps/core/tag/targets/base.py b/apps/core/tag/targets/base.py index 5479f77ca..06637058e 100644 --- a/apps/core/tag/targets/base.py +++ b/apps/core/tag/targets/base.py @@ -37,10 +37,10 @@ def handle_create_or_update(cls, instance: typing.Type["BaseTargetHelper"]): """ # 发布标签版本后，创建或更新相应的标签 tag, created = models.Tag.objects.update_or_create( - name=instance.tag_name, + target_version=instance.target_version, target_id=instance.target_id, target_type=instance.TARGET_TYPE, - defaults=dict(target_version=instance.target_version), + name=instance.tag_name, ) logger.info( f"[publish_tag_version] update_or_create tag -> {tag.name}({tag.id}) success, " diff --git a/apps/core/tag/tests/test_views.py b/apps/core/tag/tests/test_views.py index 5b591cd03..3bd9657b0 100644 --- a/apps/core/tag/tests/test_views.py +++ b/apps/core/tag/tests/test_views.py @@ -17,7 +17,7 @@ class TagPluginTestCase(utils.PluginBaseTestCase): - DEFAULT_TAG_NAME: str = "stable" + DEFAULT_TAG_NAME: str = "stable2" def create_tag(self, tag_name: str, target_version: str) -> typing.Dict[str, typing.Any]: upload_result = self.upload_plugin() diff --git a/apps/generic.py b/apps/generic.py index d8e3181d2..a612d3b3e 100644 --- a/apps/generic.py +++ b/apps/generic.py @@ -173,6 +173,10 @@ def get_serializer_class(self, *args, **kwargs): return type(self.serializer_class.__name__, (self.serializer_class,), {"Meta": self.serializer_meta}) +class ApiMixinModelViewSet(ApiMixin, _ModelViewSet): + pagination_class = DataPageNumberPagination + + def custom_exception_handler(exc, context): """ 自定义错误处理方式 diff --git a/apps/mock_data/common_unit/subscription.py b/apps/mock_data/common_unit/subscription.py index 39d003403..3b2725f54 100644 --- a/apps/mock_data/common_unit/subscription.py +++ b/apps/mock_data/common_unit/subscription.py @@ -92,7 +92,7 @@ "subscription_id": DEFAULT_SUBSCRIPTION_ID, "step_id": constants.SubStepType.AGENT.lower(), "type": constants.SubStepType.AGENT, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, "params": {"context": {}, "blueking_language": "zh-hans"}, } diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index bee4b0e2b..9ad98f95f 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -193,6 +193,7 @@ def get_optional_items(cls) -> List[str]: "UNINSTALL_AGENT", "REMOVE_AGENT", "UPGRADE_AGENT", + "DOWNGRADE_AGENT", "IMPORT_AGENT", "RESTART_AGENT", "RELOAD_AGENT", @@ -242,7 +243,7 @@ def get_optional_items(cls) -> List[str]: JobType.REINSTALL_PROXY: _("重装 Proxy"), JobType.REINSTALL_AGENT: _("重装 Agent"), JobType.UPGRADE_PROXY: _("升级 Proxy"), - JobType.UPGRADE_AGENT: _("升级 Agent"), + JobType.UPGRADE_AGENT: _("升级/回退 Agent"), JobType.REMOVE_AGENT: _("移除 Agent"), JobType.UNINSTALL_AGENT: _("卸载 Agent"), JobType.UNINSTALL_PROXY: _("卸载 Proxy"), @@ -274,6 +275,7 @@ def get_optional_items(cls) -> List[str]: "REMOVE", "REPLACE", "UPGRADE", + "DOWNGRADE", "UPDATE", "IMPORT", "UPDATE", @@ -302,6 +304,7 @@ def get_optional_items(cls) -> List[str]: OpType.RESTART: _("重启"), OpType.REPLACE: _("替换"), OpType.UPGRADE: _("升级"), + OpType.DOWNGRADE: _("回退"), OpType.REINSTALL: _("重装"), OpType.UPDATE: _("更新"), OpType.REMOVE: _("移除"), @@ -372,6 +375,11 @@ def get_optional_items(cls) -> List[str]: IAM_ACTION_CHOICES = tuple_choices(IAM_ACTION_TUPLE) IamActionType = choices_to_namedtuple(IAM_ACTION_CHOICES) +GSE_PACKAGE_ENABLE_ALIAS_MAP = { + True: _("启用"), + False: _("停用"), +} + class SubscriptionType: POLICY = "policy" @@ -958,6 +966,10 @@ class GsePackageCode(EnhanceEnum): def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.PROXY: _("2.0 Proxy Agent 安装包代号"), cls.AGENT: _("2.0 Agent 安装包代号")} + @classmethod + def values(cls): + return [member.value for member in cls] + class GsePackageEnv(EnhanceEnum): """安装包Env文件名称""" @@ -980,6 +992,13 @@ class GsePackageTemplatePattern(EnhanceEnum): AGENT = re.compile("|".join(GsePackageTemplate.AGENT.value)) +class GsePackageCacheKey(EnhanceEnum): + """GsePackageHandler的缓存key""" + + TAGS_PREFIX = "tags_" + DESCRIPTION_PREFIX = "description_" + + class GsePackageDir(EnhanceEnum): """安装包打包根路径""" @@ -1209,3 +1228,24 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: @classmethod def cpu_type__os_bit_map(cls): return {CpuType.x86: cls.BIT32.value, CpuType.x86_64: cls.BIT64.value, CpuType.aarch64: cls.ARM.value} + + +class AgentVersionType(EnhanceEnum): + UNIFIED = "unified" + BY_HOST = "by_host" + BY_SYSTEM_ARCH = "by_system_arch" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.UNIFIED: _("统一的版本"), + cls.BY_HOST: _("按主机的"), + cls.BY_SYSTEM_ARCH: _("按系统架构"), + } + + +BUILT_IN_TAG_DESCRIPTIONS: List[str] = ["稳定版本", "最新版本", "测试版本"] +BUILT_IN_TAG_NAMES: List[str] = ["stable", "latest", "test"] +TAG_NAME__TAG_DESCRIPTION = dict(zip(BUILT_IN_TAG_NAMES, BUILT_IN_TAG_DESCRIPTIONS)) +TAG_DESCRIPTION__TAG_NAME = dict(zip(BUILT_IN_TAG_DESCRIPTIONS, BUILT_IN_TAG_NAMES)) +STABLE_DESCRIPTION = "稳定版本" diff --git a/apps/node_man/exceptions.py b/apps/node_man/exceptions.py index a1c7cb2a4..c96e1ae88 100644 --- a/apps/node_man/exceptions.py +++ b/apps/node_man/exceptions.py @@ -220,3 +220,41 @@ class YunTiPolicyConfigNotExistsError(NodeManBaseException): MESSAGE = _("云梯策略配置不存在") MESSAGE_TPL = _("云梯策略配置不存在") ERROR_CODE = 43 + + +class FileDoesNotExistError(NodeManBaseException): + MESSAGE = _("文件不存在") + ERROR_CODE = 44 + + +class PluginParseError(NodeManBaseException): + MESSAGE = _("插件解析错误") + ERROR_CODE = 45 + + +class GsePackageInValidError(NodeManBaseException): + MESSAGE = _("agent包无效") + ERROR_CODE = 46 + + +class GsePackageUploadError(NodeManBaseException): + MESSAGE = _("gse包上传失败") + MESSAGE_TPL = _("插件上传失败: agent_name -> {agent_name}, error -> {error}") + ERROR_CODE = 47 + + +class PermissionDeniedError(NodeManBaseException): + MESSAGE = _("权限不足") + ERROR_CODE = 48 + + +class ModelInstanceNotFoundError(NodeManBaseException): + MESSAGE = _("模型对象不存在") + MESSAGE_TPL = _("模型对象 -> [{model_name}] 不存在") + ERROR_CODE = 49 + + +class DuplicateEntryError(NodeManBaseException): + MESSAGE = _("存在唯一索引约束") + MESSAGE_TPL = _("存在唯一索引约束 -> [{entry_info}]") + ERROR_CODE = 50 diff --git a/apps/node_man/handlers/gse_package.py b/apps/node_man/handlers/gse_package.py new file mode 100644 index 000000000..8a4cd51f1 --- /dev/null +++ b/apps/node_man/handlers/gse_package.py @@ -0,0 +1,260 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from collections import defaultdict +from typing import Any, Dict, List + +from django.core.signals import request_finished +from django.db.models import Q, QuerySet +from django.dispatch import receiver +from django.utils.translation import ugettext_lazy as _ + +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.node_man import constants +from apps.node_man.models import GsePackageDesc, GsePackages +from apps.node_man.tools.gse_package import GsePackageTools + + +class GsePackageHandler: + PROJECT_VERSION__TAGS_MAP = "project_version__tags_map" + PROJECT__DESCRIPTION_MAP = "project__description_map" + + def __init__(self): + self._init_caches() + + def _init_caches(self): + """初始化缓存""" + self.cache = { + self.PROJECT_VERSION__TAGS_MAP: defaultdict(list), + self.PROJECT__DESCRIPTION_MAP: {}, + } + + self.cache_counter = { + self.PROJECT_VERSION__TAGS_MAP: 0, + self.PROJECT__DESCRIPTION_MAP: 0, + } + + def clear_caches(self): + """清理缓存""" + self._init_caches() + + def _init_project_version__tags_map(self): + """初始化项目版本标签映射""" + for project in constants.GsePackageCode.values(): + tags: QuerySet = self.get_tag_objs(project).values("name", "description", "target_version") + + for tag in tags: + tag["description"] = _(tag["description"]) + cache_key = self.get_tags_cache_key(project, tag.pop("target_version")) + self.cache[self.PROJECT_VERSION__TAGS_MAP][cache_key].append(tag) + + self.cache_counter[self.PROJECT_VERSION__TAGS_MAP] += 1 + + def _init_project__description_map(self): + """初始化项目描述映射""" + for project in constants.GsePackageCode.values(): + description = GsePackageDesc.objects.filter(project=project).first().description + cache_key = self.get_description_cache_key(project) + self.cache[self.PROJECT__DESCRIPTION_MAP][cache_key] = description + + self.cache_counter[self.PROJECT__DESCRIPTION_MAP] += 1 + + @classmethod + def get_tags_cache_key(cls, project: str, version: str) -> str: + """获取标签缓存key""" + return f"{constants.GsePackageCacheKey.TAGS_PREFIX.value}:{project}:{version}" + + @classmethod + def get_description_cache_key(cls, project: str) -> str: + """获取描述缓存key""" + return f"{constants.GsePackageCacheKey.DESCRIPTION_PREFIX.value}:{project}" + + @classmethod + def get_tag_objs(cls, project: str, version: str = None) -> QuerySet: + """ + 获取标签对象 + :params project: gse_agent或gse_proxy + :params version: agent包版本 + """ + target_ids = GsePackageDesc.objects.filter(project=project).values_list("id", flat=True) + + if not version: + return Tag.objects.filter(target_id__in=target_ids) + return Tag.objects.filter(target_id__in=target_ids, target_version=version) + + def get_tags( + self, + project: str, + version: str, + enable_tag_separation: bool = False, + ) -> List[Dict[str, Any]]: + """ + 获取标签列表 + :params project: gse_agent或gse_proxy + :params version: agent包版本 + :params enable_tag_separation: 是否需要将标签分割为内置和自定义 + """ + if not self.cache_counter[self.PROJECT_VERSION__TAGS_MAP]: + self._init_project_version__tags_map() + + cache_key = self.get_tags_cache_key(project, version) + tags = self.cache[self.PROJECT_VERSION__TAGS_MAP].get(cache_key, []) + + return self.handle_tags(tags, enable_tag_separation=enable_tag_separation) + + def get_description(self, project: str) -> str: + """ + 获取包描述信息 + :params project: gse_agent或gse_proxy + """ + if not self.cache_counter[self.PROJECT__DESCRIPTION_MAP]: + self._init_project__description_map() + + cache_key: str = self.get_description_cache_key(project) + return self.cache[self.PROJECT__DESCRIPTION_MAP].get(cache_key, "") + + def handle_tags( + self, + tags: List[Dict[str, str]], + enable_tag_separation: bool = False, + tag_description=None, + ) -> List[Dict[str, Any]]: + """ + 处理标签列表 + tags: 原始标签列表 + enable_tag_separation: 是否需要将标签分割为内置和自定义 + tag_description: 模糊匹配标签描述 + """ + if tag_description: + tags = [tag for tag in tags if tag_description in tag["description"]] + + if not enable_tag_separation: + return tags + + built_in_tags, custom_tags = self.split_tags_into_builtin_and_custom(tags) + + parent_tags: List[Dict[str, Any]] = [ + {"name": "builtin", "description": _("内置标签"), "children": built_in_tags}, + {"name": "custom", "description": _("自定义标签"), "children": custom_tags}, + ] + + return [parent_tag for parent_tag in parent_tags if parent_tag.get("children")] + + @classmethod + def filter_tags(cls, queryset: QuerySet, project: str, tag_names: List[str] = None) -> QuerySet: + """筛选标签queryset""" + project__id_map: Dict[str, int] = dict(GsePackageDesc.objects.values_list("project", "id")) + combined_tag_names_conditions: Q = Q() + + for tag_name in tag_names or []: + combined_tag_names_conditions |= Q(name__contains=tag_name) + + filter_conditions: Q = Q(target_id=project__id_map.get(project)) & combined_tag_names_conditions + + target_versions: QuerySet = Tag.objects.filter(filter_conditions).values_list("target_version", flat=True) + + return queryset.filter(version__in=target_versions) + + @classmethod + def split_tags_into_builtin_and_custom( + cls, tags: List[Dict[str, Any]] + ) -> (List[Dict[str, Any]], List[Dict[str, Any]]): + """将标签拆分为内置的和自定义的""" + built_in_tags, custom_tags = [], [] + for tag in tags: + if tag["name"] in constants.BUILT_IN_TAG_NAMES: + built_in_tags.append(tag) + else: + custom_tags.append(tag) + + return built_in_tags, custom_tags + + @classmethod + def handle_add_tag(cls, tag_description: str, package_obj: GsePackages, package_desc_obj: GsePackageDesc): + """ + 给已有的agent包新增标签，即添加名为tag_name的标签 + :param tag_description: 添加的标签名 + :param package_obj: Gse包记录 + :param package_desc_obj: Gse包描述记录 + """ + # 如果新增的是内置标签，将原有的内置标签中的target_version进行修改即可，否则创建一个新的标签 + if tag_description in constants.BUILT_IN_TAG_DESCRIPTIONS + constants.BUILT_IN_TAG_NAMES: + Tag.objects.filter( + name=constants.TAG_DESCRIPTION__TAG_NAME.get(tag_description, tag_description), + target_id=package_desc_obj.id, + ).update(target_version=package_obj.version) + else: + tag: Tag = Tag.objects.filter(description=tag_description, target_id=package_desc_obj.id).first() + Tag.objects.create( + name=tag.name if tag else GsePackageTools.generate_name_by_description(tag_description), + description=tag_description, + target_type=TargetType.AGENT.value, + target_id=package_desc_obj.id, + target_version=package_obj.version, + ) + + @classmethod + def handle_update_tag( + cls, tag_description: str, package_obj: GsePackages, package_desc_obj: GsePackageDesc, tag_obj: Tag + ): + """ + 给已有的agent包修改标签，即将原有的tag_obj中的描述修改为tag_name + :param tag_description: 待修改的标签描述 + :param package_obj: Gse包记录 + :param package_desc_obj: Gse包描述记录 + :param tag_obj: 原有的标签记录 + """ + # 1. 内置标签(target_version置空) -> 内置标签(覆盖原有的target_version) + # 2. 自定义标签(删除) -> 内置标签(覆盖原有的target_version) + # 3. 内置标签(target_version置空) -> 自定义标签(新增) + # 4. 自定义标签 -> 自定义标签(将原有的标签描述进行修改) + + # 如果目标标签为内置标签的话，将内置标签的target_version进行覆盖，并对原来的标签进行删除或者清空 + # 如果目标标签为自定义标签，原有标签为内置标签的话，原有标签target_version置空，并新增自定义标签 + # 否则(目标和原有都为自定义标签)将直接修改原有标签的target_version + if tag_description in constants.BUILT_IN_TAG_DESCRIPTIONS + constants.BUILT_IN_TAG_NAMES: + Tag.objects.filter( + name=constants.TAG_DESCRIPTION__TAG_NAME.get(tag_description, tag_description), + target_id=package_desc_obj.id, + ).update(target_version=package_obj.version) + cls.handle_delete_tag(tag_obj.name, tag_obj) + elif tag_obj.name in constants.BUILT_IN_TAG_NAMES: + tag_obj.target_version = "" + tag_obj.save() + cls.handle_add_tag(tag_description, package_obj, package_desc_obj) + else: + tag: Tag = Tag.objects.filter(description=tag_description, target_id=package_desc_obj.id).first() + tag_obj.name = tag.name if tag else GsePackageTools.generate_name_by_description(tag_description) + tag_obj.description = tag_description + tag_obj.save() + + @classmethod + def handle_delete_tag(cls, tag_name: str, tag_obj: Tag): + """ + 删除已有的agent包标签，即删除id为tag_id的标签 + :param tag_name: 待删除的标签id + :param tag_obj: 待删除的标签记录 + """ + # 如果是删除内置标签，将target_version置空即可，不需要删除 + if tag_name in constants.BUILT_IN_TAG_NAMES: + tag_obj.target_version = "" + tag_obj.save() + else: + tag_obj.delete() + + +@receiver(request_finished) +def clear_gse_package_handler_cache(sender, **kwargs): + """每次视图结束后清除缓存，保证每次视图获取的都是最新数据""" + gse_package_handler.clear_caches() + + +gse_package_handler = GsePackageHandler() diff --git a/apps/node_man/handlers/host.py b/apps/node_man/handlers/host.py index 8f357fec5..390849436 100644 --- a/apps/node_man/handlers/host.py +++ b/apps/node_man/handlers/host.py @@ -527,6 +527,10 @@ def update_proxy_info(params: dict): setattr(identity, kwarg, identity_kwargs[kwarg]) identity.save() + # 更新ProcessStatus中Proxy包版本信息 + if params.get("version"): + ProcessStatus.objects.filter(bk_host_id=kwargs["bk_host_id"]).update(version=params["version"]) + @staticmethod def get_host_infos_gby_ip_key(ips: Iterable[str], ip_version: int): """ diff --git a/apps/node_man/handlers/meta.py b/apps/node_man/handlers/meta.py index d6447ee87..37a3966e8 100644 --- a/apps/node_man/handlers/meta.py +++ b/apps/node_man/handlers/meta.py @@ -10,17 +10,20 @@ """ import re from collections import ChainMap -from typing import Any, Callable, Dict, Tuple +from typing import Any, Callable, Dict, List, Tuple from django.conf import settings from django.db import connection from django.utils.translation import ugettext as _ from apps.core.concurrent.cache import FuncCacheDecorator -from apps.node_man import constants, models, tools +from apps.node_man import constants, exceptions, models, tools from apps.node_man.handlers.cloud import CloudHandler from apps.node_man.handlers.cmdb import CmdbHandler +from apps.node_man.handlers.gse_package import gse_package_handler from apps.node_man.handlers.install_channel import InstallChannelHandler +from apps.node_man.models import GsePackages +from apps.node_man.permissions.package_manage import PackageManagePermission from apps.node_man.tools import JobTools from apps.utils import APIModel @@ -602,6 +605,74 @@ def fetch_dept_name_children(dept_names: Tuple): dept_name_children.append({"id": dept_name, "name": dept_name}) return dept_name_children + def fetch_agent_pkg_manager_children(self, params=None): + params: Dict[str, Any] = params or {} + project: str = params.get("project", "gse_agent") + + if not PackageManagePermission().has_permission(None, None): + raise exceptions.PermissionDeniedError(_("该用户不是管理员")) + + versions, tag_name__description_map, creators, is_readys = set(), dict(), set(), set() + gse_packages = GsePackages.objects.filter(project=project).values("version", "created_by", "is_ready") + for package in gse_packages: + tags: List[Dict[str, Any]] = gse_package_handler.get_tags( + version=package["version"], + project=project, + enable_tag_separation=False, + ) + versions.add(package["version"]) + creators.add(package["created_by"]) + is_readys.add(package["is_ready"]) + for tag in tags: + tag_name__description_map[tag["name"]] = tag["description"] + + return [ + { + "name": _("版本号"), + "id": "version", + "children": [ + { + "id": version, + "name": version, + } + for version in versions + ], + }, + { + "name": _("标签信息"), + "id": "tag_names", + "children": [ + { + "id": tag_name, + "name": tag_description, + } + for tag_name, tag_description in tag_name__description_map.items() + ], + }, + { + "name": _("上传用户"), + "id": "created_by", + "children": [ + { + "id": creator, + "name": creator, + } + for creator in creators + ], + }, + { + "name": _("状态"), + "id": "is_ready", + "children": [ + { + "id": is_ready, + "name": constants.GSE_PACKAGE_ENABLE_ALIAS_MAP.get(is_ready, is_ready), + } + for is_ready in is_readys + ], + }, + ] + def filter_condition(self, category, params=None): """ 获取过滤条件 @@ -629,6 +700,8 @@ def filter_condition(self, category, params=None): elif category == "os_type": ret = self.fetch_os_type_children() return ret + elif category == "agent_pkg_manage": + return self.fetch_agent_pkg_manager_children(params=params) @staticmethod def install_default_values_formatter(install_default_values: Dict[str, Dict[str, Any]]): diff --git a/apps/node_man/handlers/plugin_v2.py b/apps/node_man/handlers/plugin_v2.py index 983136b83..526ec6266 100644 --- a/apps/node_man/handlers/plugin_v2.py +++ b/apps/node_man/handlers/plugin_v2.py @@ -8,22 +8,16 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import json -import os import random from collections import ChainMap, defaultdict from typing import Any, Dict, List, Union -import requests from django.conf import settings from django.core.cache import cache -from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models import Count from django.utils.translation import ugettext_lazy as _ from apps.component.esbclient import client_v2 -from apps.core.files import core_files_constants -from apps.core.files.storage import get_storage from apps.node_man import constants, exceptions, models, tools from apps.node_man.constants import IamActionType from apps.node_man.handlers.cmdb import CmdbHandler @@ -31,72 +25,11 @@ from apps.utils.basic import distinct_dict_list, list_slice from apps.utils.batch_request import batch_request from apps.utils.concurrent import batch_call -from apps.utils.files import md5sum from apps.utils.local import get_request_username from common.api import NodeApi class PluginV2Handler: - @staticmethod - def upload(package_file: InMemoryUploadedFile, module: str) -> Dict[str, Any]: - """ - 将文件上传至 - :param package_file: InMemoryUploadedFile - :param module: 所属模块 - :return: - { - "result": True, - "message": "", - "code": "00", - "data": { - "id": record.id, # 上传文件记录ID - "name": record.file_name, # 包名 - "pkg_size": record.file_size, # 大小， - } - } - """ - with package_file.open("rb") as tf: - - # 计算上传文件的md5 - md5 = md5sum(file_obj=tf, closed=False) - - base_params = {"module": module, "md5": md5} - - # 如果采用对象存储，文件直接上传至仓库，并将返回的目标路径传到后台，由后台进行校验并创建上传记录 - # TODO 后续应该由前端上传文件并提供md5 - if settings.STORAGE_TYPE in core_files_constants.StorageType.list_cos_member_values(): - storage = get_storage() - - try: - storage_path = storage.save(name=os.path.join(settings.UPLOAD_PATH, tf.name), content=tf) - except Exception as e: - raise exceptions.PluginUploadError(plugin_name=tf.name, error=e) - - return NodeApi.upload( - { - **base_params, - # 最初文件上传的名称，后台会使用该文件名保存并覆盖同名文件 - "file_name": tf.name, - "file_path": storage_path, - "download_url": storage.url(storage_path), - } - ) - - else: - - response = requests.post( - url=settings.DEFAULT_FILE_UPLOAD_API, - data={ - **base_params, - "bk_app_code": settings.APP_CODE, - "bk_username": get_request_username(), - }, - # 本地文件系统仍通过上传文件到Nginx并回调后台 - files={"package_file": tf}, - ) - - return json.loads(response.content) - @staticmethod def list_plugin(query_params: Dict): plugin_page = NodeApi.plugin_list(query_params) diff --git a/apps/node_man/migrations/0084_auto_20241112_1539.py b/apps/node_man/migrations/0084_auto_20241112_1539.py new file mode 100644 index 000000000..fc82cf1c4 --- /dev/null +++ b/apps/node_man/migrations/0084_auto_20241112_1539.py @@ -0,0 +1,139 @@ +# Generated by Django 3.2.4 on 2024-11-12 07:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0083_subscription_operate_info"), + ] + + operations = [ + migrations.CreateModel( + name="GsePackageDesc", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("created_by", models.CharField(default="", max_length=32, verbose_name="创建者")), + ("updated_time", models.DateTimeField(auto_now=True, null=True, verbose_name="更新时间")), + ("updated_by", models.CharField(blank=True, default="", max_length=32, verbose_name="修改者")), + ("project", models.CharField(db_index=True, max_length=32, unique=True, verbose_name="工程名")), + ("description", models.TextField(verbose_name="安装包描述")), + ("description_en", models.TextField(blank=True, null=True, verbose_name="英文插件描述")), + ( + "category", + models.CharField( + choices=[("official", "official"), ("external", "external"), ("scripts", "scripts")], + max_length=32, + verbose_name="所属范围", + ), + ), + ], + options={ + "verbose_name": "Gse包描述（GsePackageDesc）", + "verbose_name_plural": "Gse包描述（GsePackageDesc）", + }, + ), + migrations.CreateModel( + name="GsePackages", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("created_by", models.CharField(default="", max_length=32, verbose_name="创建者")), + ("updated_time", models.DateTimeField(auto_now=True, null=True, verbose_name="更新时间")), + ("updated_by", models.CharField(blank=True, default="", max_length=32, verbose_name="修改者")), + ("pkg_name", models.CharField(max_length=128, verbose_name="压缩包名")), + ("version", models.CharField(max_length=128, verbose_name="版本号")), + ("project", models.CharField(db_index=True, max_length=32, verbose_name="工程名")), + ("pkg_size", models.IntegerField(verbose_name="包大小")), + ("pkg_path", models.CharField(max_length=128, verbose_name="包路径")), + ("md5", models.CharField(max_length=32, verbose_name="md5值")), + ("location", models.CharField(max_length=512, verbose_name="安装包链接")), + ( + "os", + models.CharField( + choices=[ + ("windows", "windows"), + ("linux", "linux"), + ("aix", "aix"), + ("solaris", "solaris"), + ("darwin", "darwin"), + ], + db_index=True, + default="linux", + max_length=32, + verbose_name="系统类型", + ), + ), + ( + "cpu_arch", + models.CharField( + choices=[ + ("x86", "x86"), + ("x86_64", "x86_64"), + ("powerpc", "powerpc"), + ("aarch64", "aarch64"), + ("sparc", "sparc"), + ], + db_index=True, + default="x86_64", + max_length=32, + verbose_name="CPU类型", + ), + ), + ("is_ready", models.BooleanField(default=True, verbose_name="插件是否可用")), + ("version_log", models.TextField(blank=True, null=True, verbose_name="版本日志")), + ("version_log_en", models.TextField(blank=True, null=True, verbose_name="英文版本日志")), + ], + options={ + "verbose_name": "Gse包（GsePackages）", + "verbose_name_plural": "Gse包（GsePackages）", + }, + ), + migrations.AlterField( + model_name="job", + name="job_type", + field=models.CharField( + choices=[ + ("INSTALL_AGENT", "INSTALL_AGENT"), + ("RESTART_AGENT", "RESTART_AGENT"), + ("REINSTALL_AGENT", "REINSTALL_AGENT"), + ("UNINSTALL_AGENT", "UNINSTALL_AGENT"), + ("REMOVE_AGENT", "REMOVE_AGENT"), + ("UPGRADE_AGENT", "UPGRADE_AGENT"), + ("DOWNGRADE_AGENT", "DOWNGRADE_AGENT"), + ("IMPORT_AGENT", "IMPORT_AGENT"), + ("RESTART_AGENT", "RESTART_AGENT"), + ("RELOAD_AGENT", "RELOAD_AGENT"), + ("ACTIVATE_AGENT", "ACTIVATE_AGENT"), + ("MAIN_START_PLUGIN", "MAIN_START_PLUGIN"), + ("MAIN_STOP_PLUGIN", "MAIN_STOP_PLUGIN"), + ("MAIN_RESTART_PLUGIN", "MAIN_RESTART_PLUGIN"), + ("MAIN_RELOAD_PLUGIN", "MAIN_RELOAD_PLUGIN"), + ("MAIN_DELEGATE_PLUGIN", "MAIN_DELEGATE_PLUGIN"), + ("MAIN_UNDELEGATE_PLUGIN", "MAIN_UNDELEGATE_PLUGIN"), + ("MAIN_INSTALL_PLUGIN", "MAIN_INSTALL_PLUGIN"), + ("MAIN_STOP_AND_DELETE_PLUGIN", "MAIN_STOP_AND_DELETE_PLUGIN"), + ("DEBUG_PLUGIN", "DEBUG_PLUGIN"), + ("STOP_DEBUG_PLUGIN", "STOP_DEBUG_PLUGIN"), + ("PUSH_CONFIG_PLUGIN", "PUSH_CONFIG_PLUGIN"), + ("REMOVE_CONFIG_PLUGIN", "REMOVE_CONFIG_PLUGIN"), + ("PACKING_PLUGIN", "PACKING_PLUGIN"), + ("INSTALL_PROXY", "INSTALL_PROXY"), + ("RESTART_PROXY", "RESTART_PROXY"), + ("REINSTALL_PROXY", "REINSTALL_PROXY"), + ("REPLACE_PROXY", "REPLACE_PROXY"), + ("UNINSTALL_PROXY", "UNINSTALL_PROXY"), + ("UPGRADE_PROXY", "UPGRADE_PROXY"), + ("IMPORT_PROXY", "IMPORT_PROXY"), + ("RESTART_PROXY", "RESTART_PROXY"), + ("RELOAD_PROXY", "RELOAD_PROXY"), + ], + db_index=True, + default="INSTALL_PROXY", + max_length=45, + verbose_name="作业类型", + ), + ), + ] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 7c9a84ddb..7cb84ab5c 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -2533,3 +2533,49 @@ class Meta: index_together = [ ["bk_biz_id", "enable"], ] + + +class GsePackages(orm.OperateRecordModel): + pkg_name = models.CharField(_("压缩包名"), max_length=128) + version = models.CharField(_("版本号"), max_length=128) + project = models.CharField(_("工程名"), max_length=32, db_index=True) + pkg_size = models.IntegerField(_("包大小")) + pkg_path = models.CharField(_("包路径"), max_length=128) + md5 = models.CharField(_("md5值"), max_length=32) + location = models.CharField(_("安装包链接"), max_length=512) + os = models.CharField( + _("系统类型"), + max_length=32, + choices=constants.PLUGIN_OS_CHOICES, + default=constants.PluginOsType.linux, + db_index=True, + ) + cpu_arch = models.CharField( + _("CPU类型"), max_length=32, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True + ) + + # 由于创建记录时，文件可能仍然在传输过程中，因此需要标志位判断是否已经可用 + is_ready = models.BooleanField(_("插件是否可用"), default=True) + + version_log = models.TextField(_("版本日志"), null=True, blank=True) + version_log_en = models.TextField(_("英文版本日志"), null=True, blank=True) + + class Meta: + verbose_name = _("Gse包（GsePackages）") + verbose_name_plural = _("Gse包（GsePackages）") + + +class GsePackageDesc(orm.OperateRecordModel): + """ + Gse包描述表 + """ + + # 安装包名需要全局唯一，防止冲突 + project = models.CharField(_("工程名"), max_length=32, unique=True, db_index=True) + description = models.TextField(_("安装包描述")) + description_en = models.TextField(_("英文插件描述"), null=True, blank=True) + category = models.CharField(_("所属范围"), max_length=32, choices=constants.CATEGORY_CHOICES) + + class Meta: + verbose_name = _("Gse包描述（GsePackageDesc）") + verbose_name_plural = _("Gse包描述（GsePackageDesc）") diff --git a/apps/node_man/periodic_tasks/register_gse_package.py b/apps/node_man/periodic_tasks/register_gse_package.py new file mode 100644 index 000000000..aa6fbb5a3 --- /dev/null +++ b/apps/node_man/periodic_tasks/register_gse_package.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from apps.backend.celery import app +from apps.node_man.tools.gse_package import GsePackageTools + +logger = logging.getLogger("app") + + +@app.task(queue="default") +def register_gse_package_task(file_name: str, tags: List[str]): + upload_package_obj = GsePackageTools.get_latest_upload_record(file_name=file_name) + + project, artifact_builder_class = GsePackageTools.distinguish_gse_package(file_path=upload_package_obj.file_path) + + with artifact_builder_class( + initial_artifact_path=upload_package_obj.file_path, + tags=tags, + ) as builder: + builder.make() diff --git a/apps/node_man/permissions/__init__.py b/apps/node_man/permissions/__init__.py new file mode 100644 index 000000000..29ed269e0 --- /dev/null +++ b/apps/node_man/permissions/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/apps/node_man/permissions/package_manage.py b/apps/node_man/permissions/package_manage.py new file mode 100644 index 000000000..bc8ae81d8 --- /dev/null +++ b/apps/node_man/permissions/package_manage.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ +from rest_framework import permissions + +from apps.node_man.handlers.iam import IamHandler +from apps.utils.local import get_request_username + + +class PackageManagePermission(permissions.BasePermission): + message = _("您没有该操作的权限") + + def has_permission(self, request, view): + + if IamHandler().is_superuser(get_request_username()): + return True + + return False diff --git a/apps/node_man/serializers/host.py b/apps/node_man/serializers/host.py index 2d2b07ae1..81018d5c3 100644 --- a/apps/node_man/serializers/host.py +++ b/apps/node_man/serializers/host.py @@ -55,6 +55,7 @@ class HostUpdateSerializer(serializers.Serializer): bt_speed_limit = serializers.IntegerField(label=_("加速"), required=False) enable_compression = serializers.BooleanField(label=_("数据压缩开关配置"), required=False, default=False) data_path = serializers.CharField(label=_("数据文件路径"), required=False) + version = serializers.CharField(label=_("Proxy包版本"), required=False) def validate(self, attrs): cipher = tools.HostTools.get_asymmetric_cipher() diff --git a/apps/node_man/serializers/job.py b/apps/node_man/serializers/job.py index dba4db497..9d5ba8503 100644 --- a/apps/node_man/serializers/job.py +++ b/apps/node_man/serializers/job.py @@ -17,7 +17,10 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from apps.backend.subscription.steps.agent_adapter.adapter import LEGACY +from apps.backend.subscription.steps.agent_adapter.adapter import ( + LEGACY, + AgentVersionSerializer, +) from apps.core.gray.constants import INSTALL_OTHER_AGENT_AP_ID_OFFSET from apps.core.gray.handlers import GrayHandler from apps.core.gray.tools import GrayTools @@ -52,11 +55,18 @@ def set_agent_setup_info_to_attrs(attrs): # 如果开启 DHCP，安装 2.0 Agent，开启 AgentID 特性 # 在执行模块根据主机接入点所属的 GSE 版本决定是否采用下列的 agent_setup_info name = ("gse_agent", "gse_proxy")[attrs["node_type"] == "PROXY"] + # attrs["agent_setup_info"]["name"] = name + # 处理重装类型setup_info结构 + agent_setup_info = attrs.get("agent_setup_info", {}) + global_settings_agent_version = models.GlobalSettings.get_config( + models.GlobalSettings.KeyEnum.GSE_AGENT2_VERSION.value, default="stable" + ) + attrs["agent_setup_info"] = { "name": name, - "version": models.GlobalSettings.get_config( - models.GlobalSettings.KeyEnum.GSE_AGENT2_VERSION.value, default="stable" - ), + "version": agent_setup_info.get("version") or global_settings_agent_version, + "choice_version_type": agent_setup_info.get("choice_version_type") or constants.AgentVersionType.UNIFIED.value, + "version_map_list": agent_setup_info.get("version_map_list", []), } @@ -113,7 +123,6 @@ def backfill_bk_host_id(self, hosts): else: sub_query.children.append(("inner_ipv6", _host["inner_ipv6"])) ip_key = _host["inner_ipv6"] - cloud_ip_host_info_map[f"{_host['bk_cloud_id']}:{ip_key}"] = _host query_params.children.append(sub_query) @@ -265,6 +274,11 @@ class AgentSetupInfoSerializer(serializers.Serializer): # LEGACY 表示旧版本 Agent，仅做兼容 version = serializers.CharField(required=False, label="构件版本", default=LEGACY) + choice_version_type = serializers.ChoiceField( + required=False, choices=constants.AgentVersionType.list_choices(), label=_("选择Agent Version类型") + ) + version_map_list = AgentVersionSerializer(required=False, many=True) + class ScriptHook(serializers.Serializer): name = serializers.CharField(label=_("脚本名称"), min_length=1) diff --git a/apps/node_man/serializers/meta.py b/apps/node_man/serializers/meta.py index 2ceb0068a..86b07856b 100644 --- a/apps/node_man/serializers/meta.py +++ b/apps/node_man/serializers/meta.py @@ -30,6 +30,7 @@ class JobSettingSerializer(serializers.Serializer): class FilterConditionSerializer(serializers.Serializer): category = serializers.CharField(label=_("分类"), required=False, default="") bk_biz_ids = serializers.ListField(label=_("业务列表"), required=False, default=[], child=serializers.IntegerField()) + project = serializers.CharField(label=_("工程名"), required=False) # 时间范围 start_time = serializers.DateTimeField(label=_("起始时间"), required=False) diff --git a/apps/node_man/serializers/package_manage.py b/apps/node_man/serializers/package_manage.py new file mode 100644 index 000000000..2827b8816 --- /dev/null +++ b/apps/node_man/serializers/package_manage.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from apps.core.tag.constants import TargetType +from apps.exceptions import ValidationError +from apps.node_man.constants import GsePackageCode +from apps.node_man.handlers.gse_package import gse_package_handler +from apps.node_man.models import UploadPackage + + +class TagSerializer(serializers.Serializer): + name = serializers.CharField() + description = serializers.CharField() + + +class TagProjectSerializer(serializers.Serializer): + project = serializers.CharField(default=GsePackageCode.AGENT.value) + + +class TagCreateSerializer(serializers.Serializer): + tag_descriptions = serializers.ListField(child=serializers.CharField(), default=[]) + project = serializers.CharField() + + def validate(self, attrs): + project = attrs["project"] + if project not in GsePackageCode.values(): + raise ValidationError(_("project可选项[ gse_agent | gse_plugin ]")) + + return attrs + + class Meta: + ref_name = "tag_create" + + +class ParentTagSerializer(serializers.Serializer): + name = serializers.CharField() + description = serializers.CharField() + children = TagSerializer(many=True) + + +class ConditionsSerializer(serializers.Serializer): + key = serializers.ChoiceField(choices=["version", "os_cpu_arch", "tags", "is_ready"]) + values = serializers.ListField() + + +class BasePackageSerializer(serializers.Serializer): + @staticmethod + def get_tags(obj, enable_tag_separation=True): + return gse_package_handler.get_tags( + project=obj.project, + version=obj.version, + enable_tag_separation=enable_tag_separation, + ) + + @classmethod + def get_description(cls, obj): + return gse_package_handler.get_description(project=obj.project) + + +class PackageSerializer(BasePackageSerializer): + id = serializers.IntegerField() + pkg_name = serializers.CharField() + version = serializers.CharField() + os = serializers.CharField() + cpu_arch = serializers.CharField() + tags = serializers.SerializerMethodField() + created_by = serializers.CharField() + created_time = serializers.DateTimeField() + is_ready = serializers.BooleanField() + + +class ListResponseSerializer(serializers.Serializer): + total = serializers.IntegerField() + list = PackageSerializer(many=True) + + +class OperateSerializer(serializers.Serializer): + is_ready = serializers.BooleanField(required=False) + tags = serializers.ListField(required=False) + + def update(self, instance, validated_data): + for attr, value in validated_data.items(): + setattr(instance, attr, value) + + instance.save() + return instance + + +# class QuickSearchSerializer(serializers.Serializer): +# project = serializers.ChoiceField(choices=GsePackageCode.list_choices()) + + +class UploadSerializer(serializers.Serializer): + overload = serializers.BooleanField(default=False, help_text="是否覆盖上传") + package_file = serializers.FileField() + + def validate(self, data): + overload = data.get("overload") + package_file = data.get("package_file") + + file_name = package_file.name + + if not (file_name.endswith(".tgz") or file_name.endswith(".tar.gz")): + raise ValidationError(_("仅支持'tgz', 'tar.gz'拓展名的文件")) + + if not overload: + upload_package: UploadPackage = UploadPackage.objects.filter( + file_name__contains=file_name, module=TargetType.AGENT.value + ).first() + if upload_package: + raise ValidationError( + data={ + "message": _("存在同名agent包"), + "file_name": upload_package.file_name, + "md5": upload_package.md5, + }, + code=3800002, + ) + + return data + + +class UploadResponseSerializer(serializers.Serializer): + name = serializers.CharField() + pkg_size = serializers.IntegerField() + + +class ParseSerializer(serializers.Serializer): + file_name = serializers.CharField() + + +class ParseResponseSerializer(serializers.Serializer): + class ParsePackageSerializer(serializers.Serializer): + project = serializers.ChoiceField(choices=["gse_agent", "gse_proxy"], required=False) + pkg_name = serializers.CharField(required=False, source="pkg_relative_path") + version = serializers.CharField(required=False) + os = serializers.CharField() + cpu_arch = serializers.CharField() + config_templates = serializers.ListField(default=[]) + + def to_representation(self, instance): + data = super().to_representation(instance) + data["project"] = self.context.get("project", "") + data["version"] = self.context.get("version", "") + return data + + description = serializers.CharField() + packages = ParsePackageSerializer(many=True) + + +class AgentRegisterSerializer(serializers.Serializer): + file_name = serializers.CharField() + tags = serializers.ListField(child=serializers.CharField(), default=[]) + tag_descriptions = serializers.ListField(child=serializers.CharField(), default=[]) + project = serializers.CharField(default=GsePackageCode.AGENT.value) + + +class AgentRegisterTaskSerializer(serializers.Serializer): + task_id = serializers.CharField() + version = serializers.CharField() + + +class AgentRegisterTaskResponseSerializer(serializers.Serializer): + is_finish = serializers.BooleanField() + status = serializers.ChoiceField(choices=["SUCCESS", "FAILED", "RUNNING"]) + message = serializers.CharField() + + +class DeployedAgentCountSerializer(serializers.Serializer): + items = serializers.JSONField(default=[]) + project = serializers.CharField(default=GsePackageCode.AGENT.value) + + +class VersionQuerySerializer(serializers.Serializer): + project = serializers.CharField() + os = serializers.CharField(required=False, allow_blank=True) + cpu_arch = serializers.CharField(required=False, allow_blank=True) + versions = serializers.ListField(child=serializers.CharField(), required=False) + + +class VersionCompareSerializer(serializers.Serializer): + current_version = serializers.CharField() + version_to_compares = serializers.ListField(child=serializers.CharField()) diff --git a/apps/node_man/tests/test_views/test_package_manage_views.py b/apps/node_man/tests/test_views/test_package_manage_views.py new file mode 100644 index 000000000..c110e1501 --- /dev/null +++ b/apps/node_man/tests/test_views/test_package_manage_views.py @@ -0,0 +1,471 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import io +import os +import uuid +from unittest.mock import patch + +from django.conf import settings +from django.core.files.uploadedfile import InMemoryUploadedFile + +from apps.backend.plugin.handler import PluginHandler +from apps.backend.sync_task.constants import SyncTaskType +from apps.backend.tests.subscription.agent_adapter.test_adapter import ( + Proxy2StepAdapterTestCase, +) +from apps.core.files.storage import get_storage +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.node_man import constants +from apps.node_man.constants import ( + CPU_TUPLE, + GSE_PACKAGE_ENABLE_ALIAS_MAP, + OS_TYPE, + CategoryType, + GsePackageCode, +) +from apps.node_man.handlers.meta import MetaHandler +from apps.node_man.models import ( + GsePackageDesc, + GsePackages, + Host, + ProcessStatus, + UploadPackage, +) +from apps.node_man.tests.utils import create_gse_package, create_host +from apps.utils.files import md5sum +from common.api.modules.utils import add_esb_info_before_request + + +def delay(self, *args, **kwargs): + self.task_func(*args, **kwargs) + return "1" + + +class PackageManageViewsTestCaseUsingProxy(Proxy2StepAdapterTestCase): + def init_tags(self): + try: + agent_target_id = GsePackageDesc.objects.get( + project=GsePackageCode.AGENT.value, category=CategoryType.official + ).id + + proxy_target_id = GsePackageDesc.objects.get( + project=GsePackageCode.PROXY.value, category=CategoryType.official + ).id + except GsePackageDesc.DoesNotExist: + agent_target_id = GsePackageDesc.objects.create( + project=GsePackageCode.AGENT.value, category=CategoryType.official + ).id + + proxy_target_id = GsePackageDesc.objects.create( + project=GsePackageCode.PROXY.value, category=CategoryType.official + ).id + + for target_id in [proxy_target_id, agent_target_id]: + # 添加Tag记录 + Tag.objects.create( + name="stable", + description="稳定版本", + target_id=target_id, + target_type=TargetType.AGENT.value, + ) + Tag.objects.create( + name="latest", + description="最新版本", + target_id=target_id, + target_type=TargetType.AGENT.value, + ) + Tag.objects.create( + name="test", + description="测试版本", + target_id=target_id, + target_type=TargetType.AGENT.value, + ) + + def upload_file(self): + with open(self.ARCHIVE_PATH, "rb") as f: + file_content = f.read() + + memory_uploaded_file = InMemoryUploadedFile( + file=io.BytesIO(file_content), + field_name=None, + name=os.path.basename(self.ARCHIVE_PATH), + content_type="text/plain", + size=len(file_content), + charset=None, + ) + + self.storage = get_storage() + with memory_uploaded_file.open("rb") as tf: + self.md5 = md5sum(file_obj=tf, closed=False) + self.storage_path = self.storage.save(name=os.path.join(settings.UPLOAD_PATH, tf.name), content=tf) + + self.file_name = tf.name + + params = { + "md5": self.md5, + "module": "agent", + "file_name": tf.name, + "file_path": self.storage_path, + "download_url": self.storage.url(self.storage_path), + } + + add_esb_info_before_request(params) + + PluginHandler.upload( + md5=params["md5"], + origin_file_name=params["file_name"], + module=params["module"], + operator=params["bk_username"], + app_code=params["bk_app_code"], + file_path=params.get("file_path"), + download_url=params.get("download_url"), + ) + + def setUp(self, *args, **kwargs): + super().setUp() + + self.init_tags() + + with self.ARTIFACT_BUILDER_CLASS(initial_artifact_path=self.ARCHIVE_PATH, tags=["stable", "latest"]) as builder: + builder.make() + + self.upload_file() + + gse_package = GsePackages.objects.first() + gse_package.created_at = "admin" + gse_package.save() + + self.task_map = {} + + @classmethod + def clear_agent_data(cls): + GsePackages.objects.all().delete() + GsePackageDesc.objects.all().delete() + Tag.objects.all().delete() + + def tearDown(self): + super().tearDown() + + if os.path.exists(self.storage_path): + os.remove(self.storage_path) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_list(self, *args, **kwargs): + # 和之前的builder.make加起来100 + create_gse_package(99, start_id=1000, project="gse_proxy") + + result = self.client.get(path="/api/agent/package/", data={"project": "gse_proxy"}) + self.assertEqual(result["result"], True) + self.assertEqual(len(result["data"]), 100) + + result = self.client.get(path="/api/agent/package/", data={"page": 1, "pagesize": 2, "project": "gse_proxy"}) + self.assertEqual(result["result"], True) + self.assertEqual(result["data"]["total"], 100) + self.assertEqual(len(result["data"]["list"]), 2) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_list_with_filter_condition(self, *args, **kwargs): + # 不筛选 + result = self.client.get(path="/api/agent/package/", data={"page": 1, "pagesize": 2, "project": "gse_proxy"}) + self.assertEqual(result["data"]["total"], 1) + self.assertEqual(len(result["data"]["list"]), 1) + + gse_package = GsePackages.objects.first() + + # 筛选tags + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "tag_names": "stable", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertIn("stable", self.collect_all_tag_names(result["data"]["list"][0]["tags"])) + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "tag_names": "latest", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertIn("latest", self.collect_all_tag_names(result["data"]["list"][0]["tags"])) + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "tag_names": "test", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选os_cpu_arch + computer_os, cpu_arch = "linux", "x86_64" + result = self.client.get( + path="/api/agent/package/", + data={ + "page": 1, + "pagesize": 2, + "os": gse_package.os, + "cpu_arch": gse_package.cpu_arch, + "project": "gse_proxy", + }, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["os"], computer_os) + self.assertEqual(result["data"]["list"][0]["cpu_arch"], cpu_arch) + result = self.client.get( + path="/api/agent/package/", + data={"page": 1, "pagesize": 2, "os": "windows", "cpu_arch": "x86_64", "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选created_by + result = self.client.get( + path="/api/agent/package/", + data={"page": 1, "pagesize": 2, "created_by": gse_package.created_by, "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["created_by"], gse_package.created_by) + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "created_by": "system", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选is_ready + result = self.client.get( + path="/api/agent/package/", + data={"page": 1, "pagesize": 2, "is_ready": str(gse_package.is_ready), "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["is_ready"], True) + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "is_ready": "False", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选version + result = self.client.get( + path="/api/agent/package/", + data={"page": 1, "pagesize": 2, "version": gse_package.version, "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["version"], "1.0.1") + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "version": "1.0.2", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + @classmethod + def collect_all_tag_names(cls, tags, *args, **kwargs): + """ + tags: [ + { + "id": "builtin", + "name": "内置标签", + "children": [ + {"id": "stable", "name": "稳定版本", "children": []}, + {"id": "latest", "name": "最新版本", "children": []}, + ], + }, + { + "id": "custom", + "name": "自定义标签", + "children": [ + {"id": "custom", "name": "自定义版本", "children": []} + ] + }, + ] + """ + tag_name_set = set() + for parent_tag in tags: + for children_tag in parent_tag["children"]: + tag_name_set.add(children_tag["name"]) + + return tag_name_set + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_update(self, *args, **kwargs): + first_gse_package = GsePackages.objects.first() + self.assertEqual(first_gse_package.is_ready, True) + self.client.put( + path=f"/api/agent/package/{first_gse_package.id}/", data={"is_ready": False, "project": "gse_proxy"} + ) + self.assertEqual(GsePackages.objects.first().is_ready, False) + + # 测试更新不存在的id是否服务器异常 + self.client.put(path="/api/agent/package/10000/", data={"is_ready": False}) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_destroy(self, *args, **kwargs): + gse_packages = GsePackages.objects.all() + self.assertEqual(len(gse_packages), 1) + self.client.delete(path=f"/api/agent/package/{gse_packages.first().id}/", data={"project": "gse_proxy"}) + self.assertEqual(len(GsePackages.objects.all()), 0) + + # 测试删除存在的id是否服务器异常 + self.client.delete(path="/api/agent/package/10000/") + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_quick_search_condition(self, *args, **kwargs): + result = self.client.get(path="/api/agent/package/quick_search_condition/", data={"project": "gse_proxy"}) + for condition in result["data"]: + if condition["id"] == "os_cpu_arch": + self.assertCountEqual( + condition["children"], + [ + {"id": "linux_x86_64", "name": "Linux_x86_64", "count": 1, "description": "### 1.0.1\nchange"}, + ], + ) + self.assertEqual(condition["count"], 1) + + elif condition["id"] == "version": + self.assertCountEqual( + condition["children"], + [ + {"id": "1.0.1", "name": "1.0.1", "count": 1, "description": "### 1.0.1\nchange"}, + ], + ) + self.assertEqual(condition["count"], 1) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_filter_condition_with_agent_pkg_manage(self, *args, **kwargs): + result = MetaHandler().filter_condition("agent_pkg_manage", params={"project": "gse_proxy"}) + self.assertEqual(len(GsePackages.objects.all()), 1) + gse_package = GsePackages.objects.first() + is_ready = gse_package.is_ready + for condition in result: + if condition["id"] == "version": + self.assertCountEqual( + condition["children"], + [ + {"id": gse_package.version, "name": gse_package.version}, + ], + ) + elif condition["id"] == "tags": + self.assertCountEqual( + condition["children"], + [ + {"id": "stable", "name": "稳定版本"}, + {"id": "latest", "name": "最新版本"}, + ], + ) + elif condition["id"] == "creator": + self.assertCountEqual( + condition["children"], + [ + {"id": gse_package.created_by, "name": gse_package.created_by}, + ], + ) + elif condition["id"] == "is_ready": + self.assertCountEqual( + condition["children"], + [ + {"id": is_ready, "name": GSE_PACKAGE_ENABLE_ALIAS_MAP.get(is_ready, is_ready)}, + ], + ) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_deployed_hosts_count(self, *args, **kwargs): + data = { + "items": [{"os_type": "linux", "cpu_arch": "x86_64"}, {"os_type": "windows", "cpu_arch": "x86_64"}], + "project": "gse_agent", + } + + # 100台主机都是LINUX，cpu_arch都为x86_64 + Host.objects.all().delete() + ProcessStatus.objects.all().delete() + create_host(100, os_type=constants.OsType.LINUX, node_type=constants.NodeType.AGENT) + result = self.client.post(path="/api/agent/package/deployed_hosts_count/", data=data) + self.assertEqual( + result["data"], + [ + {"os_type": "linux", "cpu_arch": "x86_64", "count": 100}, + {"os_type": "windows", "cpu_arch": "x86_64", "count": 0}, + ], + ) + + # 100台主机都是WINDOWS，cpu_arch都为x86_64 + Host.objects.all().delete() + ProcessStatus.objects.all().delete() + create_host(100, os_type=constants.OsType.WINDOWS, node_type=constants.NodeType.AGENT, start_idx=100) + result = self.client.post(path="/api/agent/package/deployed_hosts_count/", data=data) + self.assertEqual( + result["data"], + [ + {"os_type": "linux", "cpu_arch": "x86_64", "count": 0}, + {"os_type": "windows", "cpu_arch": "x86_64", "count": 100}, + ], + ) + + # 50台主机是WINDOWS，50台主机是LINUX，cpu_arch都为x86_64 + Host.objects.all().delete() + ProcessStatus.objects.all().delete() + create_host(50, os_type=constants.OsType.WINDOWS, node_type=constants.NodeType.AGENT, start_idx=200) + create_host(50, os_type=constants.OsType.LINUX, node_type=constants.NodeType.AGENT, start_idx=250) + result = self.client.post(path="/api/agent/package/deployed_hosts_count/", data=data) + self.assertEqual( + result["data"], + [ + {"os_type": "linux", "cpu_arch": "x86_64", "count": 50}, + {"os_type": "windows", "cpu_arch": "x86_64", "count": 50}, + ], + ) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_upload(self, *args, **kwargs): + self.assertEqual(UploadPackage.objects.count(), 1) + upload_package = UploadPackage.objects.first() + self.assertEqual(upload_package.module, "agent") + self.assertEqual(upload_package.file_path, self.storage_path) + self.assertEqual(upload_package.md5, self.md5) + self.assertEqual(upload_package.file_name, self.file_name) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_parse(self, *args, **kwargs): + res = self.client.post( + path="/backend/api/agent/parse/", + data={ + "file_name": self.file_name, + }, + ) + self.assertEqual(res["result"], True) + self.assertIn("description", res["data"]) + self.assertIn("packages", res["data"]) + + for packages in res["data"]["packages"]: + self.assertIn(packages["os"].upper(), OS_TYPE.values()) + self.assertIn(packages["project"], GsePackageCode.get_member_value__alias_map()) + self.assertIn(packages["cpu_arch"], CPU_TUPLE) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + @patch("apps.backend.sync_task.manager.AsyncTaskManager.delay", delay) + def test_create_register_task(self, *args, **kwargs): + file_name = kwargs.get("file_name") if "file_name" in kwargs else self.file_name + task_id = str(uuid.uuid4()) + self.task_map[task_id] = "PENDING" + + res = self.client.post( + path="/backend/api/sync_task/create/", + data={ + "task_name": SyncTaskType.REGISTER_GSE_PACKAGE.value, + "task_params": { + "file_name": file_name, + "tags": [], + }, + }, + ) + if res["result"] is True: + self.task_map[task_id] = "SUCCESS" + self.assertIn("task_id", res["data"]) + else: + self.task_map[task_id] = "FAILURE" + + return task_id + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + @patch("apps.backend.sync_task.manager.AsyncTaskManager.delay", delay) + def test_query_register_task(self, *args, **kwargs): + task_id = self.test_create_register_task() + self.assertEqual(self.task_map[task_id], "SUCCESS") + + wrong_task_id = self.test_create_register_task(file_name=self.file_name + "...") + self.assertEqual(self.task_map[wrong_task_id], "FAILURE") diff --git a/apps/node_man/tests/utils.py b/apps/node_man/tests/utils.py index 576f7883d..46a56d76c 100644 --- a/apps/node_man/tests/utils.py +++ b/apps/node_man/tests/utils.py @@ -27,6 +27,7 @@ from apps.node_man.models import ( AccessPoint, Cloud, + GsePackages, Host, IdentityData, InstallChannel, @@ -146,6 +147,7 @@ def create_host( login_ip=None, proc_type=None, os_type=None, + start_idx=0, ): # 若传了bk_host_id，number必须为1 host_to_create = [] @@ -156,8 +158,8 @@ def create_host( if number - index * max_count > max_count: # 若还要分批创建 host, process, identity = create_host_from_a_to_b( - index * max_count, - (index + 1) * max_count, + start_idx + index * max_count, + start_idx + (index + 1) * max_count, bk_host_id=bk_host_id, ip=ip, auth_type=auth_type, @@ -171,8 +173,8 @@ def create_host( ) else: host, process, identity = create_host_from_a_to_b( - index * max_count, - number, + start_idx + index * max_count, + start_idx + number, bk_host_id=bk_host_id, ip=ip, auth_type=auth_type, @@ -1267,3 +1269,40 @@ def ret_to_validate_data(data): login_ip_info = HostHandler.get_host_infos_gby_ip_key(login_ips, constants.CmdbIpVersion.V4.value) return biz_info, data, cloud_info, ap_id_name, inner_ip_info, outer_ip_info, login_ip_info, bk_biz_scope + + +def create_gse_package( + number, + start_id=1, + version=None, + project=None, + pkg_size=None, + is_ready=None, + pkg_path=None, + location=None, + created_by=None, + os=None, + cpu_arch=None, +): + gse_packages = [] + for i in range(start_id, number + start_id): + gse_package = GsePackages( + id=i, + pkg_name=f"pkg_name{i+1}", + version=version or random.choice(["version1", "version2", "version3"]), + project=project or random.choice(["agent", "proxy"]), + pkg_size=pkg_size or random.randint(1, 2000), + pkg_path=pkg_path or "/tmp/", + md5="", + location=location or "", + os=os or random.choice(constants.OS_TUPLE), + cpu_arch=cpu_arch or random.choice(constants.CPU_TUPLE), + is_ready=is_ready or random.choice(list(constants.GSE_PACKAGE_ENABLE_ALIAS_MAP.keys())), + created_by=created_by or "admin", + updated_by=created_by or "admin", + version_log="", + version_log_en="", + ) + gse_packages.append(gse_package) + gse_packages = GsePackages.objects.bulk_create(gse_packages) + return [gse_package.id for gse_package in gse_packages] diff --git a/apps/node_man/tools/gse_package.py b/apps/node_man/tools/gse_package.py new file mode 100644 index 000000000..15a96b4eb --- /dev/null +++ b/apps/node_man/tools/gse_package.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import hashlib +import os +import re +import tarfile +import time +from typing import Dict, List, Type + +from django.utils.translation import ugettext as _ + +from apps.backend.agent.artifact_builder import agent, proxy +from apps.backend.agent.artifact_builder.base import BaseArtifactBuilder +from apps.core.files.storage import get_storage +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.node_man import constants, exceptions, models +from apps.node_man.constants import CategoryType +from apps.node_man.models import GsePackageDesc, UploadPackage + + +class GsePackageTools: + @classmethod + def get_latest_upload_record(cls, file_name: str) -> models.UploadPackage: + """ + 获取最新的agent上传包记录 + :param file_name: agent包文件名 + """ + upload_package_obj: models.UploadPackage = ( + models.UploadPackage.objects.filter(file_name=file_name, module=TargetType.AGENT.value) + .order_by("-upload_time") + .first() + ) + if upload_package_obj is None: + raise exceptions.FileDoesNotExistError(_("找不到请求发布的文件，请确认后重试")) + + return upload_package_obj + + @classmethod + def distinguish_gse_package(cls, file_path: str) -> (str, Type[BaseArtifactBuilder]): + """ + 区分agent和proxy包 + :param file_path: agent包文件路径 + """ + storage = get_storage() + with storage.open(name=file_path) as fs: + with tarfile.open(fileobj=fs) as tf: + directory_members = tf.getmembers() + + for directory in directory_members: + if directory.name == "gse/server": + return constants.GsePackageCode.PROXY.value, proxy.ProxyArtifactBuilder + elif directory.name.startswith("gse/") and constants.AGENT_PATH_RE.match(directory.name[4:]): + return constants.GsePackageCode.AGENT.value, agent.AgentArtifactBuilder + + # 文件解析失败，将上传记录和包都干掉 + storage.delete(name=file_path) + UploadPackage.objects.filter(file_path=file_path).delete() + raise exceptions.GsePackageUploadError( + agent_name=os.path.basename(file_path), + error=_("该agent包无效，" "gse_proxy的gse目录中应该包含server文件夹，" "gse_agent的gse目录中应该包含agent_(os)_(cpu_arch)的文件夹"), + ) + + @classmethod + def generate_name_by_description(cls, description: str) -> str: + """ + 根据标签描述生成对应唯一的id + :param description: agent包标签描述 + """ + current_time: str = str(time.time()) + unique_string: str = description + current_time + return hashlib.md5(unique_string.encode("utf-8")).hexdigest() + + @classmethod + def create_agent_tags(cls, tag_descriptions, project): + """ + 根据agent包标签描述列表自动创建或返回已有的标签信息 + + :input + { + "project": "gse_agent", + "tag_descriptions": ["aaa", "bbb"] + } + + :return + [ + { + "name": "7188612c63753ec339500e72083fe8ac", + "description": "aaa" + }, + { + "name": "381b9dc36b32195acb53418b588bb99b", + "description": "bbb" + } + ] + + :params tag_descriptions: 标签描述列表 + :params project: gse_agent或gse_proxy + + """ + tags: List[Dict[str, str]] = [] + for tag_description in tag_descriptions: + gse_package_desc_obj, _ = GsePackageDesc.objects.get_or_create( + project=project, category=CategoryType.official + ) + + if tag_description in constants.BUILT_IN_TAG_DESCRIPTIONS + constants.BUILT_IN_TAG_NAMES: + # 内置标签，手动指定name和description + name: str = constants.TAG_DESCRIPTION__TAG_NAME[tag_description] + tag_description = constants.TAG_NAME__TAG_DESCRIPTION.get(tag_description, tag_description) + else: + # 自定义标签，自动生成name + name: str = GsePackageTools.generate_name_by_description(tag_description) + + tag_queryset = Tag.objects.filter( + description=tag_description, + target_id=gse_package_desc_obj.id, + target_type=TargetType.AGENT.value, + ) + + # 如果已存在标签，直接返回已存在的标签，否则创建一个新的标签 + if tag_queryset.exists(): + tag_obj: Tag = tag_queryset.first() + else: + tag_obj, _ = Tag.objects.update_or_create( + defaults={"description": tag_description}, + name=name, + target_id=gse_package_desc_obj.id, + target_type=TargetType.AGENT.value, + ) + + tags.append({"name": tag_obj.name, "description": tag_obj.description}) + + return tags + + @staticmethod + def extract_numbers(s): + """从字符串中提取所有的数字，并返回它们的整数列表""" + numbers = re.findall(r"\d+", s) + return [int(num) for num in numbers] + + @staticmethod + def compare_version(a, b): + return GsePackageTools.extract_numbers(a) > GsePackageTools.extract_numbers(b) + + @classmethod + def match_criteria(cls, pkg_version_info, validated_data, filter_keys): + for key in filter_keys: + if key == "os" and validated_data["os"] not in pkg_version_info["os_choices"]: + return False + elif key == "cpu_arch" and validated_data["cpu_arch"] not in pkg_version_info["cpu_arch_choices"]: + return False + return True diff --git a/apps/node_man/tools/package.py b/apps/node_man/tools/package.py new file mode 100644 index 000000000..0d8c6f6c7 --- /dev/null +++ b/apps/node_man/tools/package.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import json +import os +from typing import Any, Dict + +import requests +from django.conf import settings +from django.core.files.uploadedfile import InMemoryUploadedFile + +from apps.core.files import core_files_constants +from apps.core.files.storage import get_storage +from apps.node_man import exceptions +from apps.utils.files import md5sum +from apps.utils.local import get_request_username +from common.api import NodeApi + + +class PackageTools: + @staticmethod + def upload(package_file: InMemoryUploadedFile, module: str) -> Dict[str, Any]: + """ + 将文件上传至 + :param package_file: InMemoryUploadedFile + :param module: 所属模块 + :return: + { + "result": True, + "message": "", + "code": "00", + "data": { + "id": record.id, # 上传文件记录ID + "name": record.file_name, # 包名 + "pkg_size": record.file_size, # 大小， + } + } + """ + with package_file.open("rb") as tf: + + # 计算上传文件的md5 + md5 = md5sum(file_obj=tf, closed=False) + + base_params = {"module": module, "md5": md5} + + # 如果采用对象存储，文件直接上传至仓库，并将返回的目标路径传到后台，由后台进行校验并创建上传记录 + # TODO 后续应该由前端上传文件并提供md5 + if settings.STORAGE_TYPE in core_files_constants.StorageType.list_cos_member_values(): + storage = get_storage() + + try: + storage_path = storage.save(name=os.path.join(settings.UPLOAD_PATH, tf.name), content=tf) + except Exception as e: + raise exceptions.PluginUploadError(agent_name=tf.name, error=e) + + return NodeApi.upload( + { + **base_params, + # 最初文件上传的名称，后台会使用该文件名保存并覆盖同名文件 + "file_name": tf.name, + "file_path": storage_path, + "download_url": storage.url(storage_path), + } + ) + + else: + + response = requests.post( + url=settings.DEFAULT_FILE_UPLOAD_API, + data={ + **base_params, + "bk_app_code": settings.APP_CODE, + "bk_username": get_request_username(), + }, + # 本地文件系统仍通过上传文件到Nginx并回调后台 + files={"package_file": tf}, + ) + + return json.loads(response.content) diff --git a/apps/node_man/urls.py b/apps/node_man/urls.py index faa90fc48..d2f0724e2 100644 --- a/apps/node_man/urls.py +++ b/apps/node_man/urls.py @@ -40,6 +40,9 @@ ) from apps.node_man.views.healthz import HealthzViewSet from apps.node_man.views.host_v2 import HostV2ViewSet +from apps.node_man.views.package_manage import ( # AgentPackageDescViewSet, + PackageManageViewSet, +) from apps.node_man.views.plugin import GsePluginViewSet from apps.node_man.views.plugin_v2 import PluginV2ViewSet from apps.node_man.views.sync_task import SyncTaskViewSet @@ -67,6 +70,8 @@ router.register(r"v2/plugin", PluginV2ViewSet, basename="plugin_v2") router.register(r"healthz", HealthzViewSet, basename="healthz") router.register(r"sync_task", SyncTaskViewSet, basename="sync_task") +router.register(r"agent/package", PackageManageViewSet, basename="package_manage") +# router.register(r"agent/package_desc", AgentPackageDescViewSet, basename="package_desc") biz_dispatcher = DjangoBasicResourceApiDispatcher(iam, settings.BK_IAM_SYSTEM_ID) biz_dispatcher.register("biz", BusinessResourceProvider()) diff --git a/apps/node_man/views/meta.py b/apps/node_man/views/meta.py index 0a9f0c75b..3736d5bbf 100644 --- a/apps/node_man/views/meta.py +++ b/apps/node_man/views/meta.py @@ -29,6 +29,7 @@ class MetaViews(APIViewSet): @swagger_auto_schema( operation_summary="获取过滤条件", + query_serializer=FilterConditionSerializer, tags=META_VIEW_TAGS, methods=["GET", "POST"], ) diff --git a/apps/node_man/views/package_manage.py b/apps/node_man/views/package_manage.py new file mode 100644 index 000000000..bd70240f6 --- /dev/null +++ b/apps/node_man/views/package_manage.py @@ -0,0 +1,828 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from collections import defaultdict +from typing import Any, Dict, List + +import django_filters +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.db import transaction +from django.db.models import Min, Q, QuerySet +from django.http import JsonResponse +from django.utils.translation import get_language +from django.utils.translation import ugettext_lazy as _ +from django_filters.rest_framework import DjangoFilterBackend, FilterSet +from drf_yasg import openapi +from rest_framework import filters +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.status import HTTP_200_OK + +from apps.backend.sync_task.constants import SyncTaskType +from apps.core.files.storage import get_storage +from apps.core.ipchooser.tools.base import HostQuerySqlHelper +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.exceptions import ValidationError +from apps.generic import ApiMixinModelViewSet as ModelViewSet +from apps.generic import ValidationMixin +from apps.node_man import constants, exceptions, models +from apps.node_man.constants import STABLE_DESCRIPTION +from apps.node_man.handlers.gse_package import GsePackageHandler, gse_package_handler +from apps.node_man.models import GsePackageDesc, GsePackages, UploadPackage +from apps.node_man.permissions import package_manage as pkg_permission +from apps.node_man.serializers import package_manage as pkg_manage +from apps.node_man.tools.gse_package import GsePackageTools +from apps.node_man.tools.package import PackageTools +from common.api import NodeApi +from common.utils.drf_utils import swagger_auto_schema + +PACKAGE_MANAGE_VIEW_TAGS = ["PKG_Manager"] +PACKAGE_DES_VIEW_TAGS = ["PKG_Desc"] +logger = logging.getLogger("app") + + +class PackageManageOrderingFilterSet(filters.OrderingFilter): + def get_ordering(self, request, queryset, view): + # 这里由原来的request.params变为了request.data，其余不变 + params = request.data.get(self.ordering_param) + if params: + fields = [param.strip() for param in params.split(",")] + ordering = self.remove_invalid_fields(queryset, fields, view, request) + if ordering: + return ordering + + return self.get_default_ordering(view) + + def filter_queryset(self, request, queryset, view): + ordering = self.get_ordering(request, queryset, view) + if not ordering: + return queryset + + for field in ordering[::-1]: + reverse = field.startswith("-") + if field.lstrip("-") == "version": + # 版本按这样排 V2.1.6-beta.10 -> [2, 1, 5, 10] + queryset: List[GsePackages] = sorted( + queryset, key=lambda obj: GsePackageTools.extract_numbers(obj.version), reverse=reverse + ) + else: + queryset: List[GsePackages] = sorted( + queryset, key=lambda obj: getattr(obj, field.lstrip("-")), reverse=reverse + ) + + return queryset + + +class PackageManageFilterClass(FilterSet): + os = django_filters.BaseInFilter(field_name="os", lookup_expr="in") + cpu_arch = django_filters.BaseInFilter(field_name="cpu_arch", lookup_expr="in") + os_cpu_arch = django_filters.BaseInFilter(field_name="os_cpu_arch", method="filter_os_cpu_arch") + tag_names = django_filters.BaseInFilter(lookup_expr="in", method="filter_tag_names") + created_by = django_filters.BaseInFilter(field_name="created_by", lookup_expr="in") + is_ready = django_filters.BooleanFilter(field_name="is_ready") + version = django_filters.BaseInFilter(field_name="version", lookup_expr="in") + created_time = django_filters.DateTimeFromToRangeFilter() + condition = django_filters.Filter(method="filter_condition") + + def filter_tag_names(self, queryset, name, tag_names): + if "project" not in self.request.data: + raise ValidationError(_("筛选tag_names时必须传入project")) + return gse_package_handler.filter_tags(queryset, self.request.data["project"], tag_names=tag_names) + + def filter_os_cpu_arch(self, queryset, name, os_cpu_archs): + package_query = Q() + for os_cpu_arch in os_cpu_archs: + try: + os, cpu_arch = os_cpu_arch.split("_", 1) + except ValueError: + raise ValidationError(_("筛选格式应该为{os}_{cpu_arch}")) + + package_query |= Q(os=os, cpu_arch=cpu_arch) + + return queryset.filter(package_query) + + def filter_condition(self, queryset, name, query_list): + if not isinstance(query_list, list): + return queryset + + fields_to_search = ["os", "cpu_arch", "created_by", "is_ready", "version"] + + model_field_query, tag_query = Q(), Q() + tag_names: List[str] = [] + for query_info in query_list: + if not isinstance(query_info, dict) or query_info.get("key") != "query" or "value" not in query_info: + continue + + tag_names.append(query_info["value"]) + + for field in fields_to_search: + model_field_query |= Q(**{f"{field}__icontains": query_info["value"]}) + + if "project" in self.request.data and tag_names: + tag_query = Q( + id__in=gse_package_handler.filter_tags( + queryset, self.request.data["project"], tag_names=tag_names + ).values_list("id", flat=True) + ) + + return queryset.filter(model_field_query | tag_query) + + class Meta: + model = GsePackages + fields = ["tag_names", "project", "created_by", "is_ready", "version", "os", "cpu_arch"] + + +class PackageManageFilterBackend(DjangoFilterBackend): + def get_filterset_kwargs(self, request, queryset, view): + return { + "data": {**request.data, **dict(request.query_params.items())}, + "queryset": queryset, + "request": request, + } + + +class PackageManageViewSet(ValidationMixin, ModelViewSet): + serializer_class = pkg_manage.PackageSerializer + permission_classes = (pkg_permission.PackageManagePermission,) + filter_backends = (PackageManageFilterBackend, filters.SearchFilter, PackageManageOrderingFilterSet) + filter_class = PackageManageFilterClass + ordering_fields = ["version", "created_time"] + + def get_queryset(self): + if self.action not in ["search", "quick_search_condition"]: # noqa + self.filter_class = None + return models.GsePackages.objects.all().order_by("-is_ready") + + @swagger_auto_schema( + responses={200: pkg_manage.ListResponseSerializer}, + operation_summary="安装包列表", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["POST"]) + def search(self, request, *args, **kwargs): + """ + return: { + "total": 2, + "list": [ + { + "id": 1, + "pkg_name": "pkg_name", + "version": "1.1.1", + "os": "Linux", + "cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + }, + { + "id": 2, + "pkg_name": "pkg_name", + "version": "1.1.2", + "os": "Linux", + "os_cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + }, + ], + } + """ + return super().list(request, *args, **kwargs) + + def perform_update(self, serializer): + serializer.save() + + if "tags" not in serializer.validated_data: + return + + package_obj: GsePackages = self.get_object() + tags: QuerySet = gse_package_handler.get_tag_objs(package_obj.project, package_obj.version) + with transaction.atomic(): + tags.filter(name__in=["latest", "test"]).update(target_version="") + tags.exclude(name__in=["latest", "test", "stable"]).delete() + + package_desc_obj: GsePackageDesc = GsePackageDesc.objects.get(project=package_obj.project) + for tag_description in serializer.validated_data["tags"]: + GsePackageHandler.handle_add_tag( + tag_description=tag_description, + package_obj=package_obj, + package_desc_obj=package_desc_obj, + ) + + @swagger_auto_schema( + operation_summary="操作类动作：启用/停用/修改(新增, 删除)标签", + body_in=pkg_manage.OperateSerializer, + responses={200: pkg_manage.PackageSerializer}, + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + def update(self, request, validated_data, *args, **kwargs): + """ + return: { + "id": 1, + "pkg_name": "pkg_name", + "version": "1.1.1", + "os": "Linux", + "cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + } + """ + instance: GsePackages = self.get_object() + serializer = pkg_manage.OperateSerializer(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) + + updated_instance: GsePackages = self.get_object() + + return Response(pkg_manage.PackageSerializer(updated_instance).data) + + @swagger_auto_schema( + operation_summary="删除安装包", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + def destroy(self, request, *args, **kwargs): + gse_package_obj: GsePackages = self.get_object() + + # 如果最后一个版本的包被清除了，将标签的target_version置空，防止下次上传这个版本的包时留下以前的标签 + if GsePackages.objects.filter(version=gse_package_obj.version).count() == 1: + Tag.objects.filter(target_version=gse_package_obj.version).update(target_version=None) + + return super().destroy(request, *args, **kwargs) + + @swagger_auto_schema( + operation_summary="获取快速筛选信息", + manual_parameters=[ + openapi.Parameter( + "project", in_=openapi.TYPE_STRING, description="区分gse_agent, gse_proxy", type=openapi.TYPE_STRING + ) + ], + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["GET"]) + def quick_search_condition(self, request, *args, **kwargs): + """ + return: [ + { + "name": "操作系统/架构", + "id": "os_cpu_arch", + "children": [ + {"name": "Linux_x86_64", "id": "linux_x86_64", "count": 10}, + {"name": "Linux_x86", "id": "linux_x86", "count": 10}, + {"name": "ALL", "id": "ALL", "count": 20}, + ] + }, + { + "name": "版本号", + "id": "version", + "children": [ + {"name": "2.1.8", "id": "2.1.8", "count": 10}, + {"name": "2.1.7", "id": "2.1.7", "count": 10}, + {"name": "ALL", "id": "ALL", "count": 20}, + ] + }, + ] + """ + gse_packages = self.filter_queryset(self.get_queryset()).values("version", "os", "cpu_arch", "version_log") + + version__count_map: Dict[str, int] = defaultdict(int) + os_cpu_arch__count_map: Dict[str, int] = defaultdict(int) + + for package in gse_packages.values("version", "os", "cpu_arch", "version_log"): + version, os_cpu_arch = package["version"], f"{package['os']}_{package['cpu_arch']}" + + version__count_map[version] += 1 + os_cpu_arch__count_map[os_cpu_arch] += 1 + + return Response( + [ + { + "name": _("操作系统/架构"), + "id": "os_cpu_arch", + "children": [ + { + "id": os_cpu_arch, + "name": os_cpu_arch.capitalize(), + "count": count, + } + for os_cpu_arch, count in os_cpu_arch__count_map.items() + ], + "count": sum(os_cpu_arch__count_map.values()), + }, + { + "name": _("版本号"), + "id": "version", + "children": [ + { + "id": version, + "name": version.capitalize(), + "count": version__count_map[version], + } + # 版本按这样排 V2.1.6-beta.10 -> [2, 1, 5, 10] + for version in sorted(version__count_map, reverse=True, key=GsePackageTools.extract_numbers) + ], + "count": sum(version__count_map.values()), + }, + ] + ) + + @swagger_auto_schema( + operation_summary="Agent包上传", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.UploadResponseSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.UploadSerializer) + def upload(self, request): + """ + return: { + "id": 116, + "name": "HR7vt0c_gse_ce-v2.1.3-beta.13.tgz", + "pkg_size": "336252435" + } + """ + request_serializer = self.serializer_class(data=request.data) + request_serializer.is_valid(raise_exception=True) + validated_data = request_serializer.validated_data + + if validated_data.get("overload"): + storage = get_storage() + package_file: InMemoryUploadedFile = validated_data["package_file"] + + # 只选择最新上传的记录 + # 使用contains是因为上传之后得到的文件名前面带有前缀 + # gse_ce-v2.1.3-beta.13.tgz -> HR7vt0c_gse_ce-v2.1.3-beta.13.tgz + upload_package: UploadPackage = UploadPackage.objects.filter( + file_name__contains=package_file.name, module=TargetType.AGENT.value + ).first() + + # 如果需要覆盖，且数据库找得到该记录并且storage存在的情况下，将记录和相对应的包清掉 + if upload_package and storage.exists(name=upload_package.file_path): + storage.delete(name=upload_package.file_path) + upload_package.delete() + + res = PackageTools.upload(package_file=validated_data["package_file"], module=TargetType.AGENT.value) + + if "result" in res: + return JsonResponse(res) + else: + return Response(res) + + @swagger_auto_schema( + operation_summary="解析Agent包", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.ParseResponseSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.ParseSerializer) + def parse(self, request): + """ + return: { + "description": "test", + "packages": [ + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86_64.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86_64", + }, + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86", + }, + ], + } + """ + return Response(NodeApi.agent_parse(self.validated_data)) + + @swagger_auto_schema( + operation_summary="创建Agent包注册任务", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.AgentRegisterTaskSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.AgentRegisterSerializer) + def create_register_task(self, request): + """ + return: {"task_id": 1} + """ + validated_data = self.validated_data + + extra_tag_names: List[str] = [ + tag_info["name"] + for tag_info in GsePackageTools.create_agent_tags( + tag_descriptions=validated_data["tag_descriptions"], + project=validated_data["project"], + ) + ] + response = NodeApi.sync_task_create( + { + "task_name": SyncTaskType.REGISTER_GSE_PACKAGE.value, + "task_params": { + "file_name": validated_data["file_name"], + "tags": validated_data["tags"] + extra_tag_names, + }, + } + ) + + return Response({"task_id": response["task_id"]}) + + @swagger_auto_schema( + operation_summary="查询Agent包注册任务", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.AgentRegisterTaskResponseSerializer}, + ) + @action(detail=False, methods=["GET"], serializer_class=pkg_manage.AgentRegisterTaskSerializer) + def query_register_task(self, request): + """ + return: { + "is_finish": True, + "status": "SUCCESS", + "message": "", + } + """ + + validated_data = self.validated_data + + task_result = NodeApi.sync_task_status({"task_id": validated_data["task_id"]}) + + # 在celery任务中无法获取正确的用户名，当上传成功时，更新用户名 + if task_result["status"] == "SUCCESS": + GsePackages.objects.filter(version=validated_data["version"]).update(created_by=request.user.username) + + return Response(task_result) + + @swagger_auto_schema( + operation_summary="获取Agent包标签", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["GET"], serializer_class=pkg_manage.TagProjectSerializer) + def tags(self, request): + """ + return: [ + { + "name": "builtin", + "description": "内置标签", + "children": [ + { + "id": 95, + "name": "stable", + "description": "稳定版本" + }, + { + "id": 96, + "name": "latest", + "description": "最新版本" + }, + { + "id": 97, + "name": "test", + "description": "测试版本" + } + ] + }, + { + "name": "custom", + "description": "自定义标签", + "children": [ + { + "id": 145, + "name": "custom3", + "description": "自定义标签3" + }, + { + "id": 146, + "name": "custom4", + "description": "自定义标签4" + }, + { + "id": 147, + "name": "custom5", + "description": "自定义标签5" + } + ] + } + ] + """ + validated_data = self.validated_data + try: + return Response( + gse_package_handler.handle_tags( + tags=list( + Tag.objects.filter(target_id=GsePackageDesc.objects.get(project=validated_data["project"]).id) + .values("name") + .distinct() + .annotate( + id=Min("id"), + description=Min("description"), + ) + ), + tag_description=request.query_params.get("tag_description"), + enable_tag_separation=True, + ) + ) + except GsePackageDesc.DoesNotExist: + raise exceptions.ModelInstanceNotFoundError(model_name="GsePackageDesc") + + @swagger_auto_schema( + operation_summary="获取Agent包版本", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action( + detail=False, + methods=["POST"], + serializer_class=pkg_manage.VersionQuerySerializer, + permission_classes=[IsAuthenticated], + ) + def version(self, request): + """ + return: { + "total": 10, + "list": [ + { + "version": "2.1.2", + "tags": [{"id": "stable", "name": "稳定版本"}], + "is_ready": True, + "description": "", + "packages": [ + { + "pkg_name": "gseagent-2.1.2.tgz", + "tags": [{"id": "stable", "name": "稳定版本1"}, {"id": "latest", "name": "最新版本"}], + }, + { + "pkg_name": "gseagent-2.1.2.tgz", + "tags": [{"id": "stable", "name": "稳定版本2"}, {"id": "latest", "name": "最新版本"}], + } + ], + } + ], + } + """ + validated_data = self.validated_data + + gse_packages: QuerySet = ( + self.get_queryset() + .filter(project=validated_data["project"], is_ready=True) + .values("version", "project", "pkg_name", "os", "cpu_arch", "version_log", "version_log_en") + ) + + language = get_language() + + version__pkg_info_map: Dict[str, Dict[str, Any]] = {} + max_version_count: int = 0 + default_version: str = "" + for package in gse_packages: + version, project, pkg_name = package["version"], package["project"], package["pkg_name"] + tags: List[Dict[str, Any]] = gse_package_handler.get_tags( + version=version, + project=project, + enable_tag_separation=False, + ) + + # 获取默认标签 + if not default_version and any(tag["description"] == STABLE_DESCRIPTION for tag in tags): + default_version = version + + # 初始化某个版本的包 + if version not in version__pkg_info_map: + version__pkg_info_map[version] = { + "version": version, + "project": project, + "packages": [], + "tags": tags, + "description": package["version_log"] if language == "zh-hans" else package["version_log_en"], + "count": 0, + "os_choices": set(), + "cpu_arch_choices": set(), + } + + # 累加同个版本包的数量，并统计版本包最大数量 + version__pkg_info_map[version]["count"] += 1 + max_version_count = max(max_version_count, version__pkg_info_map[version]["count"]) + + # 聚合操作系统和cpu架构信息 + version__pkg_info_map[version]["os_choices"].add(package["os"]) + version__pkg_info_map[version]["cpu_arch_choices"].add(package["cpu_arch"]) + + # 添加小包包名和小包标签信息 + version__pkg_info_map[version]["packages"].append( + {"pkg_name": pkg_name, "tags": tags, "os": package["os"], "cpu_arch": package["cpu_arch"]} + ) + + # 将上一次的标签和这次的标签取共同的部分 + last_tags: List[Dict[str, Any]] = version__pkg_info_map[version]["tags"] + if last_tags != tags: + version__pkg_info_map[version]["tags"] = [tag for tag in last_tags if tag in tags] + + # 按版本排序 + version__pkg_info_map = dict( + sorted( + version__pkg_info_map.items(), + key=lambda version__pkg_info_tuple: GsePackageTools.extract_numbers(version__pkg_info_tuple[0]), + reverse=True, + ) + ) + + filter_keys = [key for key in ["os", "cpu_arch"] if validated_data.get(key)] + if filter_keys: + # 筛选 + version__pkg_version_info_map = { + version: pkg_version_info + for version, pkg_version_info in version__pkg_info_map.copy().items() + if GsePackageTools.match_criteria(pkg_version_info, validated_data, filter_keys) + } + else: + # 不筛选，默认为统一版本，统一版本需要各个系统的包都齐了才能算入 + # 如果count数量不等于包版本最大数量，则说明缺少某些系统的包 + version__pkg_version_info_map = { + version: pkg_version_info + for version, pkg_version_info in version__pkg_info_map.copy().items() + if pkg_version_info["count"] == max_version_count + } + + machine_latest_version: str = "" + if validated_data.get("versions", ""): + machine_latest_version = max(validated_data["versions"], key=GsePackageTools.extract_numbers) + package_latest_version = list(version__pkg_version_info_map.keys())[0] if version__pkg_version_info_map else "" + + return Response( + { + "machine_latest_version": machine_latest_version, + "package_latest_version": package_latest_version, + "default_version": default_version, + "pkg_info": list(version__pkg_version_info_map.values()), + "versions_count": len(validated_data["versions"]) if validated_data.get("versions") else 0, + } + ) + + @swagger_auto_schema( + operation_summary="Agent包版本比较", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action( + detail=False, + methods=["POST"], + serializer_class=pkg_manage.VersionCompareSerializer, + permission_classes=[IsAuthenticated], + ) + def version_compare(self, request): + validated_data = self.validated_data + + extracted_current_version: List[int] = GsePackageTools.extract_numbers(validated_data["current_version"]) + version_to_compares: List[str] = validated_data["version_to_compares"] + + upgrade_count, downgrade_count, no_change_count = 0, 0, 0 + + for version_to_compare in version_to_compares: + if GsePackageTools.extract_numbers(version_to_compare) > extracted_current_version: + downgrade_count += 1 + elif GsePackageTools.extract_numbers(version_to_compare) < extracted_current_version: + upgrade_count += 1 + else: + no_change_count += 1 + + return Response( + { + "upgrade_count": upgrade_count, + "downgrade_count": downgrade_count, + "no_change_count": no_change_count, + } + ) + + @swagger_auto_schema( + operation_summary="获取已部署主机数量", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.DeployedAgentCountSerializer) + def deployed_hosts_count(self, request): + """ + input: { + "items": [ + { + "os_type": "linux", + "version": "3.6.21" + }, + { + "os_type": "windows", + "version": "3.6.21" + }, + { + "os_type": "windows", + "version": "3.6.22" + } + ], + "project": "gse_agent" + } + + return: [ + { + "os_type": "linux", + "version": "3.6.21", + "count": 3 + }, + { + "os_type": "windows", + "version": "3.6.21", + "count": 2 + }, + { + "os_type": "windows", + "version": "3.6.22", + "count": 0 + } + ] + """ + validated_data = self.validated_data + + items = validated_data["items"] + project = validated_data["project"] + if not items: + return Response() + + # 划分维度到主机和进程 + dimensions: List[str] = list(items[0].keys()) + host_dimensions: List[str] = [d for d in dimensions if d in [field.name for field in models.Host._meta.fields]] + process_dimensions: List[str] = list(set(dimensions) - set(host_dimensions)) + + # 主机筛选条件 + host_kwargs: Dict[str, list] = { + f"{dimension}__in": [item[dimension] for item in items] for dimension in host_dimensions + } + + # 进程筛选条件 + process_params: Dict[str, list] = { + "conditions": [{"key": "status", "value": [constants.ProcStateType.RUNNING]}] + } + for dimension in process_dimensions: + process_params["conditions"].append({"key": dimension, "value": [item[dimension] for item in items]}) + + # 主机和进程连表查询 + host_queryset: QuerySet = HostQuerySqlHelper.multiple_cond_sql( + params=process_params, + biz_scope=[], + need_biz_scope=False, + is_proxy=False if project == constants.GsePackageCode.AGENT.value else True, + ).filter(**host_kwargs) + + # 分组统计数量 + dimension__count_map: Dict[str, int] = defaultdict(int) + for host in host_queryset.values(*dimensions): + dimension__count_map["|".join(host.get(d, "").lower() for d in dimensions)] += 1 + + # 填充count到item + for item in items: + item["count"] = dimension__count_map.get("|".join(item.get(d, "").lower() for d in dimensions), 0) + + return Response(items) + + @swagger_auto_schema( + operation_summary="批量编辑agent标签", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.TagCreateSerializer) + def create_agent_tags(self, request): + """ + input: { + "project": "gse_agent", + "tag_descriptions": ["stable", "恭喜", "发财"] + } + + return: [ + { + "name": "stable", + "description": "稳定版本" + }, + { + "name": "43f5242cbf2181dc8818a9b8c1c48da6", + "description": "恭喜" + }, + { + "name": "7d602b6a7e590b232c9c5d1f871601a4", + "description": "发财" + } + ] + """ + validated_data = self.validated_data + + return Response( + data=GsePackageTools.create_agent_tags( + tag_descriptions=validated_data["tag_descriptions"], + project=validated_data["project"], + ) + ) diff --git a/apps/node_man/views/plugin_v2.py b/apps/node_man/views/plugin_v2.py index 7df98ba81..78fb1a8bb 100644 --- a/apps/node_man/views/plugin_v2.py +++ b/apps/node_man/views/plugin_v2.py @@ -23,6 +23,7 @@ from apps.node_man.handlers.plugin_v2 import PluginV2Handler from apps.node_man.models import GsePluginDesc from apps.node_man.serializers import plugin_v2 +from apps.node_man.tools.package import PackageTools from apps.utils.local import get_request_username from common.api import NodeApi @@ -466,7 +467,7 @@ def upload(self, request): ser = self.serializer_class(data=request.data) ser.is_valid(raise_exception=True) data = ser.validated_data - result = PluginV2Handler.upload(package_file=data["package_file"], module=data["module"]) + result = PackageTools.upload(package_file=data["package_file"], module=data["module"]) if "result" in result: return JsonResponse(result) else: diff --git a/common/api/modules/bk_node.py b/common/api/modules/bk_node.py index f0f390b43..9e095833e 100644 --- a/common/api/modules/bk_node.py +++ b/common/api/modules/bk_node.py @@ -329,3 +329,12 @@ def __init__(self): before_request=add_esb_info_before_request, api_name="job_details", ) + self.agent_parse = DataAPI( + method="POST", + url=BK_NODE_APIGATEWAY_ROOT + "backend/api/agent/parse/", + module=self.MODULE, + simple_module=self.SIMPLE_MODULE, + description="解析agent包", + before_request=add_esb_info_before_request, + api_name="agent_parse", + ) diff --git a/common/utils/drf_utils.py b/common/utils/drf_utils.py new file mode 100644 index 000000000..bfb568a01 --- /dev/null +++ b/common/utils/drf_utils.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import copy +import functools +from collections import namedtuple +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union + +from django.conf import settings +from django.http.response import HttpResponseBase +from django.utils.module_loading import import_string +from rest_framework import status +from rest_framework.exceptions import ValidationError +from rest_framework.fields import empty +from rest_framework.serializers import BaseSerializer +from rest_framework.settings import api_settings +from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList + +if TYPE_CHECKING: + from rest_framework.request import Request + + +def stringify_validation_error(error: ValidationError) -> List[str]: + """Transform DRF's ValidationError into a list of error strings + + >>> stringify_validation_error(ValidationError({'foo': ErrorDetail('err')})) + ['foo: err'] + """ + results: List[str] = [] + + def traverse(err_detail: Any, keys: List[str]): + """Traverse error data to collect all error messages""" + + # Dig deeper when structure is list or dict + if isinstance(err_detail, (ReturnList, list, tuple)): + for err in err_detail: + traverse(err, keys) + elif isinstance(err_detail, (ReturnDict, dict)): + for key, err in err_detail.items(): + # Make a copy of keys so the inner loop won't affect outer scope + _keys = copy.copy(keys) + if key != api_settings.NON_FIELD_ERRORS_KEY: + _keys.append(str(key)) + traverse(err, _keys) + else: + if not keys: + results.append(str(err_detail)) + else: + results.append("{}: {}".format(".".join(keys), str(err_detail))) + + traverse(error.detail, []) + return sorted(results) + + +############# +# drf crown # +############# +class WearOptions: + is_unittest = False + skip_swagger_schema = False + + +try: + from drf_yasg.utils import swagger_auto_schema as drf_swagger_auto_schema + +except ImportError: + WearOptions.skip_swagger_schema = True + + +ResponseParams = namedtuple("ResponseParams", "data,params") + + +_DEFAULT_SETTINGS_PREFIX = "DRF_CROWN_" + + +def enable_unittest(): + """Call me when you running testing""" + WearOptions.is_unittest = True + + +@dataclass +class Config: + """Config for Injector, control the process of injecting""" + + return_validated_data: bool = True + remain_request: bool = False + # sometime return raw data instead of serializer + skip_out_cls: bool = False + default_return_status: status = status.HTTP_200_OK + + +@dataclass +class ViewCrown: + """A injector for injecting serializer as dependency""" + + body_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] + query_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] + out: Union[Type[BaseSerializer], BaseSerializer] + config_params: Optional[dict] = field(default_factory=dict) + valid_params: dict = field(default_factory=dict) + + def __post_init__(self): + if self.query_in and self.body_in: + raise ValueError("there should be only one param between in_body & in_query") + + self.valid_params = self.valid_params or {"raise_exception": True} + + # Priority decreases + # 1. config as parameter from decorator + # 2. config from django.settings + # 3. config from Config class(above) + _config = getattr(settings, _DEFAULT_SETTINGS_PREFIX + "DEFAULT_CONFIG", {}).copy() + _config.update(self.config_params or {}) + self.config = Config(**_config) + + # remain an entrance for custom response class + try: + self.resp_cls = import_string(getattr(settings, _DEFAULT_SETTINGS_PREFIX + "RESP_CLS")) + except AttributeError: + self.resp_cls = import_string("rest_framework.response.Response") + + def get_in_serializer_instance(self, request: Optional["Request"] = None) -> "BaseSerializer": + if not self.body_in and not self.query_in: + raise ValueError("should given at least one serializer input") + + _data = empty + if self.body_in: + _in = self.body_in + + if request is not None: + _data = getattr(request, "data") + else: + _in = self.query_in + + if request is not None: + _data = getattr(request, "query_params") + + if isinstance(_in, BaseSerializer): + # 由于传入的是全局对象，会残留上一次请求的结果 + # 这里需要手动清理一下 + if hasattr(_in, "_validated_data"): + delattr(_in, "_validated_data") + + _in.initial_data = _data + slz_obj = _in + elif issubclass(_in, BaseSerializer): + slz_obj = _in(data=_data) + else: + raise ValueError("unknown serializer input") + + return slz_obj + + def get_serializer_instance_by_request(self, request: "Request") -> "BaseSerializer": + """Get in serializer instance""" + slz_obj = self.get_in_serializer_instance(request) + slz_obj.is_valid(**self.valid_params) + return slz_obj + + def get_validated_data(self, request: "Request") -> dict: + """Get validated data via in_serializer""" + return self.get_serializer_instance_by_request(request).validated_data + + def get_in_params(self, request: "Request") -> dict: + """Get extra params before view logic""" + if WearOptions.is_unittest: + return {} + + if self.config.return_validated_data: + return {"validated_data": self.get_validated_data(request)} + else: + return {"serializer_instance": self.get_serializer_instance_by_request(request)} + + def get_response(self, data, out_params: dict) -> Any: + """Get Response data""" + if WearOptions.is_unittest: + return data + + if self.config.skip_out_cls: + return data + + if isinstance(data, (self.resp_cls, HttpResponseBase)): + return data + + if isinstance(self.out, BaseSerializer): + # 由于传入的是全局对象，会残留上一次请求的结果 + # 这里需要手动清理一下 + if hasattr(self.out, "_data"): + delattr(self.out, "_data") + + self.out.instance = data + _data = self.out.data + elif issubclass(self.out, BaseSerializer): + _data = self.out(data, **out_params).data + else: + raise ValueError("unknown serializer output") + + return self.resp_cls(data=_data, status=self.config.default_return_status) + + +def generate_swagger_params(crown: ViewCrown, swagger_params: dict) -> dict: + """ + assemble params for swagger_auto_schema by crown + """ + default_params = {} + if crown.body_in: + default_params = {"request_body": crown.get_in_serializer_instance()} + elif crown.query_in: + default_params = {"query_serializer": crown.get_in_serializer_instance()} + + if crown.out: + default_params.update({"responses": {crown.config.default_return_status: crown.out}}) + + default_params.update(swagger_params or {}) + return default_params + + +def swagger_auto_schema( + body_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + query_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + out: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + config: Optional[dict] = None, + **swagger_kwargs +): + """ + Sugar for simpling drf serializer specification + :param body_in: input serializer (request body) + :param query_in: input serializer (query) + :param out: output serializer + :param config: initial info of Config + :param swagger_kwargs: pass to swagger_auto_schema of drf-yasg + """ + + def decorator_serializer_inject(func): + crown = ViewCrown(body_in, query_in, out, config) + + if not WearOptions.skip_swagger_schema: + func = drf_swagger_auto_schema(**generate_swagger_params(crown, swagger_kwargs))(func) + + @functools.wraps(func) + def decorated(*args, **kwargs): + new_args = list(args) + in_content: Dict[str, Any] = {} + if body_in or query_in: + in_content.update(**crown.get_in_params(new_args[1])) + + if not crown.config.remain_request: + del new_args[1] + + original_data = func(*new_args, **kwargs, **in_content) + if not out: + return original_data + + # support runtime serializer params, like "context" + params = {} + if isinstance(original_data, ResponseParams): + params = original_data.params + original_data = original_data.data + + return crown.get_response(original_data, params) + + return decorated + + return decorator_serializer_inject diff --git a/config/default.py b/config/default.py index 9634e0e0b..64918bc46 100644 --- a/config/default.py +++ b/config/default.py @@ -262,6 +262,7 @@ "apps.node_man.periodic_tasks", # 避免 subscription.tools 循环导入，故单独引入 "apps.node_man.periodic_tasks.add_biz_to_gse2_gray_scope", + "apps.node_man.periodic_tasks.register_gse_package", ) BK_NODEMAN_CELERY_RESULT_BACKEND_BROKER_URL = "amqp://{user}:{passwd}@{host}:{port}/{vhost}".format( @@ -830,6 +831,8 @@ def get_standard_redis_mode(cls, config_redis_mode: str, default: Optional[str] if env.BKAPP_MONITOR_REPORTER_ENABLE: monitor_report_config() +DRF_CROWN_DEFAULT_CONFIG = {"remain_request": True} + # remove disabled apps if locals().get("DISABLED_APPS"): INSTALLED_APPS = locals().get("INSTALLED_APPS", []) diff --git a/locale/en/LC_MESSAGES/django.mo b/locale/en/LC_MESSAGES/django.mo index 5f0987a6c..6245f31c0 100644 Binary files a/locale/en/LC_MESSAGES/django.mo and b/locale/en/LC_MESSAGES/django.mo differ diff --git a/locale/en/LC_MESSAGES/django.po b/locale/en/LC_MESSAGES/django.po index 892d49019..8f5d54210 100644 --- a/locale/en/LC_MESSAGES/django.po +++ b/locale/en/LC_MESSAGES/django.po @@ -1029,12 +1029,12 @@ msgstr "" #: apps/backend/components/collections/plugin.py:1503 #: apps/backend/constants.py:137 msgid "启用" -msgstr "Enable" +msgstr "Enabled" #: apps/backend/components/collections/plugin.py:1488 #: apps/backend/components/collections/plugin.py:1503 msgid "停用" -msgstr "Deactivate" +msgstr "Disabled" #: apps/backend/components/collections/plugin.py:1499 #, python-brace-format @@ -1765,6 +1765,9 @@ msgstr "" msgid "升级" msgstr "Upgrade" +msgid "回退" +msgstr "Downgrade" + #: apps/backend/subscription/steps/agent.py:351 #: apps/backend/subscription/steps/agent.py:384 #: apps/backend/subscription/steps/agent.py:426 @@ -2818,8 +2821,8 @@ msgid "升级 Proxy" msgstr "Upgrade Proxy" #: apps/node_man/constants.py:233 -msgid "升级 Agent" -msgstr "Upgrade Agent" +msgid "升级/回退 Agent" +msgstr "Upgrade/Downgrade Agent" #: apps/node_man/constants.py:234 msgid "移除 Agent" @@ -6643,6 +6646,30 @@ msgstr "Variable available or not" msgid "Variable变量" msgstr "Variable variable" +msgid "内置标签" +msgstr "built-in tag" + +msgid "自定义标签" +msgstr "custom tag" + +msgid "稳定版本" +msgstr "stable version" + +msgid "最新版本" +msgstr "latest version" + +msgid "测试版本" +msgstr "test version" + +msgid "操作系统/架构" +msgstr "System/Arch" + +msgid "标签信息" +msgstr "Tag information" + +msgid "上传用户" +msgstr "Upload user" + #~ msgid "用户名" #~ msgstr "Username"