From 43d5fa409c6718a8a649949cf9776727a80b3c7c Mon Sep 17 00:00:00 2001 From: wyyalt Date: Mon, 9 Oct 2023 10:13:13 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feature:=20=20=E6=8F=90=E4=BE=9B=20Agent=20?= =?UTF-8?q?=E5=8C=85=E7=AE=A1=E7=90=86=E5=90=8E=E5=8F=B0=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=20(closed=20#1683)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/agent/artifact_builder/base.py | 99 ++- apps/backend/agent/artifact_builder/proxy.py | 1 + apps/backend/agent/tools.py | 5 +- apps/backend/agent/views.py | 89 ++ .../components/collections/agent_new/base.py | 4 +- apps/backend/constants.py | 1 + apps/backend/exceptions.py | 6 + apps/backend/subscription/errors.py | 8 + apps/backend/subscription/steps/agent.py | 27 + .../steps/agent_adapter/adapter.py | 90 +- apps/backend/subscription/views.py | 8 +- apps/backend/sync_task/constants.py | 13 +- apps/backend/sync_task/handler.py | 19 + .../agent/artifact_builder/test_agent.py | 23 +- .../artifact_builder/test_manage_commands.py | 5 +- .../agent/artifact_builder/test_proxy.py | 1 + .../collections/agent_new/test_install.py | 13 +- .../agent_new/test_push_agent_pkg_to_proxy.py | 1 + .../agent_new/test_push_upgrade_package.py | 1 + .../agent_new/test_run_upgrade_command.py | 1 + .../components/collections/agent_new/utils.py | 15 +- .../agent_adapter/test_adapter.py | 28 +- .../backend/tests/view/test_get_gse_config.py | 14 +- apps/backend/urls.py | 2 + apps/core/ipchooser/tools/base.py | 106 ++- apps/core/tag/constants.py | 8 - apps/core/tag/exceptions.py | 5 + .../0004_alter_tag_unique_together.py | 17 + .../migrations/0005_create_initial_tags.py | 56 ++ .../0006_create_initial_gsepackages.py | 42 + apps/core/tag/models.py | 2 +- apps/core/tag/targets/agent.py | 40 +- apps/core/tag/targets/base.py | 4 +- apps/core/tag/tests/test_views.py | 2 +- apps/generic.py | 4 + apps/mock_data/common_unit/subscription.py | 2 +- apps/node_man/constants.py | 42 +- apps/node_man/exceptions.py | 38 + apps/node_man/handlers/gse_package.py | 260 ++++++ apps/node_man/handlers/host.py | 4 + apps/node_man/handlers/meta.py | 77 +- apps/node_man/handlers/plugin_v2.py | 67 -- .../0082_gsepackagedesc_gsepackages.py | 97 ++ .../migrations/0083_merge_20240911_1050.py | 13 + apps/node_man/models.py | 46 + .../periodic_tasks/register_gse_package.py | 30 + apps/node_man/permissions/__init__.py | 10 + apps/node_man/permissions/package_manage.py | 27 + apps/node_man/serializers/host.py | 1 + apps/node_man/serializers/job.py | 24 +- apps/node_man/serializers/meta.py | 1 + apps/node_man/serializers/package_manage.py | 193 ++++ .../test_views/test_package_manage_views.py | 471 ++++++++++ apps/node_man/tests/utils.py | 47 +- apps/node_man/tools/gse_package.py | 161 ++++ apps/node_man/tools/package.py | 86 ++ apps/node_man/urls.py | 5 + apps/node_man/views/meta.py | 1 + apps/node_man/views/package_manage.py | 828 ++++++++++++++++++ apps/node_man/views/plugin_v2.py | 3 +- common/api/modules/bk_node.py | 9 + common/utils/drf_utils.py | 271 ++++++ config/default.py | 3 + locale/en/LC_MESSAGES/django.mo | Bin 118269 -> 118763 bytes locale/en/LC_MESSAGES/django.po | 35 +- 65 files changed, 3407 insertions(+), 205 deletions(-) create mode 100644 apps/backend/agent/views.py create mode 100644 apps/core/tag/migrations/0004_alter_tag_unique_together.py create mode 100644 apps/core/tag/migrations/0005_create_initial_tags.py create mode 100644 apps/core/tag/migrations/0006_create_initial_gsepackages.py create mode 100644 apps/node_man/handlers/gse_package.py create mode 100644 apps/node_man/migrations/0082_gsepackagedesc_gsepackages.py create mode 100644 apps/node_man/migrations/0083_merge_20240911_1050.py create mode 100644 apps/node_man/periodic_tasks/register_gse_package.py create mode 100644 apps/node_man/permissions/__init__.py create mode 100644 apps/node_man/permissions/package_manage.py create mode 100644 apps/node_man/serializers/package_manage.py create mode 100644 apps/node_man/tests/test_views/test_package_manage_views.py create mode 100644 apps/node_man/tools/gse_package.py create mode 100644 apps/node_man/tools/package.py create mode 100644 apps/node_man/views/package_manage.py create mode 100644 common/utils/drf_utils.py diff --git a/apps/backend/agent/artifact_builder/base.py b/apps/backend/agent/artifact_builder/base.py index 8bcb8e8c7..c569e5eee 100644 --- a/apps/backend/agent/artifact_builder/base.py +++ b/apps/backend/agent/artifact_builder/base.py @@ -23,8 +23,9 @@ from apps.backend.agent.config_parser import GseConfigParser from apps.core.files import core_files_constants from apps.core.files.storage import get_storage -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.constants import TargetType from apps.core.tag.handlers import TagHandler +from apps.core.tag.targets import AgentTargetHelper from apps.node_man import constants, models from apps.utils import cache, files @@ -65,6 +66,7 @@ def __init__( overwrite_version: typing.Optional[str] = None, tags: typing.Optional[typing.List[str]] = None, enable_agent_pkg_manage: bool = False, + username: str = "", ): """ :param initial_artifact_path: 原始制品所在路径 @@ -84,6 +86,7 @@ def __init__( self.applied_tmp_dirs = set() # 文件源 self.storage = get_storage(file_overwrite=True) + self.username = username @staticmethod def download_file(file_path: str, target_path: str): @@ -428,26 +431,76 @@ def _get_version(self, extract_dir: str) -> str: raise exceptions.NotSemanticVersionError({"version": version_match}) @cache.class_member_cache() - def _get_changelog(self, extract_dir: str) -> str: + def _get_description(self, extract_dir: str) -> typing.Tuple[str, str]: """ 获取版本日志 :param extract_dir: 解压目录 :return: """ - changelog_file_path: str = os.path.join(extract_dir, "CHANGELOG.md") - if not os.path.exists(changelog_file_path): - raise exceptions.FileNotExistError(_("版本日志文件不存在")) - with open(changelog_file_path, "r", encoding="utf-8") as changelog_fs: - changelog: str = changelog_fs.read() - return changelog - - def update_or_create_record(self, artifact_meta_info: typing.Dict[str, typing.Any]): + description_file_path: str = os.path.join(extract_dir, "DESCRIPTION") + description_en_file_path: str = os.path.join(extract_dir, "DESCRIPTION_EN") + description, description_en = "", "" + if os.path.exists(description_file_path): + with open(description_file_path, "r", encoding="utf-8") as description_fs: + description: str = description_fs.read() + + if os.path.exists(description_en_file_path): + with open(description_en_file_path, "r", encoding="utf-8") as description_en_fs: + description_en: str = description_en_fs.read() + return description, description_en + + def generate_location_path(self, upload_path: str, pkg_name: str) -> str: + if settings.STORAGE_TYPE == core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value: + location_path: str = f"{settings.BKREPO_ENDPOINT_URL}/generic/blueking/bknodeman/{upload_path}/{pkg_name}" + else: + location_path: str = f"http://{settings.BKAPP_LAN_IP}/{upload_path}/{pkg_name}" + + return location_path + + def update_or_create_package_records(self, package_infos: typing.List[typing.Dict[str, typing.Any]]): """ - 创建或更新制品记录,待 Agent 包管理完善 - :param artifact_meta_info: + 创建或更新制品记录 + :param package_infos: :return: """ - pass + for package_info in package_infos: + models.GsePackages.objects.update_or_create( + defaults={ + "pkg_size": package_info["package_upload_info"]["pkg_size"], + "pkg_path": package_info["package_upload_info"]["pkg_path"], + "md5": package_info["package_upload_info"]["md5"], + "location": self.generate_location_path( + package_info["package_upload_info"]["pkg_path"], + package_info["package_upload_info"]["pkg_name"], + ), + "version_log": package_info["artifact_meta_info"]["description"], + "version_log_en": package_info["artifact_meta_info"]["description_en"], + }, + pkg_name=package_info["package_upload_info"]["pkg_name"], + version=package_info["artifact_meta_info"]["version"], + project=package_info["artifact_meta_info"]["name"], + os=package_info["package_dir_info"]["os"], + cpu_arch=package_info["package_dir_info"]["cpu_arch"], + ) + logger.info( + f"[update_or_create_package_record] " + f"package name -> {package_info['package_upload_info']['pkg_name']} success" + ) + + if package_infos: + models.GsePackageDesc.objects.update_or_create( + defaults={ + "description": package_infos[0]["artifact_meta_info"]["description"], + }, + project=package_infos[0]["artifact_meta_info"]["name"], + category=constants.CategoryType.official, + ) + + logger.info( + f"[update_or_create_package_record] " + f"package desc -> {package_info['package_upload_info']['pkg_name']}, " + f"project -> {package_infos[0]['artifact_meta_info']['name']} success" + ) def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]): """ @@ -455,11 +508,12 @@ def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]) :param artifact_meta_info: :return: """ + agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map() for tag in self.tags: TagHandler.publish_tag_version( name=tag, target_type=TargetType.AGENT.value, - target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME], + target_id=agent_name_target_id_map[self.NAME], target_version=artifact_meta_info["version"], ) logger.info( @@ -517,14 +571,6 @@ def update_or_create_support_files(self, package_infos: typing.List[typing.Dict] agent_name=self.NAME, ) - def update_or_create_package_records(self, v): - """ - 创建或更新安装包记录,待 Agent 包管理完善 - :param package_infos: - :return: - """ - pass - def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]: """ 获取制品的基础信息、配置文件信息 @@ -535,13 +581,14 @@ def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.An version_str: str = self._get_version(extract_dir) # 配置文件 support_files_info = self._get_support_files_info(extract_dir) - # changelog - changelog: str = self._get_changelog(extract_dir) + # description + description, description_en = self._get_description(extract_dir) return { "name": self.NAME, "version": version_str, - "changelog": changelog, + "description": description, + "description_en": description_en, "support_files_info": support_files_info, } @@ -591,8 +638,6 @@ def make( artifact_meta_info["operator"] = operator # Agent 包先导入文件源 -> 写配置文件 -> 创建包记录 -> 创建 Tag self.update_or_create_support_files(package_infos) - # TODO update_or_create_record & update_or_create_package_records 似乎是一样的功能? - self.update_or_create_record(artifact_meta_info) self.update_or_create_package_records(package_infos) self.update_or_create_tag(artifact_meta_info) diff --git a/apps/backend/agent/artifact_builder/proxy.py b/apps/backend/agent/artifact_builder/proxy.py index 5223ca675..3d9b94d39 100644 --- a/apps/backend/agent/artifact_builder/proxy.py +++ b/apps/backend/agent/artifact_builder/proxy.py @@ -36,6 +36,7 @@ class ProxyArtifactBuilder(base.BaseArtifactBuilder): PROXY_SVR_EXES: typing.List[str] = ["gse_data", "gse_file"] def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir: str): + # todo: 是否使用Archive(initial_artifact_local_path).extractall(extract_dir, auto_create_dir=True) with tarfile.open(name=initial_artifact_local_path) as tf: tf.extractall(path=extract_dir) diff --git a/apps/backend/agent/tools.py b/apps/backend/agent/tools.py index 6887b1776..a10a18447 100644 --- a/apps/backend/agent/tools.py +++ b/apps/backend/agent/tools.py @@ -334,7 +334,7 @@ def check_run_commands(run_commands): def batch_gen_commands( - base_agent_setup_info: AgentSetupInfo, + agent_step_adapter, hosts: List[models.Host], pipeline_id: str, is_uninstall: bool, @@ -350,7 +350,6 @@ def batch_gen_commands( # 批量查出主机的属性并设置为property,避免在循环中进行ORM查询,提高效率 host_id__installation_tool_map = {} bk_host_ids = [host.bk_host_id for host in hosts] - base_agent_setup_info_dict: Dict[str, Any] = asdict(base_agent_setup_info) host_id_identity_map = { identity.bk_host_id: identity for identity in models.IdentityData.objects.filter(bk_host_id__in=bk_host_ids) } @@ -368,7 +367,7 @@ def batch_gen_commands( host_id__installation_tool_map[host.bk_host_id] = gen_commands( agent_setup_info=AgentSetupInfo( **{ - **base_agent_setup_info_dict, + **asdict(agent_step_adapter.get_host_setup_info(host)), "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False), } ), diff --git a/apps/backend/agent/views.py b/apps/backend/agent/views.py new file mode 100644 index 000000000..a4a6196e9 --- /dev/null +++ b/apps/backend/agent/views.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import typing + +from django.utils.translation import get_language +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.status import HTTP_200_OK + +from apps.generic import ApiMixinModelViewSet as ModelViewSet +from apps.generic import ValidationMixin +from apps.node_man.serializers import package_manage as pkg_manage +from apps.node_man.tools.gse_package import GsePackageTools +from apps.node_man.views.package_manage import PACKAGE_MANAGE_VIEW_TAGS +from common.utils.drf_utils import swagger_auto_schema + + +class AgentViewSet(ModelViewSet, ValidationMixin): + """ + agent相关API + """ + + @swagger_auto_schema( + operation_summary="解析Agent包", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.ParseResponseSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.ParseSerializer) + def parse(self, request): + """ + return: { + "description": "test", + "packages": [ + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86_64.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86_64", + }, + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86", + }, + ], + } + """ + validated_data = self.validated_data + + # 获取最新的agent上传包记录 + upload_package_obj = GsePackageTools.get_latest_upload_record(file_name=validated_data["file_name"]) + + # 区分agent和proxy包 + project, artifact_builder_class = GsePackageTools.distinguish_gse_package( + file_path=upload_package_obj.file_path + ) + + # 解析包 + with artifact_builder_class(initial_artifact_path=upload_package_obj.file_path) as builder: + extract_dir, package_dir_infos = builder.list_package_dir_infos() + artifact_meta_info: typing.Dict[str, typing.Any] = builder.get_artifact_meta_info(extract_dir) + + language = get_language() + res = { + "description": artifact_meta_info["description"] + if language == "zh-hans" + else artifact_meta_info["description_en"], + "packages": package_dir_infos, + } + + context = { + "project": project, + "version": artifact_meta_info["version"], + } + + return Response(pkg_manage.ParseResponseSerializer(res, context=context).data) diff --git a/apps/backend/components/collections/agent_new/base.py b/apps/backend/components/collections/agent_new/base.py index 3c8fad0cd..9679ab59c 100644 --- a/apps/backend/components/collections/agent_new/base.py +++ b/apps/backend/components/collections/agent_new/base.py @@ -141,7 +141,7 @@ def get_agent_pkg_name( package_type = ("client", "proxy")[host.node_type == constants.NodeType.PROXY] agent_step_adapter = common_data.agent_step_adapter if not agent_step_adapter.is_legacy: - setup_info = agent_step_adapter.setup_info + setup_info = agent_step_adapter.get_host_setup_info(host) return f"{setup_info.name}-{setup_info.version}.tgz" # GSE1.0 的升级包是独立的,添加了 _upgrade 后缀 @@ -297,7 +297,7 @@ def get_host_id__installation_tool_map( host for host in hosts_need_gen_commands if host.bk_host_id in host_id__install_channel_map ] host_id__installation_tool_map = batch_gen_commands( - base_agent_setup_info=common_data.agent_step_adapter.setup_info, + agent_step_adapter=common_data.agent_step_adapter, hosts=hosts_need_gen_commands, pipeline_id=self.id, is_uninstall=is_uninstall, diff --git a/apps/backend/constants.py b/apps/backend/constants.py index e024154c6..68f7e98f4 100644 --- a/apps/backend/constants.py +++ b/apps/backend/constants.py @@ -64,6 +64,7 @@ class InstNodeType(object): "INSTALL_AGENT", "REINSTALL_AGENT", "UPGRADE_AGENT", + "DOWNGRADE_AGENT", "RESTART_AGENT", "UNINSTALL_AGENT", "RELOAD_AGENT", diff --git a/apps/backend/exceptions.py b/apps/backend/exceptions.py index a2e810d9f..cbd73129d 100644 --- a/apps/backend/exceptions.py +++ b/apps/backend/exceptions.py @@ -94,3 +94,9 @@ class AgentConfigTemplateEnvNotExistError(BackendBaseException): MESSAGE = _("配置模板Env不存在") MESSAGE_TPL = _("配置模板Env不存在[{name}-{version}-{os_type}-{cpu_arch}]不存在") ERROR_CODE = 15 + + +class ModelInstanceNotFoundError(BackendBaseException): + MESSAGE = _("模型对象不存在") + MESSAGE_TPL = _("模型对象 -> [{model_name}] 不存在") + ERROR_CODE = 16 diff --git a/apps/backend/subscription/errors.py b/apps/backend/subscription/errors.py index 93f432e0a..b6d814673 100644 --- a/apps/backend/subscription/errors.py +++ b/apps/backend/subscription/errors.py @@ -169,3 +169,11 @@ class SubscriptionIncludeGrayBizError(AppBaseException): ERROR_CODE = 19 MESSAGE = _("订阅任务包含Gse2.0灰度业务,任务将暂缓执行无需重复点击") MESSAGE_TPL = _("订阅任务包含Gse2.0灰度业务,任务将暂缓执行无需重复点击") + + +class AgentPackageValidationError(AppBaseException): + """AgentPackage校验错误""" + + ERROR_CODE = 20 + MESSAGE = _("AgentPackage校验错误") + MESSAGE_TPL = _("{msg}") diff --git a/apps/backend/subscription/steps/agent.py b/apps/backend/subscription/steps/agent.py index ec6bb466b..bb627dff2 100644 --- a/apps/backend/subscription/steps/agent.py +++ b/apps/backend/subscription/steps/agent.py @@ -20,6 +20,7 @@ from apps.node_man import constants, models from apps.node_man.constants import DEFAULT_CLOUD from apps.node_man.models import GsePluginDesc, SubscriptionStep +from apps.node_man.tools.gse_package import GsePackageTools from env.constants import GseVersion from pipeline import builder from pipeline.builder import Var @@ -49,6 +50,7 @@ def get_supported_actions(self): ReinstallAgent, UninstallAgent, UpgradeAgent, + DowngradeAgent, RestartAgent, InstallProxy, ReinstallProxy, @@ -115,6 +117,24 @@ def make_instances_migrate_actions(self, instances, auto_trigger=False, preview_ continue instance_actions[instance_id] = job_type_map[self.subscription_step.config["job_type"]] + if instance_actions[instance_id] == backend_const.ActionNameType.UPGRADE_AGENT: + version_map: Dict[int, str] = { + version_map["bk_host_id"]: version_map["version"] + for version_map in self.subscription_step.config.get("version_map_list", []) + } + + bk_host_id: int = instance["host"]["bk_host_id"] + agent_version: str = models.ProcessStatus.objects.get( + bk_host_id=bk_host_id, name=models.ProcessStatus.GSE_AGENT_PROCESS_NAME + ).version + if all( + [ + version_map.get(bk_host_id), + not GsePackageTools.compare_version(version_map[bk_host_id], agent_version), + ] + ): + instance_actions[instance_id] = backend_const.ActionNameType.DOWNGRADE_AGENT + return {"instance_actions": instance_actions, "migrate_reasons": migrate_reasons} @@ -325,6 +345,13 @@ def _generate_activities(self, agent_manager: AgentManager): return activities, None +class DowngradeAgent(UpgradeAgent): + """回退Agent""" + + ACTION_NAME = backend_const.ActionNameType.DOWNGRADE_AGENT + ACTION_DESCRIPTION = _("回退") + + class RestartAgent(AgentAction): """ 重启Agent diff --git a/apps/backend/subscription/steps/agent_adapter/adapter.py b/apps/backend/subscription/steps/agent_adapter/adapter.py index aea7b186b..dfdafe305 100644 --- a/apps/backend/subscription/steps/agent_adapter/adapter.py +++ b/apps/backend/subscription/steps/agent_adapter/adapter.py @@ -18,7 +18,8 @@ from apps.backend.agent.tools import fetch_proxies from apps.backend.constants import ProxyConfigFile -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.backend.subscription import errors +from apps.core.tag.constants import TargetType from apps.core.tag.targets import get_target_helper from apps.node_man import constants, models from apps.utils import cache @@ -34,11 +35,21 @@ LEGACY = "legacy" +class AgentVersionSerializer(serializers.Serializer): + os_cpu_arch = serializers.CharField(label="系统CPU架构", required=False) + bk_host_id = serializers.IntegerField(label="主机ID", required=False) + version = serializers.CharField(label="Agent Version") + + class AgentStepConfigSerializer(serializers.Serializer): name = serializers.CharField(required=False, label="构件名称") # LEGACY 表示旧版本 Agent,仅做兼容 version = serializers.CharField(required=False, label="构件版本", default=LEGACY) job_type = serializers.ChoiceField(required=True, choices=constants.JOB_TUPLE) + choice_version_type = serializers.ChoiceField( + required=False, choices=constants.AgentVersionType.list_choices(), label="选择Agent Version类型" + ) + version_map_list = AgentVersionSerializer(many=True) @dataclass @@ -57,6 +68,9 @@ class AgentStepAdapter: log_prefix: str = field(init=False) # 配置处理模块缓存 _config_handler_cache: typing.Dict[str, GseConfigHandler] = field(init=False) + _setup_info_cache: typing.Dict[str, base.AgentSetupInfo] = field(init=False) + _target_version_cache: typing.Dict[str, str] = field(init=False) + agent_name: str = field(init=False) def __post_init__(self): self.is_legacy = self.gse_version == GseVersion.V1.value @@ -64,6 +78,9 @@ def __post_init__(self): f"[{self.__class__.__name__}({self.subscription_step.step_id})] | {self.subscription_step} |" ) self._config_handler_cache: typing.Dict[str, GseConfigHandler] = {} + self._setup_info_cache: typing.Dict[str, base.AgentSetupInfo] = {} + self._target_version_cache: typing.Dict[str, str] = {} + self.agent_name = self.config.get("name") def get_config_handler(self, agent_name: str, target_version: str) -> GseConfigHandler: @@ -104,11 +121,12 @@ def _get_config( install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]], target_version: typing.Optional[typing.Dict[int, str]] = None, ) -> str: - agent_setup_info: base.AgentSetupInfo = self.setup_info + agent_setup_info: base.AgentSetupInfo = self.get_host_setup_info(host) # 目标版本优先使用传入版本,传入版本必不会是标签所以可直接使用 config_handler: GseConfigHandler = self.get_config_handler( agent_setup_info.name, target_version or agent_setup_info.version ) + config_tmpl_obj: base.AgentConfigTemplate = config_handler.get_matching_config_tmpl( os_type=host.os_type, cpu_arch=host.cpu_arch, @@ -168,29 +186,65 @@ def get_config( @property @cache.class_member_cache() - def setup_info(self) -> base.AgentSetupInfo: + def bk_host_id_version_map(self) -> typing.Dict[int, str]: + return {versiom_map["bk_host_id"]: versiom_map["version"] for versiom_map in self.config["version_map_list"]} + + def get_host_setup_info(self, host: models.Host) -> base.AgentSetupInfo: """ 获取 Agent 设置信息 - TODO 后续如需支持多版本,该方法改造为 `get_host_setup_info`,根据维度进行缓存,参考 _config_handler_cache :return: """ # 如果版本号匹配到标签名称,取对应标签下的真实版本号,否则取原来的版本号 - agent_name: typing.Optional[str] = self.config.get("name") - if agent_name not in AGENT_NAME_TARGET_ID_MAP: - # 1.0 Install + if self.agent_name is None: + # 1.0 Install 或者 2.0统一版本 target_version = self.config.get("version") + setup_info_cache_key: str = f"agent_name_is_none:version:{target_version}" else: - target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( - target_id=AGENT_NAME_TARGET_ID_MAP[agent_name], - target_version=self.config.get("version"), - ) + if self.config["choice_version_type"] == constants.AgentVersionType.UNIFIED.value: + agent_version = self.config.get("version") + setup_info_cache_key: str = ( + f"agent_name:{self.agent_name}:" + f"type:{constants.AgentVersionType.UNIFIED.value}:version:{agent_version}" + ) + elif self.config["choice_version_type"] == constants.AgentVersionType.BY_SYSTEM_ARCH.value: + # TODO 按系统架构维度, 当前只支持按系统,后续需求完善按系统架构 + os_cpu_arch_version_list: typing.List[str] = [ + versiom_map["version"] + for versiom_map in self.config["version_map_list"] + if host.os_type.lower() in versiom_map["os_cpu_arch"] + ] + agent_version: str = os_cpu_arch_version_list[0] if os_cpu_arch_version_list else "stable" + setup_info_cache_key: str = ( + f"agent_name:{self.agent_name}:type:{constants.AgentVersionType.BY_SYSTEM_ARCH.value}:" + f"os:{host.os_type.lower()}:version:{agent_version}" + ) + else: + # 按主机维度 + agent_version: str = self.bk_host_id_version_map[host.bk_host_id] + + target_version_cache_key: str = f"agent_desc_id:{self.agent_desc.id}:agent_version:{agent_version}" + target_version: str = self._target_version_cache.get(target_version_cache_key) + if target_version is None: + target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( + target_id=self.agent_desc.id, + target_version=agent_version, + ) + self._target_version_cache[target_version_cache_key] = target_version - return base.AgentSetupInfo( + if self.config["choice_version_type"] != constants.AgentVersionType.BY_HOST.value: + agent_setup_info: typing.Optional[base.AgentSetupInfo] = self._setup_info_cache.get(setup_info_cache_key) + if agent_setup_info: + return agent_setup_info + + agent_setup_info: base.AgentSetupInfo = base.AgentSetupInfo( is_legacy=self.is_legacy, agent_tools_relative_dir=("agent_tools/agent2", "")[self.is_legacy], name=self.config.get("name"), version=target_version, ) + if self.config["choice_version_type"] != constants.AgentVersionType.BY_HOST.value: + self._setup_info_cache[setup_info_cache_key] = agent_setup_info + return agent_setup_info @staticmethod def validated_data(data, serializer) -> OrderedDict: @@ -204,3 +258,15 @@ def get_os_key(os_type: str, cpu_arch: str) -> str: os_type = os_type or constants.OsType.LINUX cpu_arch = cpu_arch or constants.CpuType.x86_64 return f"{os_type.lower()}-{cpu_arch}" + + @property + def agent_desc(self) -> models.GsePackageDesc: + if hasattr(self, "_agent_desc") and self._agent_desc: + return self._agent_desc + try: + agent_desc = models.GsePackageDesc.objects.get(project=self.agent_name) + except models.GsePackageDesc.DoesNotExist: + raise errors.AgentPackageValidationError(msg="GsePackageDesc [{name}] 不存在".format(name=self.agent_name)) + + setattr(self, "_agent_desc", agent_desc) + return self._agent_desc diff --git a/apps/backend/subscription/views.py b/apps/backend/subscription/views.py index 2b8ac6d74..47a18f6fb 100644 --- a/apps/backend/subscription/views.py +++ b/apps/backend/subscription/views.py @@ -14,7 +14,7 @@ import operator from dataclasses import asdict from functools import cmp_to_key, reduce -from typing import Any, Dict, List, Set +from typing import Dict, List, Set from django.core.cache import caches from django.db import transaction @@ -609,14 +609,14 @@ def fetch_commands(self, request): ap_id_obj_map: Dict[int, models.AccessPoint] = models.AccessPoint.ap_id_obj_map() host_ap: models.AccessPoint = ap_id_obj_map[host_ap_id] - base_agent_setup_info_dict: Dict[str, Any] = asdict( - AgentStepAdapter(subscription_step=sub_step_obj, gse_version=host_ap.gse_version).setup_info + agent_setup_adapter: AgentStepAdapter = AgentStepAdapter( + subscription_step=sub_step_obj, gse_version=host_ap.gse_version ) agent_setup_extra_info_dict = sub_inst.instance_info["host"].get("agent_setup_extra_info") or {} installation_tool = gen_commands( agent_setup_info=AgentSetupInfo( **{ - **base_agent_setup_info_dict, + **asdict(agent_setup_adapter.get_host_setup_info(host)), "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False), } ), diff --git a/apps/backend/sync_task/constants.py b/apps/backend/sync_task/constants.py index f9be617d6..c6090585b 100644 --- a/apps/backend/sync_task/constants.py +++ b/apps/backend/sync_task/constants.py @@ -18,22 +18,29 @@ class SyncTaskType(EnhanceEnum): """同步任务类型""" SYNC_CMDB_HOST = "sync_cmdb_host" + REGISTER_GSE_PACKAGE = "register_gse_package" @classmethod def _get_member__alias_map(cls): - return {cls.SYNC_CMDB_HOST: _("同步 CMDB 主机数据")} + return { + cls.SYNC_CMDB_HOST: _("同步 CMDB 主机数据"), + cls.REGISTER_GSE_PACKAGE: _("注册gse package任务"), + } @classmethod def get_member__cache_key_map(cls): """获取缓存键名""" cache_key_map = { - cls.SYNC_CMDB_HOST: f"{settings.APP_CODE}:backend:sync_task:sync_cmdb_host:" + "biz:{bk_biz_id}" + cls.SYNC_CMDB_HOST: f"{settings.APP_CODE}:backend:sync_task:sync_cmdb_host:" + "biz:{bk_biz_id}", + cls.REGISTER_GSE_PACKAGE: f"{settings.APP_CODE}:backend:sync_task:register_gse_package:" + + "file_name={file_name}:tags={tags}", } return cache_key_map @classmethod def get_member__import_path_map(cls): import_path_map = { - cls.SYNC_CMDB_HOST: "apps.node_man.periodic_tasks.sync_cmdb_host.sync_cmdb_host_task" + cls.SYNC_CMDB_HOST: "apps.node_man.periodic_tasks.sync_cmdb_host.sync_cmdb_host_task", + cls.REGISTER_GSE_PACKAGE: "apps.node_man.periodic_tasks.register_gse_package.register_gse_package_task", } return import_path_map diff --git a/apps/backend/sync_task/handler.py b/apps/backend/sync_task/handler.py index b527f4a29..415dac207 100644 --- a/apps/backend/sync_task/handler.py +++ b/apps/backend/sync_task/handler.py @@ -15,6 +15,7 @@ class AsyncTaskHandler: """写入同步任务的公共逻辑,提供给多方调用""" + @staticmethod def sync_cmdb_host(bk_biz_id=None): # 获取缓存中该业务主机是否正在同步 @@ -36,3 +37,21 @@ def sync_cmdb_host(bk_biz_id=None): REDIS_INST.set(task_key_tpl.format(bk_biz_id=bk_biz_id if bk_biz_id else "all"), task_id, 10) return task_id + + @staticmethod + def register_gse_package(*args, **kwargs): + task_key_tpl_map = constants.SyncTaskType.get_member__cache_key_map() + task_key_tpl = task_key_tpl_map[constants.SyncTaskType.REGISTER_GSE_PACKAGE] + task_key = task_key_tpl.format(*args, **kwargs) + + task_id = REDIS_INST.get(task_key) + if task_id: + return task_id + + async_task = AsyncTaskManager() + async_task.as_task(constants.SyncTaskType.REGISTER_GSE_PACKAGE) + task_id = async_task.delay(*args, **kwargs) + + REDIS_INST.set(task_key, task_id, 10) + + return task_id diff --git a/apps/backend/tests/agent/artifact_builder/test_agent.py b/apps/backend/tests/agent/artifact_builder/test_agent.py index b6d2bbf51..5e108eaf9 100644 --- a/apps/backend/tests/agent/artifact_builder/test_agent.py +++ b/apps/backend/tests/agent/artifact_builder/test_agent.py @@ -16,8 +16,9 @@ from apps.backend.subscription.steps.agent_adapter.handlers import GseConfigHandler from apps.backend.tests.agent import template_env, utils -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.constants import TargetType from apps.core.tag.models import Tag +from apps.core.tag.targets.agent import AgentTargetHelper from apps.mock_data import utils as mock_data_utils from apps.node_man import constants, models @@ -49,10 +50,11 @@ def pkg_checker(self, version_str: str): ) self.assertTrue(os.path.exists(package_path)) - def tag_checker(self, target_id: int): + def tag_checker(self): + agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map() agent_target_version = Tag.objects.get( - target_id=target_id, + target_id=agent_name_target_id_map[self.NAME], name=self.OVERWRITE_VERSION, target_type=TargetType.AGENT.value, ).target_version @@ -106,12 +108,24 @@ def template_and_env_checker(self, version_str): self.assertTrue(models.GseConfigTemplate.objects.filter(**filter_kwargs).exists()) + def gse_package_and_desc_records_checker(self, version_str): + for package_os, cpu_arch in self.OS_CPU_CHOICES: + filter_kwargs: dict = { + "project": self.NAME, + "os": package_os, + "cpu_arch": cpu_arch, + "version": version_str, + } + self.assertTrue(models.GsePackages.objects.filter(**filter_kwargs).exists()) + self.assertTrue(models.GsePackageDesc.objects.filter(**{"project": filter_kwargs.pop("project")}).exists()) + def test_make(self): """测试安装包制作""" with self.ARTIFACT_BUILDER_CLASS(initial_artifact_path=self.ARCHIVE_PATH) as builder: builder.make() self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) def test_make__overwrite_version(self): """测试版本号覆盖""" @@ -129,7 +143,8 @@ def test_make__overwrite_version(self): self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) self.pkg_checker(version_str=self.OVERWRITE_VERSION) - self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.tag_checker() + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoTestCase(FileSystemTestCase): diff --git a/apps/backend/tests/agent/artifact_builder/test_manage_commands.py b/apps/backend/tests/agent/artifact_builder/test_manage_commands.py index ed2d7bb48..ee6f57429 100644 --- a/apps/backend/tests/agent/artifact_builder/test_manage_commands.py +++ b/apps/backend/tests/agent/artifact_builder/test_manage_commands.py @@ -17,7 +17,6 @@ from django.core.management import call_command from apps.backend.tests.agent import utils -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP from apps.mock_data import utils as mock_data_utils from apps.node_man import models @@ -44,6 +43,7 @@ def test_make(self): self.assertTrue(models.UploadPackage.objects.all().exists()) self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) def test_make__overwrite_version(self): """测试版本号覆盖""" @@ -51,7 +51,8 @@ def test_make__overwrite_version(self): self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) self.pkg_checker(version_str=self.OVERWRITE_VERSION) - self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.tag_checker() + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoImportAgentTestCase(FileSystemImportAgentTestCase): diff --git a/apps/backend/tests/agent/artifact_builder/test_proxy.py b/apps/backend/tests/agent/artifact_builder/test_proxy.py index 15a5468ad..d182aabe5 100644 --- a/apps/backend/tests/agent/artifact_builder/test_proxy.py +++ b/apps/backend/tests/agent/artifact_builder/test_proxy.py @@ -34,6 +34,7 @@ def test_make(self): builder.make() self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoTestCase(FileSystemTestCase): diff --git a/apps/backend/tests/components/collections/agent_new/test_install.py b/apps/backend/tests/components/collections/agent_new/test_install.py index bfd7b131b..1ed5f26fc 100644 --- a/apps/backend/tests/components/collections/agent_new/test_install.py +++ b/apps/backend/tests/components/collections/agent_new/test_install.py @@ -184,6 +184,7 @@ def start_patch(self): fs.write("哈哈哈113343ddfd") def setUp(self) -> None: + self.obj_factory.init_gse_package_desc() self.update_callback_url() self.init_mock_clients() self.init_hosts() @@ -412,7 +413,9 @@ async def connect(self): class InstallAgent2WindowsTestCase(InstallWindowsTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] - sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.config.update( + {"name": "gse_agent", "version": "2.0.0", "version_map_list": [], "choice_version_type": "unified"} + ) sub_step_obj.save(update_fields=["config"]) def structure_common_inputs(self): @@ -428,7 +431,7 @@ def test_batch_solution(self): gse_version=GseVersion.V2.value, ) installation_tool = gen_commands( - agent_step_adapter.setup_info, + agent_step_adapter.get_host_setup_info(host), host, mock_data_utils.JOB_TASK_PIPELINE_ID, is_uninstall=False, @@ -885,7 +888,9 @@ class LinuxAgent2InstallTestCase(InstallBaseTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] - sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.config.update( + {"name": "gse_agent", "version": "2.0.0", "version_map_list": [], "choice_version_type": "unified"} + ) sub_step_obj.save() def structure_common_inputs(self): @@ -901,7 +906,7 @@ def test_shell_solution(self): gse_version=GseVersion.V2.value, ) installation_tool = gen_commands( - agent_step_adapter.setup_info, + agent_step_adapter.get_host_setup_info(host), host, mock_data_utils.JOB_TASK_PIPELINE_ID, is_uninstall=False, diff --git a/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py b/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py index ed1922fc0..b29b4eba9 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py @@ -127,6 +127,7 @@ def component_cls(self): @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) sub_step_obj.save() diff --git a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py index 69af15a7e..9f8a56276 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py @@ -53,6 +53,7 @@ def get_default_case_name(cls) -> str: @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() models.Host.objects.filter(bk_host_id__in=cls.obj_factory.bk_host_ids).update(bk_agent_id=get_random_string()) sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) diff --git a/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py b/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py index f2cba8d4b..5233a94cd 100644 --- a/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py +++ b/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py @@ -85,6 +85,7 @@ def structure_common_inputs(self): @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) sub_step_obj.save() diff --git a/apps/backend/tests/components/collections/agent_new/utils.py b/apps/backend/tests/components/collections/agent_new/utils.py index cc6c3c9f2..ded153510 100644 --- a/apps/backend/tests/components/collections/agent_new/utils.py +++ b/apps/backend/tests/components/collections/agent_new/utils.py @@ -295,7 +295,11 @@ def structure_sub_step_data_list(self) -> List[Dict[str, Any]]: sub_step_data.update( { "subscription_id": self.sub_obj.id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": { + "job_type": constants.JobType.INSTALL_AGENT, + "version_map_list": [], + "choice_version_type": "unified", + }, } ) return [sub_step_data] @@ -371,6 +375,15 @@ def init_host_related_data_in_db(self): self.bulk_create_model(model=models.IdentityData, create_data_list=identity_data_list) self.identity_data_objs = models.IdentityData.objects.filter(bk_host_id__in=self.bk_host_ids) + @classmethod + def init_gse_package_desc(cls): + models.GsePackageDesc.objects.update_or_create( + defaults={"description": ""}, project="gse_agent", category="official" + ) + models.GsePackageDesc.objects.update_or_create( + defaults={"description": ""}, project="gse_proxy", category="official" + ) + def init_db(self): """ 初始化DB测试数据 diff --git a/apps/backend/tests/subscription/agent_adapter/test_adapter.py b/apps/backend/tests/subscription/agent_adapter/test_adapter.py index 62f4b6013..56b5aec76 100644 --- a/apps/backend/tests/subscription/agent_adapter/test_adapter.py +++ b/apps/backend/tests/subscription/agent_adapter/test_adapter.py @@ -13,6 +13,9 @@ from apps.backend.subscription.steps.agent_adapter.adapter import AgentStepAdapter from apps.backend.tests.agent.utils import VERSION, AgentBaseTestCase, ProxyBaseTestCase +from apps.backend.tests.components.collections.agent_new.utils import ( + AgentTestObjFactory, +) from apps.mock_data import common_unit from apps.node_man import constants, models from apps.utils import basic @@ -31,6 +34,7 @@ def setUpTestData(cls): super().setUpTestData() host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) cls.host = models.Host.objects.create(**host_model_data) + AgentTestObjFactory.init_gse_package_desc() # 创建订阅相关数据 sub_inst_data = basic.remove_keys_from_dict( @@ -45,7 +49,7 @@ def setUpTestData(cls): **sub_step_data, **{ "subscription_id": cls.sub_inst_record_obj.subscription_id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, }, } ) @@ -85,6 +89,8 @@ def setUpTestData(cls): "job_type": constants.JobType.INSTALL_AGENT, "name": constants.GsePackageCode.PROXY.value, "version": VERSION, + "version_map_list": [], + "choice_version_type": "unified", } cls.sub_step_obj.save() cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=GseVersion.V2.value) @@ -93,12 +99,13 @@ def test_get_config(self): self.clear_agent_data() self.host.node_type = "PROXY" self.host.bk_cloud_id = 1 + agent_setup_info = self.agent_step_adapter.get_host_setup_info(self.host) for config_name in constants.GsePackageTemplate.PROXY.value: self.get_config(config_name) self.assertEqual( self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ) .get_matching_config_tmpl(self.host.os_type, self.host.cpu_arch, config_name) .agent_name_from, @@ -108,8 +115,8 @@ def test_get_config(self): self.get_config(config_name) self.assertEqual( self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ) .get_matching_config_tmpl(self.host.os_type, self.host.cpu_arch, config_name) .agent_name_from, @@ -128,16 +135,17 @@ def clear_agent_data(cls): pass def test_get_env(self): + agent_setup_info = self.agent_step_adapter.get_host_setup_info(self.host) agent_env = self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ).get_matching_template_env(self.host.os_type, self.host.cpu_arch, constants.GsePackageCode.AGENT.value) self.assertEqual(agent_env["BK_GSE_HOME_DIR"], "/usr/local/gse/agent") proxy_env = self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ).get_matching_template_env(self.host.os_type, self.host.cpu_arch, constants.GsePackageCode.PROXY.value) self.assertEqual(proxy_env["BK_GSE_HOME_DIR"], "/usr/local/gse/proxy") @@ -155,6 +163,8 @@ def setUpTestData(cls): "job_type": constants.JobType.INSTALL_AGENT, "name": constants.GsePackageCode.AGENT.value, "version": VERSION, + "version_map_list": [], + "choice_version_type": "unified", } cls.sub_step_obj.save() cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=GseVersion.V2.value) diff --git a/apps/backend/tests/view/test_get_gse_config.py b/apps/backend/tests/view/test_get_gse_config.py index 37e29d55a..cc9e98090 100644 --- a/apps/backend/tests/view/test_get_gse_config.py +++ b/apps/backend/tests/view/test_get_gse_config.py @@ -38,7 +38,7 @@ def setUpTestData(cls): host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) cls.host = models.Host.objects.create(**host_model_data) models.AccessPoint.objects.all().update(gse_version=cls.GSE_VERSION) - + models.GsePackageDesc.objects.create(project="gse_agent", description="", category="official") # 创建订阅相关数据 sub_inst_data = basic.remove_keys_from_dict( origin_data=common_unit.subscription.SUB_INST_RECORD_MODEL_DATA, keys=["id"] @@ -52,7 +52,7 @@ def setUpTestData(cls): **sub_step_data, **{ "subscription_id": cls.sub_inst_record_obj.subscription_id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, }, } ) @@ -117,13 +117,19 @@ def setUp(self) -> None: @classmethod def setUpTestData(cls): super().setUpTestData() - cls.sub_step_obj.config = {"job_type": constants.JobType.INSTALL_AGENT, "name": "gse_agent", "version": "2.0.0"} + cls.sub_step_obj.config = { + "job_type": constants.JobType.INSTALL_AGENT, + "name": "gse_agent", + "version": "2.0.0", + "version_map_list": [], + "choice_version_type": "unified", + } cls.sub_step_obj.save() cls.redis_agent_conf_key = REDIS_AGENT_CONF_KEY_TPL.format( file_name=cls.agent_step_adapter.get_main_config_filename(), sub_inst_id=cls.sub_inst_record_obj.id ) cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=cls.GSE_VERSION) - target_version = cls.agent_step_adapter.setup_info.version + target_version = cls.agent_step_adapter.get_host_setup_info(cls.host).version models.GseConfigEnv.objects.create( agent_name="gse_agent", version=target_version, diff --git a/apps/backend/urls.py b/apps/backend/urls.py index 997d9e80e..6085b2d10 100644 --- a/apps/backend/urls.py +++ b/apps/backend/urls.py @@ -13,6 +13,7 @@ from rest_framework import routers as drf_routers from apps.backend import views +from apps.backend.agent.views import AgentViewSet from apps.backend.healthz.views import HealthzViewSet from apps.backend.plugin.views import PluginViewSet, export_download, upload_package from apps.backend.subscription.views import SubscriptionViewSet @@ -28,6 +29,7 @@ routers.register("subscription", SubscriptionViewSet, basename="subscription") routers.register("healthz", HealthzViewSet, basename="healthz") routers.register("sync_task", SyncTaskViewSet, basename="sync_task") + routers.register("agent", AgentViewSet, basename="agent") export_routers = drf_routers.DefaultRouter(trailing_slash=True) urlpatterns.extend( [ diff --git a/apps/core/ipchooser/tools/base.py b/apps/core/ipchooser/tools/base.py index fc39c6b81..a6b767ce7 100644 --- a/apps/core/ipchooser/tools/base.py +++ b/apps/core/ipchooser/tools/base.py @@ -175,6 +175,31 @@ def extract_bools(cls, values: typing.List[typing.Union[str, bool, int]]) -> typ pass return list(bool_set) + @classmethod + def _handle_topo_conditions(cls, bk_module_ids, bk_set_ids, topo_biz_id, topo_host_ids): + if bk_module_ids: + extra_kwargs = { + "filter_obj_id": constants.ObjectType.MODULE.value, + "filter_inst_ids": list(set(bk_module_ids)), + } + else: + extra_kwargs = { + "filter_obj_id": constants.ObjectType.SET.value, + "filter_inst_ids": list(set(bk_set_ids)), + } + + host_infos: typing.List[types.HostInfo] = resource.ResourceQueryHelper.fetch_biz_hosts( + bk_biz_id=topo_biz_id, fields=["bk_host_id"], **extra_kwargs + ) + host_ids: typing.Set[int] = {host_info["bk_host_id"] for host_info in host_infos} + + if topo_host_ids is None: + topo_host_ids = host_ids + else: + topo_host_ids = topo_host_ids | host_ids + + return topo_host_ids + @classmethod def multiple_cond_sql( cls, @@ -183,6 +208,7 @@ def multiple_cond_sql( is_proxy: bool = False, return_all_node_type: bool = False, extra_wheres: typing.List[str] = None, + need_biz_scope: bool = True, ) -> QuerySet: """ 用于生成多条件sql查询 @@ -191,6 +217,7 @@ def multiple_cond_sql( :param biz_scope: 业务范围限制 :param is_proxy: 是否为代理 :param extra_wheres: 额外的查询条件 + :param need_biz_scope: 是否需要业务范围限制 :return: 根据条件查询的所有结果 """ select: typing.Dict[str, str] = { @@ -210,14 +237,17 @@ def multiple_cond_sql( ] wheres: typing.List[str] = extra_wheres or [] + filter_q = Q() + if params.get("bk_host_id") is not None: + filter_q &= Q(bk_host_id__in=params.get("bk_host_id")) + final_biz_scope: typing.Set[int] = set(biz_scope) # 带有业务筛选条件,需要确保落在指定业务范围内 if params.get("bk_biz_id"): final_biz_scope = final_biz_scope & set(params["bk_biz_id"]) - filter_q: Q = Q(bk_biz_id__in=final_biz_scope) - if params.get("bk_host_id") is not None: - filter_q &= Q(bk_host_id__in=params.get("bk_host_id")) + if need_biz_scope: + filter_q &= Q(bk_biz_id__in=final_biz_scope) # 条件搜索 where_or = [] @@ -278,26 +308,12 @@ def multiple_cond_sql( topo_biz_scope.add(topo_biz_id) continue - if bk_module_ids: - extra_kwargs = { - "filter_obj_id": constants.ObjectType.MODULE.value, - "filter_inst_ids": list(set(bk_module_ids)), - } - else: - extra_kwargs = { - "filter_obj_id": constants.ObjectType.SET.value, - "filter_inst_ids": list(set(bk_set_ids)), - } - - host_infos: typing.List[types.HostInfo] = resource.ResourceQueryHelper.fetch_biz_hosts( - bk_biz_id=topo_biz_id, fields=["bk_host_id"], **extra_kwargs + topo_host_ids = cls._handle_topo_conditions( + bk_module_ids=bk_module_ids, + bk_set_ids=bk_set_ids, + topo_biz_id=topo_biz_id, + topo_host_ids=topo_host_ids, ) - host_ids: typing.Set[int] = {host_info["bk_host_id"] for host_info in host_infos} - - if topo_host_ids is None: - topo_host_ids = host_ids - else: - topo_host_ids = topo_host_ids | host_ids elif condition["key"] == "query" and isinstance(condition["value"], str): fuzzy_search_fields: typing.List[str] = ( @@ -346,17 +362,49 @@ def multiple_cond_sql( if topo_host_ids is not None: topo_query = topo_query | Q(bk_host_id__in=topo_host_ids) - host_queryset: QuerySet = ( - node_man_models.Host.objects.filter( + host_queryset = cls.get_filtered_host_queryset( + is_proxy=is_proxy, + return_all_node_type=return_all_node_type, + final_biz_scope=final_biz_scope, + wheres=wheres, + sql_params=sql_params, + select=select, + topo_query=topo_query, + is_enable_cloud_area_ip_filter=is_enable_cloud_area_ip_filter, + filter_q=filter_q, + need_biz_scope=need_biz_scope, + ) + + return host_queryset + + @classmethod + def get_filtered_host_queryset( + cls, + is_proxy, + return_all_node_type, + final_biz_scope, + wheres, + sql_params, + select, + topo_query, + is_enable_cloud_area_ip_filter, + filter_q, + need_biz_scope=False, + ): + if need_biz_scope: + host_queryset: QuerySet = node_man_models.Host.objects.filter( node_type__in=cls.fetch_match_node_types(is_proxy, return_all_node_type), bk_biz_id__in=final_biz_scope ) - .extra( - select=select, tables=[node_man_models.ProcessStatus._meta.db_table], where=wheres, params=sql_params + else: + host_queryset: QuerySet = node_man_models.Host.objects.filter( + node_type__in=cls.fetch_match_node_types(is_proxy, return_all_node_type), ) - .filter(topo_query) - ) - host_queryset = handle_filter_queryset_by_flag_value(is_enable_cloud_area_ip_filter, host_queryset, filter_q) + host_queryset = host_queryset.extra( + select=select, tables=[node_man_models.ProcessStatus._meta.db_table], where=wheres, params=sql_params + ).filter(topo_query) + + host_queryset = handle_filter_queryset_by_flag_value(is_enable_cloud_area_ip_filter, host_queryset, filter_q) return host_queryset @classmethod diff --git a/apps/core/tag/constants.py b/apps/core/tag/constants.py index 8b0c27614..cf922484a 100644 --- a/apps/core/tag/constants.py +++ b/apps/core/tag/constants.py @@ -13,7 +13,6 @@ from django.utils.translation import ugettext_lazy as _ -from apps.node_man.constants import GsePackageCode from apps.utils.enum import EnhanceEnum @@ -37,10 +36,3 @@ class TagChangeAction(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DELETE: _("删除标签"), cls.CREATE: _("新建标签"), cls.UPDATE: _("更新版本"), cls.OVERWRITE: _("同版本覆盖更新")} - - -# TODO: target_id 临时写死 -AGENT_NAME_TARGET_ID_MAP: Dict[str, int] = { - GsePackageCode.AGENT.value: 1, - GsePackageCode.PROXY.value: 2, -} diff --git a/apps/core/tag/exceptions.py b/apps/core/tag/exceptions.py index 26227c56f..e505c7545 100644 --- a/apps/core/tag/exceptions.py +++ b/apps/core/tag/exceptions.py @@ -52,3 +52,8 @@ class TagInvalidNameError(TagBaseException): MESSAGE_TPL = _("标签名非法:{err_msg}") MESSAGE = _("标签名非法") ERROR_CODE = 6 + + +class ValidationError(TagBaseException): + MESSAGE = _("参数验证失败") + ERROR_CODE = 7 diff --git a/apps/core/tag/migrations/0004_alter_tag_unique_together.py b/apps/core/tag/migrations/0004_alter_tag_unique_together.py new file mode 100644 index 000000000..8c0178249 --- /dev/null +++ b/apps/core/tag/migrations/0004_alter_tag_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2024-02-29 09:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0003_auto_20231029_1336'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='tag', + unique_together={('target_type', 'target_id', 'name', 'target_version')}, + ), + ] diff --git a/apps/core/tag/migrations/0005_create_initial_tags.py b/apps/core/tag/migrations/0005_create_initial_tags.py new file mode 100644 index 000000000..2abdfaf5b --- /dev/null +++ b/apps/core/tag/migrations/0005_create_initial_tags.py @@ -0,0 +1,56 @@ +# Generated by Django 3.2.4 on 2024-02-29 09:40 + +from django.db import IntegrityError, migrations + +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.node_man.constants import CategoryType, GsePackageCode +from apps.node_man.models import GsePackageDesc + + +def create_initial_tags(apps, schema_editor): + try: + agent_target_id = GsePackageDesc.objects.get( + project=GsePackageCode.AGENT.value, category=CategoryType.official + ).id + + proxy_target_id = GsePackageDesc.objects.get( + project=GsePackageCode.PROXY.value, category=CategoryType.official + ).id + except GsePackageDesc.DoesNotExist: + agent_target_id = GsePackageDesc.objects.create( + project=GsePackageCode.AGENT.value, category=CategoryType.official + ).id + + proxy_target_id = GsePackageDesc.objects.create( + project=GsePackageCode.PROXY.value, category=CategoryType.official + ).id + + tag_name__tag_description__map = { + "stable": "稳定版本", + "latest": "最新版本", + "test": "测试版本", + } + for target_id in [proxy_target_id, agent_target_id]: + for tag_name, tag_description in tag_name__tag_description__map.items(): + try: + # 添加Tag记录 + Tag.objects.create( + name=tag_name, + description=tag_description, + target_id=target_id, + target_type=TargetType.AGENT.value, + ) + except IntegrityError: + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("tag", "0004_alter_tag_unique_together"), + ] + + operations = [ + migrations.RunPython(create_initial_tags), + ] diff --git a/apps/core/tag/migrations/0006_create_initial_gsepackages.py b/apps/core/tag/migrations/0006_create_initial_gsepackages.py new file mode 100644 index 000000000..8242b02a9 --- /dev/null +++ b/apps/core/tag/migrations/0006_create_initial_gsepackages.py @@ -0,0 +1,42 @@ +# Generated by Django 3.2.4 on 2024-02-29 09:40 +from typing import List + +from django.db import IntegrityError, migrations + +from apps.node_man.models import GseConfigEnv, GsePackages + + +def create_initial_gse_packages(apps, schema_editor): + extra_env_infos = GseConfigEnv.objects.all() + + gse_package_list: List[GsePackages] = [] + for extra_env_obj in extra_env_infos: + gse_package_list.append( + GsePackages( + pkg_name=str(extra_env_obj), + version=extra_env_obj.version, + project=extra_env_obj.agent_name, + pkg_size=0, + pkg_path="", + md5="", + location="", + os=extra_env_obj.os, + cpu_arch=extra_env_obj.cpu_arch, + ) + ) + + try: + GsePackages.objects.bulk_create(gse_package_list) + except IntegrityError: + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("tag", "0005_create_initial_tags"), + ] + + operations = [ + migrations.RunPython(create_initial_gse_packages), + ] diff --git a/apps/core/tag/models.py b/apps/core/tag/models.py index 1e61e7b42..41e530119 100644 --- a/apps/core/tag/models.py +++ b/apps/core/tag/models.py @@ -29,7 +29,7 @@ class Meta: verbose_name = _("标签") verbose_name_plural = _("标签") # 唯一性校验 - unique_together = (("target_type", "target_id", "name"),) + unique_together = (("target_type", "target_id", "name", "target_version"),) index_together = [ ["target_id", "target_type"], ] diff --git a/apps/core/tag/targets/agent.py b/apps/core/tag/targets/agent.py index 400acd7ac..64f75a36f 100644 --- a/apps/core/tag/targets/agent.py +++ b/apps/core/tag/targets/agent.py @@ -11,8 +11,10 @@ import logging +import typing from apps.core.tag.models import Tag +from apps.node_man.models import GsePackageDesc from .. import constants from . import base @@ -26,12 +28,42 @@ class AgentTargetHelper(base.BaseTargetHelper): TARGET_TYPE = constants.TargetType.AGENT.value def _publish_tag_version(self): - Tag.objects.update_or_create( - defaults={"target_version": self.target_version}, + if self.tag_name in ["stable", "latest", "test"]: + # 内置标签相互覆盖 + Tag.objects.update_or_create( + defaults={"target_version": self.target_version}, + name=self.tag_name, + target_id=self.target_id, + target_type=self.TARGET_TYPE, + ) + return + + tag: typing.Optional[Tag] = Tag.objects.filter( name=self.tag_name, target_id=self.target_id, target_type=self.TARGET_TYPE, - ) + ).first() + + if not tag: + return + + if not tag.target_version: + # 刚创建的未指定版本的标签 + tag.target_version = self.target_version + tag.save() + else: + Tag.objects.update_or_create( + name=self.tag_name, + target_id=self.target_id, + target_type=self.TARGET_TYPE, + target_version=self.target_version, + description=tag.description, + ) def _delete_tag_version(self): - return super()._delete_tag_version() + pass + + @classmethod + def get_agent_name_target_id_map(cls) -> typing.Dict[str, int]: + package_descs = GsePackageDesc.objects.values("project", "id") + return {package_desc["project"]: package_desc["id"] for package_desc in package_descs} diff --git a/apps/core/tag/targets/base.py b/apps/core/tag/targets/base.py index 5479f77ca..06637058e 100644 --- a/apps/core/tag/targets/base.py +++ b/apps/core/tag/targets/base.py @@ -37,10 +37,10 @@ def handle_create_or_update(cls, instance: typing.Type["BaseTargetHelper"]): """ # 发布标签版本后,创建或更新相应的标签 tag, created = models.Tag.objects.update_or_create( - name=instance.tag_name, + target_version=instance.target_version, target_id=instance.target_id, target_type=instance.TARGET_TYPE, - defaults=dict(target_version=instance.target_version), + name=instance.tag_name, ) logger.info( f"[publish_tag_version] update_or_create tag -> {tag.name}({tag.id}) success, " diff --git a/apps/core/tag/tests/test_views.py b/apps/core/tag/tests/test_views.py index 5b591cd03..3bd9657b0 100644 --- a/apps/core/tag/tests/test_views.py +++ b/apps/core/tag/tests/test_views.py @@ -17,7 +17,7 @@ class TagPluginTestCase(utils.PluginBaseTestCase): - DEFAULT_TAG_NAME: str = "stable" + DEFAULT_TAG_NAME: str = "stable2" def create_tag(self, tag_name: str, target_version: str) -> typing.Dict[str, typing.Any]: upload_result = self.upload_plugin() diff --git a/apps/generic.py b/apps/generic.py index d8e3181d2..a612d3b3e 100644 --- a/apps/generic.py +++ b/apps/generic.py @@ -173,6 +173,10 @@ def get_serializer_class(self, *args, **kwargs): return type(self.serializer_class.__name__, (self.serializer_class,), {"Meta": self.serializer_meta}) +class ApiMixinModelViewSet(ApiMixin, _ModelViewSet): + pagination_class = DataPageNumberPagination + + def custom_exception_handler(exc, context): """ 自定义错误处理方式 diff --git a/apps/mock_data/common_unit/subscription.py b/apps/mock_data/common_unit/subscription.py index 39d003403..3b2725f54 100644 --- a/apps/mock_data/common_unit/subscription.py +++ b/apps/mock_data/common_unit/subscription.py @@ -92,7 +92,7 @@ "subscription_id": DEFAULT_SUBSCRIPTION_ID, "step_id": constants.SubStepType.AGENT.lower(), "type": constants.SubStepType.AGENT, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, "params": {"context": {}, "blueking_language": "zh-hans"}, } diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index bee4b0e2b..9ad98f95f 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -193,6 +193,7 @@ def get_optional_items(cls) -> List[str]: "UNINSTALL_AGENT", "REMOVE_AGENT", "UPGRADE_AGENT", + "DOWNGRADE_AGENT", "IMPORT_AGENT", "RESTART_AGENT", "RELOAD_AGENT", @@ -242,7 +243,7 @@ def get_optional_items(cls) -> List[str]: JobType.REINSTALL_PROXY: _("重装 Proxy"), JobType.REINSTALL_AGENT: _("重装 Agent"), JobType.UPGRADE_PROXY: _("升级 Proxy"), - JobType.UPGRADE_AGENT: _("升级 Agent"), + JobType.UPGRADE_AGENT: _("升级/回退 Agent"), JobType.REMOVE_AGENT: _("移除 Agent"), JobType.UNINSTALL_AGENT: _("卸载 Agent"), JobType.UNINSTALL_PROXY: _("卸载 Proxy"), @@ -274,6 +275,7 @@ def get_optional_items(cls) -> List[str]: "REMOVE", "REPLACE", "UPGRADE", + "DOWNGRADE", "UPDATE", "IMPORT", "UPDATE", @@ -302,6 +304,7 @@ def get_optional_items(cls) -> List[str]: OpType.RESTART: _("重启"), OpType.REPLACE: _("替换"), OpType.UPGRADE: _("升级"), + OpType.DOWNGRADE: _("回退"), OpType.REINSTALL: _("重装"), OpType.UPDATE: _("更新"), OpType.REMOVE: _("移除"), @@ -372,6 +375,11 @@ def get_optional_items(cls) -> List[str]: IAM_ACTION_CHOICES = tuple_choices(IAM_ACTION_TUPLE) IamActionType = choices_to_namedtuple(IAM_ACTION_CHOICES) +GSE_PACKAGE_ENABLE_ALIAS_MAP = { + True: _("启用"), + False: _("停用"), +} + class SubscriptionType: POLICY = "policy" @@ -958,6 +966,10 @@ class GsePackageCode(EnhanceEnum): def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.PROXY: _("2.0 Proxy Agent 安装包代号"), cls.AGENT: _("2.0 Agent 安装包代号")} + @classmethod + def values(cls): + return [member.value for member in cls] + class GsePackageEnv(EnhanceEnum): """安装包Env文件名称""" @@ -980,6 +992,13 @@ class GsePackageTemplatePattern(EnhanceEnum): AGENT = re.compile("|".join(GsePackageTemplate.AGENT.value)) +class GsePackageCacheKey(EnhanceEnum): + """GsePackageHandler的缓存key""" + + TAGS_PREFIX = "tags_" + DESCRIPTION_PREFIX = "description_" + + class GsePackageDir(EnhanceEnum): """安装包打包根路径""" @@ -1209,3 +1228,24 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: @classmethod def cpu_type__os_bit_map(cls): return {CpuType.x86: cls.BIT32.value, CpuType.x86_64: cls.BIT64.value, CpuType.aarch64: cls.ARM.value} + + +class AgentVersionType(EnhanceEnum): + UNIFIED = "unified" + BY_HOST = "by_host" + BY_SYSTEM_ARCH = "by_system_arch" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.UNIFIED: _("统一的版本"), + cls.BY_HOST: _("按主机的"), + cls.BY_SYSTEM_ARCH: _("按系统架构"), + } + + +BUILT_IN_TAG_DESCRIPTIONS: List[str] = ["稳定版本", "最新版本", "测试版本"] +BUILT_IN_TAG_NAMES: List[str] = ["stable", "latest", "test"] +TAG_NAME__TAG_DESCRIPTION = dict(zip(BUILT_IN_TAG_NAMES, BUILT_IN_TAG_DESCRIPTIONS)) +TAG_DESCRIPTION__TAG_NAME = dict(zip(BUILT_IN_TAG_DESCRIPTIONS, BUILT_IN_TAG_NAMES)) +STABLE_DESCRIPTION = "稳定版本" diff --git a/apps/node_man/exceptions.py b/apps/node_man/exceptions.py index a1c7cb2a4..c96e1ae88 100644 --- a/apps/node_man/exceptions.py +++ b/apps/node_man/exceptions.py @@ -220,3 +220,41 @@ class YunTiPolicyConfigNotExistsError(NodeManBaseException): MESSAGE = _("云梯策略配置不存在") MESSAGE_TPL = _("云梯策略配置不存在") ERROR_CODE = 43 + + +class FileDoesNotExistError(NodeManBaseException): + MESSAGE = _("文件不存在") + ERROR_CODE = 44 + + +class PluginParseError(NodeManBaseException): + MESSAGE = _("插件解析错误") + ERROR_CODE = 45 + + +class GsePackageInValidError(NodeManBaseException): + MESSAGE = _("agent包无效") + ERROR_CODE = 46 + + +class GsePackageUploadError(NodeManBaseException): + MESSAGE = _("gse包上传失败") + MESSAGE_TPL = _("插件上传失败: agent_name -> {agent_name}, error -> {error}") + ERROR_CODE = 47 + + +class PermissionDeniedError(NodeManBaseException): + MESSAGE = _("权限不足") + ERROR_CODE = 48 + + +class ModelInstanceNotFoundError(NodeManBaseException): + MESSAGE = _("模型对象不存在") + MESSAGE_TPL = _("模型对象 -> [{model_name}] 不存在") + ERROR_CODE = 49 + + +class DuplicateEntryError(NodeManBaseException): + MESSAGE = _("存在唯一索引约束") + MESSAGE_TPL = _("存在唯一索引约束 -> [{entry_info}]") + ERROR_CODE = 50 diff --git a/apps/node_man/handlers/gse_package.py b/apps/node_man/handlers/gse_package.py new file mode 100644 index 000000000..8a4cd51f1 --- /dev/null +++ b/apps/node_man/handlers/gse_package.py @@ -0,0 +1,260 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from collections import defaultdict +from typing import Any, Dict, List + +from django.core.signals import request_finished +from django.db.models import Q, QuerySet +from django.dispatch import receiver +from django.utils.translation import ugettext_lazy as _ + +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.node_man import constants +from apps.node_man.models import GsePackageDesc, GsePackages +from apps.node_man.tools.gse_package import GsePackageTools + + +class GsePackageHandler: + PROJECT_VERSION__TAGS_MAP = "project_version__tags_map" + PROJECT__DESCRIPTION_MAP = "project__description_map" + + def __init__(self): + self._init_caches() + + def _init_caches(self): + """初始化缓存""" + self.cache = { + self.PROJECT_VERSION__TAGS_MAP: defaultdict(list), + self.PROJECT__DESCRIPTION_MAP: {}, + } + + self.cache_counter = { + self.PROJECT_VERSION__TAGS_MAP: 0, + self.PROJECT__DESCRIPTION_MAP: 0, + } + + def clear_caches(self): + """清理缓存""" + self._init_caches() + + def _init_project_version__tags_map(self): + """初始化项目版本标签映射""" + for project in constants.GsePackageCode.values(): + tags: QuerySet = self.get_tag_objs(project).values("name", "description", "target_version") + + for tag in tags: + tag["description"] = _(tag["description"]) + cache_key = self.get_tags_cache_key(project, tag.pop("target_version")) + self.cache[self.PROJECT_VERSION__TAGS_MAP][cache_key].append(tag) + + self.cache_counter[self.PROJECT_VERSION__TAGS_MAP] += 1 + + def _init_project__description_map(self): + """初始化项目描述映射""" + for project in constants.GsePackageCode.values(): + description = GsePackageDesc.objects.filter(project=project).first().description + cache_key = self.get_description_cache_key(project) + self.cache[self.PROJECT__DESCRIPTION_MAP][cache_key] = description + + self.cache_counter[self.PROJECT__DESCRIPTION_MAP] += 1 + + @classmethod + def get_tags_cache_key(cls, project: str, version: str) -> str: + """获取标签缓存key""" + return f"{constants.GsePackageCacheKey.TAGS_PREFIX.value}:{project}:{version}" + + @classmethod + def get_description_cache_key(cls, project: str) -> str: + """获取描述缓存key""" + return f"{constants.GsePackageCacheKey.DESCRIPTION_PREFIX.value}:{project}" + + @classmethod + def get_tag_objs(cls, project: str, version: str = None) -> QuerySet: + """ + 获取标签对象 + :params project: gse_agent或gse_proxy + :params version: agent包版本 + """ + target_ids = GsePackageDesc.objects.filter(project=project).values_list("id", flat=True) + + if not version: + return Tag.objects.filter(target_id__in=target_ids) + return Tag.objects.filter(target_id__in=target_ids, target_version=version) + + def get_tags( + self, + project: str, + version: str, + enable_tag_separation: bool = False, + ) -> List[Dict[str, Any]]: + """ + 获取标签列表 + :params project: gse_agent或gse_proxy + :params version: agent包版本 + :params enable_tag_separation: 是否需要将标签分割为内置和自定义 + """ + if not self.cache_counter[self.PROJECT_VERSION__TAGS_MAP]: + self._init_project_version__tags_map() + + cache_key = self.get_tags_cache_key(project, version) + tags = self.cache[self.PROJECT_VERSION__TAGS_MAP].get(cache_key, []) + + return self.handle_tags(tags, enable_tag_separation=enable_tag_separation) + + def get_description(self, project: str) -> str: + """ + 获取包描述信息 + :params project: gse_agent或gse_proxy + """ + if not self.cache_counter[self.PROJECT__DESCRIPTION_MAP]: + self._init_project__description_map() + + cache_key: str = self.get_description_cache_key(project) + return self.cache[self.PROJECT__DESCRIPTION_MAP].get(cache_key, "") + + def handle_tags( + self, + tags: List[Dict[str, str]], + enable_tag_separation: bool = False, + tag_description=None, + ) -> List[Dict[str, Any]]: + """ + 处理标签列表 + tags: 原始标签列表 + enable_tag_separation: 是否需要将标签分割为内置和自定义 + tag_description: 模糊匹配标签描述 + """ + if tag_description: + tags = [tag for tag in tags if tag_description in tag["description"]] + + if not enable_tag_separation: + return tags + + built_in_tags, custom_tags = self.split_tags_into_builtin_and_custom(tags) + + parent_tags: List[Dict[str, Any]] = [ + {"name": "builtin", "description": _("内置标签"), "children": built_in_tags}, + {"name": "custom", "description": _("自定义标签"), "children": custom_tags}, + ] + + return [parent_tag for parent_tag in parent_tags if parent_tag.get("children")] + + @classmethod + def filter_tags(cls, queryset: QuerySet, project: str, tag_names: List[str] = None) -> QuerySet: + """筛选标签queryset""" + project__id_map: Dict[str, int] = dict(GsePackageDesc.objects.values_list("project", "id")) + combined_tag_names_conditions: Q = Q() + + for tag_name in tag_names or []: + combined_tag_names_conditions |= Q(name__contains=tag_name) + + filter_conditions: Q = Q(target_id=project__id_map.get(project)) & combined_tag_names_conditions + + target_versions: QuerySet = Tag.objects.filter(filter_conditions).values_list("target_version", flat=True) + + return queryset.filter(version__in=target_versions) + + @classmethod + def split_tags_into_builtin_and_custom( + cls, tags: List[Dict[str, Any]] + ) -> (List[Dict[str, Any]], List[Dict[str, Any]]): + """将标签拆分为内置的和自定义的""" + built_in_tags, custom_tags = [], [] + for tag in tags: + if tag["name"] in constants.BUILT_IN_TAG_NAMES: + built_in_tags.append(tag) + else: + custom_tags.append(tag) + + return built_in_tags, custom_tags + + @classmethod + def handle_add_tag(cls, tag_description: str, package_obj: GsePackages, package_desc_obj: GsePackageDesc): + """ + 给已有的agent包新增标签,即添加名为tag_name的标签 + :param tag_description: 添加的标签名 + :param package_obj: Gse包记录 + :param package_desc_obj: Gse包描述记录 + """ + # 如果新增的是内置标签,将原有的内置标签中的target_version进行修改即可,否则创建一个新的标签 + if tag_description in constants.BUILT_IN_TAG_DESCRIPTIONS + constants.BUILT_IN_TAG_NAMES: + Tag.objects.filter( + name=constants.TAG_DESCRIPTION__TAG_NAME.get(tag_description, tag_description), + target_id=package_desc_obj.id, + ).update(target_version=package_obj.version) + else: + tag: Tag = Tag.objects.filter(description=tag_description, target_id=package_desc_obj.id).first() + Tag.objects.create( + name=tag.name if tag else GsePackageTools.generate_name_by_description(tag_description), + description=tag_description, + target_type=TargetType.AGENT.value, + target_id=package_desc_obj.id, + target_version=package_obj.version, + ) + + @classmethod + def handle_update_tag( + cls, tag_description: str, package_obj: GsePackages, package_desc_obj: GsePackageDesc, tag_obj: Tag + ): + """ + 给已有的agent包修改标签,即将原有的tag_obj中的描述修改为tag_name + :param tag_description: 待修改的标签描述 + :param package_obj: Gse包记录 + :param package_desc_obj: Gse包描述记录 + :param tag_obj: 原有的标签记录 + """ + # 1. 内置标签(target_version置空) -> 内置标签(覆盖原有的target_version) + # 2. 自定义标签(删除) -> 内置标签(覆盖原有的target_version) + # 3. 内置标签(target_version置空) -> 自定义标签(新增) + # 4. 自定义标签 -> 自定义标签(将原有的标签描述进行修改) + + # 如果目标标签为内置标签的话,将内置标签的target_version进行覆盖,并对原来的标签进行删除或者清空 + # 如果目标标签为自定义标签,原有标签为内置标签的话,原有标签target_version置空,并新增自定义标签 + # 否则(目标和原有都为自定义标签)将直接修改原有标签的target_version + if tag_description in constants.BUILT_IN_TAG_DESCRIPTIONS + constants.BUILT_IN_TAG_NAMES: + Tag.objects.filter( + name=constants.TAG_DESCRIPTION__TAG_NAME.get(tag_description, tag_description), + target_id=package_desc_obj.id, + ).update(target_version=package_obj.version) + cls.handle_delete_tag(tag_obj.name, tag_obj) + elif tag_obj.name in constants.BUILT_IN_TAG_NAMES: + tag_obj.target_version = "" + tag_obj.save() + cls.handle_add_tag(tag_description, package_obj, package_desc_obj) + else: + tag: Tag = Tag.objects.filter(description=tag_description, target_id=package_desc_obj.id).first() + tag_obj.name = tag.name if tag else GsePackageTools.generate_name_by_description(tag_description) + tag_obj.description = tag_description + tag_obj.save() + + @classmethod + def handle_delete_tag(cls, tag_name: str, tag_obj: Tag): + """ + 删除已有的agent包标签,即删除id为tag_id的标签 + :param tag_name: 待删除的标签id + :param tag_obj: 待删除的标签记录 + """ + # 如果是删除内置标签,将target_version置空即可,不需要删除 + if tag_name in constants.BUILT_IN_TAG_NAMES: + tag_obj.target_version = "" + tag_obj.save() + else: + tag_obj.delete() + + +@receiver(request_finished) +def clear_gse_package_handler_cache(sender, **kwargs): + """每次视图结束后清除缓存,保证每次视图获取的都是最新数据""" + gse_package_handler.clear_caches() + + +gse_package_handler = GsePackageHandler() diff --git a/apps/node_man/handlers/host.py b/apps/node_man/handlers/host.py index 8f357fec5..390849436 100644 --- a/apps/node_man/handlers/host.py +++ b/apps/node_man/handlers/host.py @@ -527,6 +527,10 @@ def update_proxy_info(params: dict): setattr(identity, kwarg, identity_kwargs[kwarg]) identity.save() + # 更新ProcessStatus中Proxy包版本信息 + if params.get("version"): + ProcessStatus.objects.filter(bk_host_id=kwargs["bk_host_id"]).update(version=params["version"]) + @staticmethod def get_host_infos_gby_ip_key(ips: Iterable[str], ip_version: int): """ diff --git a/apps/node_man/handlers/meta.py b/apps/node_man/handlers/meta.py index d6447ee87..37a3966e8 100644 --- a/apps/node_man/handlers/meta.py +++ b/apps/node_man/handlers/meta.py @@ -10,17 +10,20 @@ """ import re from collections import ChainMap -from typing import Any, Callable, Dict, Tuple +from typing import Any, Callable, Dict, List, Tuple from django.conf import settings from django.db import connection from django.utils.translation import ugettext as _ from apps.core.concurrent.cache import FuncCacheDecorator -from apps.node_man import constants, models, tools +from apps.node_man import constants, exceptions, models, tools from apps.node_man.handlers.cloud import CloudHandler from apps.node_man.handlers.cmdb import CmdbHandler +from apps.node_man.handlers.gse_package import gse_package_handler from apps.node_man.handlers.install_channel import InstallChannelHandler +from apps.node_man.models import GsePackages +from apps.node_man.permissions.package_manage import PackageManagePermission from apps.node_man.tools import JobTools from apps.utils import APIModel @@ -602,6 +605,74 @@ def fetch_dept_name_children(dept_names: Tuple): dept_name_children.append({"id": dept_name, "name": dept_name}) return dept_name_children + def fetch_agent_pkg_manager_children(self, params=None): + params: Dict[str, Any] = params or {} + project: str = params.get("project", "gse_agent") + + if not PackageManagePermission().has_permission(None, None): + raise exceptions.PermissionDeniedError(_("该用户不是管理员")) + + versions, tag_name__description_map, creators, is_readys = set(), dict(), set(), set() + gse_packages = GsePackages.objects.filter(project=project).values("version", "created_by", "is_ready") + for package in gse_packages: + tags: List[Dict[str, Any]] = gse_package_handler.get_tags( + version=package["version"], + project=project, + enable_tag_separation=False, + ) + versions.add(package["version"]) + creators.add(package["created_by"]) + is_readys.add(package["is_ready"]) + for tag in tags: + tag_name__description_map[tag["name"]] = tag["description"] + + return [ + { + "name": _("版本号"), + "id": "version", + "children": [ + { + "id": version, + "name": version, + } + for version in versions + ], + }, + { + "name": _("标签信息"), + "id": "tag_names", + "children": [ + { + "id": tag_name, + "name": tag_description, + } + for tag_name, tag_description in tag_name__description_map.items() + ], + }, + { + "name": _("上传用户"), + "id": "created_by", + "children": [ + { + "id": creator, + "name": creator, + } + for creator in creators + ], + }, + { + "name": _("状态"), + "id": "is_ready", + "children": [ + { + "id": is_ready, + "name": constants.GSE_PACKAGE_ENABLE_ALIAS_MAP.get(is_ready, is_ready), + } + for is_ready in is_readys + ], + }, + ] + def filter_condition(self, category, params=None): """ 获取过滤条件 @@ -629,6 +700,8 @@ def filter_condition(self, category, params=None): elif category == "os_type": ret = self.fetch_os_type_children() return ret + elif category == "agent_pkg_manage": + return self.fetch_agent_pkg_manager_children(params=params) @staticmethod def install_default_values_formatter(install_default_values: Dict[str, Dict[str, Any]]): diff --git a/apps/node_man/handlers/plugin_v2.py b/apps/node_man/handlers/plugin_v2.py index 983136b83..526ec6266 100644 --- a/apps/node_man/handlers/plugin_v2.py +++ b/apps/node_man/handlers/plugin_v2.py @@ -8,22 +8,16 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import json -import os import random from collections import ChainMap, defaultdict from typing import Any, Dict, List, Union -import requests from django.conf import settings from django.core.cache import cache -from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models import Count from django.utils.translation import ugettext_lazy as _ from apps.component.esbclient import client_v2 -from apps.core.files import core_files_constants -from apps.core.files.storage import get_storage from apps.node_man import constants, exceptions, models, tools from apps.node_man.constants import IamActionType from apps.node_man.handlers.cmdb import CmdbHandler @@ -31,72 +25,11 @@ from apps.utils.basic import distinct_dict_list, list_slice from apps.utils.batch_request import batch_request from apps.utils.concurrent import batch_call -from apps.utils.files import md5sum from apps.utils.local import get_request_username from common.api import NodeApi class PluginV2Handler: - @staticmethod - def upload(package_file: InMemoryUploadedFile, module: str) -> Dict[str, Any]: - """ - 将文件上传至 - :param package_file: InMemoryUploadedFile - :param module: 所属模块 - :return: - { - "result": True, - "message": "", - "code": "00", - "data": { - "id": record.id, # 上传文件记录ID - "name": record.file_name, # 包名 - "pkg_size": record.file_size, # 大小, - } - } - """ - with package_file.open("rb") as tf: - - # 计算上传文件的md5 - md5 = md5sum(file_obj=tf, closed=False) - - base_params = {"module": module, "md5": md5} - - # 如果采用对象存储,文件直接上传至仓库,并将返回的目标路径传到后台,由后台进行校验并创建上传记录 - # TODO 后续应该由前端上传文件并提供md5 - if settings.STORAGE_TYPE in core_files_constants.StorageType.list_cos_member_values(): - storage = get_storage() - - try: - storage_path = storage.save(name=os.path.join(settings.UPLOAD_PATH, tf.name), content=tf) - except Exception as e: - raise exceptions.PluginUploadError(plugin_name=tf.name, error=e) - - return NodeApi.upload( - { - **base_params, - # 最初文件上传的名称,后台会使用该文件名保存并覆盖同名文件 - "file_name": tf.name, - "file_path": storage_path, - "download_url": storage.url(storage_path), - } - ) - - else: - - response = requests.post( - url=settings.DEFAULT_FILE_UPLOAD_API, - data={ - **base_params, - "bk_app_code": settings.APP_CODE, - "bk_username": get_request_username(), - }, - # 本地文件系统仍通过上传文件到Nginx并回调后台 - files={"package_file": tf}, - ) - - return json.loads(response.content) - @staticmethod def list_plugin(query_params: Dict): plugin_page = NodeApi.plugin_list(query_params) diff --git a/apps/node_man/migrations/0082_gsepackagedesc_gsepackages.py b/apps/node_man/migrations/0082_gsepackagedesc_gsepackages.py new file mode 100644 index 000000000..f10559a4b --- /dev/null +++ b/apps/node_man/migrations/0082_gsepackagedesc_gsepackages.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0081_auto_20240307_1656"), + ] + + operations = [ + migrations.CreateModel( + name="GsePackageDesc", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("created_by", models.CharField(default="", max_length=32, verbose_name="创建者")), + ("updated_time", models.DateTimeField(auto_now=True, null=True, verbose_name="更新时间")), + ("updated_by", models.CharField(blank=True, default="", max_length=32, verbose_name="修改者")), + ("project", models.CharField(db_index=True, max_length=32, unique=True, verbose_name="工程名")), + ("description", models.TextField(verbose_name="安装包描述")), + ("description_en", models.TextField(blank=True, null=True, verbose_name="英文插件描述")), + ( + "category", + models.CharField( + choices=[("official", "official"), ("external", "external"), ("scripts", "scripts")], + max_length=32, + verbose_name="所属范围", + ), + ), + ], + options={ + "verbose_name": "Gse包描述(GsePackageDesc)", + "verbose_name_plural": "Gse包描述(GsePackageDesc)", + }, + ), + migrations.CreateModel( + name="GsePackages", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("created_by", models.CharField(default="", max_length=32, verbose_name="创建者")), + ("updated_time", models.DateTimeField(auto_now=True, null=True, verbose_name="更新时间")), + ("updated_by", models.CharField(blank=True, default="", max_length=32, verbose_name="修改者")), + ("pkg_name", models.CharField(max_length=128, verbose_name="压缩包名")), + ("version", models.CharField(max_length=128, verbose_name="版本号")), + ("project", models.CharField(db_index=True, max_length=32, verbose_name="工程名")), + ("pkg_size", models.IntegerField(verbose_name="包大小")), + ("pkg_path", models.CharField(max_length=128, verbose_name="包路径")), + ("md5", models.CharField(max_length=32, verbose_name="md5值")), + ("location", models.CharField(max_length=512, verbose_name="安装包链接")), + ( + "os", + models.CharField( + choices=[("windows", "windows"), ("linux", "linux"), ("aix", "aix"), ("solaris", "solaris")], + db_index=True, + default="linux", + max_length=32, + verbose_name="系统类型", + ), + ), + ( + "cpu_arch", + models.CharField( + choices=[ + ("x86", "x86"), + ("x86_64", "x86_64"), + ("powerpc", "powerpc"), + ("aarch64", "aarch64"), + ("sparc", "sparc"), + ], + db_index=True, + default="x86_64", + max_length=32, + verbose_name="CPU类型", + ), + ), + ("is_ready", models.BooleanField(default=True, verbose_name="插件是否可用")), + ("version_log", models.TextField(blank=True, null=True, verbose_name="版本日志")), + ("version_log_en", models.TextField(blank=True, null=True, verbose_name="英文版本日志")), + ], + options={ + "verbose_name": "Gse包(GsePackages)", + "verbose_name_plural": "Gse包(GsePackages)", + }, + ), + ] diff --git a/apps/node_man/migrations/0083_merge_20240911_1050.py b/apps/node_man/migrations/0083_merge_20240911_1050.py new file mode 100644 index 000000000..1f98b8346 --- /dev/null +++ b/apps/node_man/migrations/0083_merge_20240911_1050.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.4 on 2024-09-11 02:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0082_gsepackagedesc_gsepackages"), + ("node_man", "0082_host_dept_name"), + ] + + operations = [] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 7c9a84ddb..7cb84ab5c 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -2533,3 +2533,49 @@ class Meta: index_together = [ ["bk_biz_id", "enable"], ] + + +class GsePackages(orm.OperateRecordModel): + pkg_name = models.CharField(_("压缩包名"), max_length=128) + version = models.CharField(_("版本号"), max_length=128) + project = models.CharField(_("工程名"), max_length=32, db_index=True) + pkg_size = models.IntegerField(_("包大小")) + pkg_path = models.CharField(_("包路径"), max_length=128) + md5 = models.CharField(_("md5值"), max_length=32) + location = models.CharField(_("安装包链接"), max_length=512) + os = models.CharField( + _("系统类型"), + max_length=32, + choices=constants.PLUGIN_OS_CHOICES, + default=constants.PluginOsType.linux, + db_index=True, + ) + cpu_arch = models.CharField( + _("CPU类型"), max_length=32, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True + ) + + # 由于创建记录时,文件可能仍然在传输过程中,因此需要标志位判断是否已经可用 + is_ready = models.BooleanField(_("插件是否可用"), default=True) + + version_log = models.TextField(_("版本日志"), null=True, blank=True) + version_log_en = models.TextField(_("英文版本日志"), null=True, blank=True) + + class Meta: + verbose_name = _("Gse包(GsePackages)") + verbose_name_plural = _("Gse包(GsePackages)") + + +class GsePackageDesc(orm.OperateRecordModel): + """ + Gse包描述表 + """ + + # 安装包名需要全局唯一,防止冲突 + project = models.CharField(_("工程名"), max_length=32, unique=True, db_index=True) + description = models.TextField(_("安装包描述")) + description_en = models.TextField(_("英文插件描述"), null=True, blank=True) + category = models.CharField(_("所属范围"), max_length=32, choices=constants.CATEGORY_CHOICES) + + class Meta: + verbose_name = _("Gse包描述(GsePackageDesc)") + verbose_name_plural = _("Gse包描述(GsePackageDesc)") diff --git a/apps/node_man/periodic_tasks/register_gse_package.py b/apps/node_man/periodic_tasks/register_gse_package.py new file mode 100644 index 000000000..aa6fbb5a3 --- /dev/null +++ b/apps/node_man/periodic_tasks/register_gse_package.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from apps.backend.celery import app +from apps.node_man.tools.gse_package import GsePackageTools + +logger = logging.getLogger("app") + + +@app.task(queue="default") +def register_gse_package_task(file_name: str, tags: List[str]): + upload_package_obj = GsePackageTools.get_latest_upload_record(file_name=file_name) + + project, artifact_builder_class = GsePackageTools.distinguish_gse_package(file_path=upload_package_obj.file_path) + + with artifact_builder_class( + initial_artifact_path=upload_package_obj.file_path, + tags=tags, + ) as builder: + builder.make() diff --git a/apps/node_man/permissions/__init__.py b/apps/node_man/permissions/__init__.py new file mode 100644 index 000000000..29ed269e0 --- /dev/null +++ b/apps/node_man/permissions/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/apps/node_man/permissions/package_manage.py b/apps/node_man/permissions/package_manage.py new file mode 100644 index 000000000..bc8ae81d8 --- /dev/null +++ b/apps/node_man/permissions/package_manage.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ +from rest_framework import permissions + +from apps.node_man.handlers.iam import IamHandler +from apps.utils.local import get_request_username + + +class PackageManagePermission(permissions.BasePermission): + message = _("您没有该操作的权限") + + def has_permission(self, request, view): + + if IamHandler().is_superuser(get_request_username()): + return True + + return False diff --git a/apps/node_man/serializers/host.py b/apps/node_man/serializers/host.py index 2d2b07ae1..81018d5c3 100644 --- a/apps/node_man/serializers/host.py +++ b/apps/node_man/serializers/host.py @@ -55,6 +55,7 @@ class HostUpdateSerializer(serializers.Serializer): bt_speed_limit = serializers.IntegerField(label=_("加速"), required=False) enable_compression = serializers.BooleanField(label=_("数据压缩开关配置"), required=False, default=False) data_path = serializers.CharField(label=_("数据文件路径"), required=False) + version = serializers.CharField(label=_("Proxy包版本"), required=False) def validate(self, attrs): cipher = tools.HostTools.get_asymmetric_cipher() diff --git a/apps/node_man/serializers/job.py b/apps/node_man/serializers/job.py index dba4db497..9d5ba8503 100644 --- a/apps/node_man/serializers/job.py +++ b/apps/node_man/serializers/job.py @@ -17,7 +17,10 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from apps.backend.subscription.steps.agent_adapter.adapter import LEGACY +from apps.backend.subscription.steps.agent_adapter.adapter import ( + LEGACY, + AgentVersionSerializer, +) from apps.core.gray.constants import INSTALL_OTHER_AGENT_AP_ID_OFFSET from apps.core.gray.handlers import GrayHandler from apps.core.gray.tools import GrayTools @@ -52,11 +55,18 @@ def set_agent_setup_info_to_attrs(attrs): # 如果开启 DHCP,安装 2.0 Agent,开启 AgentID 特性 # 在执行模块根据主机接入点所属的 GSE 版本决定是否采用下列的 agent_setup_info name = ("gse_agent", "gse_proxy")[attrs["node_type"] == "PROXY"] + # attrs["agent_setup_info"]["name"] = name + # 处理重装类型setup_info结构 + agent_setup_info = attrs.get("agent_setup_info", {}) + global_settings_agent_version = models.GlobalSettings.get_config( + models.GlobalSettings.KeyEnum.GSE_AGENT2_VERSION.value, default="stable" + ) + attrs["agent_setup_info"] = { "name": name, - "version": models.GlobalSettings.get_config( - models.GlobalSettings.KeyEnum.GSE_AGENT2_VERSION.value, default="stable" - ), + "version": agent_setup_info.get("version") or global_settings_agent_version, + "choice_version_type": agent_setup_info.get("choice_version_type") or constants.AgentVersionType.UNIFIED.value, + "version_map_list": agent_setup_info.get("version_map_list", []), } @@ -113,7 +123,6 @@ def backfill_bk_host_id(self, hosts): else: sub_query.children.append(("inner_ipv6", _host["inner_ipv6"])) ip_key = _host["inner_ipv6"] - cloud_ip_host_info_map[f"{_host['bk_cloud_id']}:{ip_key}"] = _host query_params.children.append(sub_query) @@ -265,6 +274,11 @@ class AgentSetupInfoSerializer(serializers.Serializer): # LEGACY 表示旧版本 Agent,仅做兼容 version = serializers.CharField(required=False, label="构件版本", default=LEGACY) + choice_version_type = serializers.ChoiceField( + required=False, choices=constants.AgentVersionType.list_choices(), label=_("选择Agent Version类型") + ) + version_map_list = AgentVersionSerializer(required=False, many=True) + class ScriptHook(serializers.Serializer): name = serializers.CharField(label=_("脚本名称"), min_length=1) diff --git a/apps/node_man/serializers/meta.py b/apps/node_man/serializers/meta.py index 2ceb0068a..86b07856b 100644 --- a/apps/node_man/serializers/meta.py +++ b/apps/node_man/serializers/meta.py @@ -30,6 +30,7 @@ class JobSettingSerializer(serializers.Serializer): class FilterConditionSerializer(serializers.Serializer): category = serializers.CharField(label=_("分类"), required=False, default="") bk_biz_ids = serializers.ListField(label=_("业务列表"), required=False, default=[], child=serializers.IntegerField()) + project = serializers.CharField(label=_("工程名"), required=False) # 时间范围 start_time = serializers.DateTimeField(label=_("起始时间"), required=False) diff --git a/apps/node_man/serializers/package_manage.py b/apps/node_man/serializers/package_manage.py new file mode 100644 index 000000000..2827b8816 --- /dev/null +++ b/apps/node_man/serializers/package_manage.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from apps.core.tag.constants import TargetType +from apps.exceptions import ValidationError +from apps.node_man.constants import GsePackageCode +from apps.node_man.handlers.gse_package import gse_package_handler +from apps.node_man.models import UploadPackage + + +class TagSerializer(serializers.Serializer): + name = serializers.CharField() + description = serializers.CharField() + + +class TagProjectSerializer(serializers.Serializer): + project = serializers.CharField(default=GsePackageCode.AGENT.value) + + +class TagCreateSerializer(serializers.Serializer): + tag_descriptions = serializers.ListField(child=serializers.CharField(), default=[]) + project = serializers.CharField() + + def validate(self, attrs): + project = attrs["project"] + if project not in GsePackageCode.values(): + raise ValidationError(_("project可选项[ gse_agent | gse_plugin ]")) + + return attrs + + class Meta: + ref_name = "tag_create" + + +class ParentTagSerializer(serializers.Serializer): + name = serializers.CharField() + description = serializers.CharField() + children = TagSerializer(many=True) + + +class ConditionsSerializer(serializers.Serializer): + key = serializers.ChoiceField(choices=["version", "os_cpu_arch", "tags", "is_ready"]) + values = serializers.ListField() + + +class BasePackageSerializer(serializers.Serializer): + @staticmethod + def get_tags(obj, enable_tag_separation=True): + return gse_package_handler.get_tags( + project=obj.project, + version=obj.version, + enable_tag_separation=enable_tag_separation, + ) + + @classmethod + def get_description(cls, obj): + return gse_package_handler.get_description(project=obj.project) + + +class PackageSerializer(BasePackageSerializer): + id = serializers.IntegerField() + pkg_name = serializers.CharField() + version = serializers.CharField() + os = serializers.CharField() + cpu_arch = serializers.CharField() + tags = serializers.SerializerMethodField() + created_by = serializers.CharField() + created_time = serializers.DateTimeField() + is_ready = serializers.BooleanField() + + +class ListResponseSerializer(serializers.Serializer): + total = serializers.IntegerField() + list = PackageSerializer(many=True) + + +class OperateSerializer(serializers.Serializer): + is_ready = serializers.BooleanField(required=False) + tags = serializers.ListField(required=False) + + def update(self, instance, validated_data): + for attr, value in validated_data.items(): + setattr(instance, attr, value) + + instance.save() + return instance + + +# class QuickSearchSerializer(serializers.Serializer): +# project = serializers.ChoiceField(choices=GsePackageCode.list_choices()) + + +class UploadSerializer(serializers.Serializer): + overload = serializers.BooleanField(default=False, help_text="是否覆盖上传") + package_file = serializers.FileField() + + def validate(self, data): + overload = data.get("overload") + package_file = data.get("package_file") + + file_name = package_file.name + + if not (file_name.endswith(".tgz") or file_name.endswith(".tar.gz")): + raise ValidationError(_("仅支持'tgz', 'tar.gz'拓展名的文件")) + + if not overload: + upload_package: UploadPackage = UploadPackage.objects.filter( + file_name__contains=file_name, module=TargetType.AGENT.value + ).first() + if upload_package: + raise ValidationError( + data={ + "message": _("存在同名agent包"), + "file_name": upload_package.file_name, + "md5": upload_package.md5, + }, + code=3800002, + ) + + return data + + +class UploadResponseSerializer(serializers.Serializer): + name = serializers.CharField() + pkg_size = serializers.IntegerField() + + +class ParseSerializer(serializers.Serializer): + file_name = serializers.CharField() + + +class ParseResponseSerializer(serializers.Serializer): + class ParsePackageSerializer(serializers.Serializer): + project = serializers.ChoiceField(choices=["gse_agent", "gse_proxy"], required=False) + pkg_name = serializers.CharField(required=False, source="pkg_relative_path") + version = serializers.CharField(required=False) + os = serializers.CharField() + cpu_arch = serializers.CharField() + config_templates = serializers.ListField(default=[]) + + def to_representation(self, instance): + data = super().to_representation(instance) + data["project"] = self.context.get("project", "") + data["version"] = self.context.get("version", "") + return data + + description = serializers.CharField() + packages = ParsePackageSerializer(many=True) + + +class AgentRegisterSerializer(serializers.Serializer): + file_name = serializers.CharField() + tags = serializers.ListField(child=serializers.CharField(), default=[]) + tag_descriptions = serializers.ListField(child=serializers.CharField(), default=[]) + project = serializers.CharField(default=GsePackageCode.AGENT.value) + + +class AgentRegisterTaskSerializer(serializers.Serializer): + task_id = serializers.CharField() + version = serializers.CharField() + + +class AgentRegisterTaskResponseSerializer(serializers.Serializer): + is_finish = serializers.BooleanField() + status = serializers.ChoiceField(choices=["SUCCESS", "FAILED", "RUNNING"]) + message = serializers.CharField() + + +class DeployedAgentCountSerializer(serializers.Serializer): + items = serializers.JSONField(default=[]) + project = serializers.CharField(default=GsePackageCode.AGENT.value) + + +class VersionQuerySerializer(serializers.Serializer): + project = serializers.CharField() + os = serializers.CharField(required=False, allow_blank=True) + cpu_arch = serializers.CharField(required=False, allow_blank=True) + versions = serializers.ListField(child=serializers.CharField(), required=False) + + +class VersionCompareSerializer(serializers.Serializer): + current_version = serializers.CharField() + version_to_compares = serializers.ListField(child=serializers.CharField()) diff --git a/apps/node_man/tests/test_views/test_package_manage_views.py b/apps/node_man/tests/test_views/test_package_manage_views.py new file mode 100644 index 000000000..c110e1501 --- /dev/null +++ b/apps/node_man/tests/test_views/test_package_manage_views.py @@ -0,0 +1,471 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import io +import os +import uuid +from unittest.mock import patch + +from django.conf import settings +from django.core.files.uploadedfile import InMemoryUploadedFile + +from apps.backend.plugin.handler import PluginHandler +from apps.backend.sync_task.constants import SyncTaskType +from apps.backend.tests.subscription.agent_adapter.test_adapter import ( + Proxy2StepAdapterTestCase, +) +from apps.core.files.storage import get_storage +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.node_man import constants +from apps.node_man.constants import ( + CPU_TUPLE, + GSE_PACKAGE_ENABLE_ALIAS_MAP, + OS_TYPE, + CategoryType, + GsePackageCode, +) +from apps.node_man.handlers.meta import MetaHandler +from apps.node_man.models import ( + GsePackageDesc, + GsePackages, + Host, + ProcessStatus, + UploadPackage, +) +from apps.node_man.tests.utils import create_gse_package, create_host +from apps.utils.files import md5sum +from common.api.modules.utils import add_esb_info_before_request + + +def delay(self, *args, **kwargs): + self.task_func(*args, **kwargs) + return "1" + + +class PackageManageViewsTestCaseUsingProxy(Proxy2StepAdapterTestCase): + def init_tags(self): + try: + agent_target_id = GsePackageDesc.objects.get( + project=GsePackageCode.AGENT.value, category=CategoryType.official + ).id + + proxy_target_id = GsePackageDesc.objects.get( + project=GsePackageCode.PROXY.value, category=CategoryType.official + ).id + except GsePackageDesc.DoesNotExist: + agent_target_id = GsePackageDesc.objects.create( + project=GsePackageCode.AGENT.value, category=CategoryType.official + ).id + + proxy_target_id = GsePackageDesc.objects.create( + project=GsePackageCode.PROXY.value, category=CategoryType.official + ).id + + for target_id in [proxy_target_id, agent_target_id]: + # 添加Tag记录 + Tag.objects.create( + name="stable", + description="稳定版本", + target_id=target_id, + target_type=TargetType.AGENT.value, + ) + Tag.objects.create( + name="latest", + description="最新版本", + target_id=target_id, + target_type=TargetType.AGENT.value, + ) + Tag.objects.create( + name="test", + description="测试版本", + target_id=target_id, + target_type=TargetType.AGENT.value, + ) + + def upload_file(self): + with open(self.ARCHIVE_PATH, "rb") as f: + file_content = f.read() + + memory_uploaded_file = InMemoryUploadedFile( + file=io.BytesIO(file_content), + field_name=None, + name=os.path.basename(self.ARCHIVE_PATH), + content_type="text/plain", + size=len(file_content), + charset=None, + ) + + self.storage = get_storage() + with memory_uploaded_file.open("rb") as tf: + self.md5 = md5sum(file_obj=tf, closed=False) + self.storage_path = self.storage.save(name=os.path.join(settings.UPLOAD_PATH, tf.name), content=tf) + + self.file_name = tf.name + + params = { + "md5": self.md5, + "module": "agent", + "file_name": tf.name, + "file_path": self.storage_path, + "download_url": self.storage.url(self.storage_path), + } + + add_esb_info_before_request(params) + + PluginHandler.upload( + md5=params["md5"], + origin_file_name=params["file_name"], + module=params["module"], + operator=params["bk_username"], + app_code=params["bk_app_code"], + file_path=params.get("file_path"), + download_url=params.get("download_url"), + ) + + def setUp(self, *args, **kwargs): + super().setUp() + + self.init_tags() + + with self.ARTIFACT_BUILDER_CLASS(initial_artifact_path=self.ARCHIVE_PATH, tags=["stable", "latest"]) as builder: + builder.make() + + self.upload_file() + + gse_package = GsePackages.objects.first() + gse_package.created_at = "admin" + gse_package.save() + + self.task_map = {} + + @classmethod + def clear_agent_data(cls): + GsePackages.objects.all().delete() + GsePackageDesc.objects.all().delete() + Tag.objects.all().delete() + + def tearDown(self): + super().tearDown() + + if os.path.exists(self.storage_path): + os.remove(self.storage_path) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_list(self, *args, **kwargs): + # 和之前的builder.make加起来100 + create_gse_package(99, start_id=1000, project="gse_proxy") + + result = self.client.get(path="/api/agent/package/", data={"project": "gse_proxy"}) + self.assertEqual(result["result"], True) + self.assertEqual(len(result["data"]), 100) + + result = self.client.get(path="/api/agent/package/", data={"page": 1, "pagesize": 2, "project": "gse_proxy"}) + self.assertEqual(result["result"], True) + self.assertEqual(result["data"]["total"], 100) + self.assertEqual(len(result["data"]["list"]), 2) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_list_with_filter_condition(self, *args, **kwargs): + # 不筛选 + result = self.client.get(path="/api/agent/package/", data={"page": 1, "pagesize": 2, "project": "gse_proxy"}) + self.assertEqual(result["data"]["total"], 1) + self.assertEqual(len(result["data"]["list"]), 1) + + gse_package = GsePackages.objects.first() + + # 筛选tags + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "tag_names": "stable", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertIn("stable", self.collect_all_tag_names(result["data"]["list"][0]["tags"])) + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "tag_names": "latest", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertIn("latest", self.collect_all_tag_names(result["data"]["list"][0]["tags"])) + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "tag_names": "test", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选os_cpu_arch + computer_os, cpu_arch = "linux", "x86_64" + result = self.client.get( + path="/api/agent/package/", + data={ + "page": 1, + "pagesize": 2, + "os": gse_package.os, + "cpu_arch": gse_package.cpu_arch, + "project": "gse_proxy", + }, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["os"], computer_os) + self.assertEqual(result["data"]["list"][0]["cpu_arch"], cpu_arch) + result = self.client.get( + path="/api/agent/package/", + data={"page": 1, "pagesize": 2, "os": "windows", "cpu_arch": "x86_64", "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选created_by + result = self.client.get( + path="/api/agent/package/", + data={"page": 1, "pagesize": 2, "created_by": gse_package.created_by, "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["created_by"], gse_package.created_by) + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "created_by": "system", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选is_ready + result = self.client.get( + path="/api/agent/package/", + data={"page": 1, "pagesize": 2, "is_ready": str(gse_package.is_ready), "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["is_ready"], True) + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "is_ready": "False", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + # 筛选version + result = self.client.get( + path="/api/agent/package/", + data={"page": 1, "pagesize": 2, "version": gse_package.version, "project": "gse_proxy"}, + ) + self.assertEqual(len(result["data"]["list"]), 1) + self.assertEqual(result["data"]["list"][0]["version"], "1.0.1") + result = self.client.get( + path="/api/agent/package/", data={"page": 1, "pagesize": 2, "version": "1.0.2", "project": "gse_proxy"} + ) + self.assertEqual(len(result["data"]["list"]), 0) + + @classmethod + def collect_all_tag_names(cls, tags, *args, **kwargs): + """ + tags: [ + { + "id": "builtin", + "name": "内置标签", + "children": [ + {"id": "stable", "name": "稳定版本", "children": []}, + {"id": "latest", "name": "最新版本", "children": []}, + ], + }, + { + "id": "custom", + "name": "自定义标签", + "children": [ + {"id": "custom", "name": "自定义版本", "children": []} + ] + }, + ] + """ + tag_name_set = set() + for parent_tag in tags: + for children_tag in parent_tag["children"]: + tag_name_set.add(children_tag["name"]) + + return tag_name_set + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_update(self, *args, **kwargs): + first_gse_package = GsePackages.objects.first() + self.assertEqual(first_gse_package.is_ready, True) + self.client.put( + path=f"/api/agent/package/{first_gse_package.id}/", data={"is_ready": False, "project": "gse_proxy"} + ) + self.assertEqual(GsePackages.objects.first().is_ready, False) + + # 测试更新不存在的id是否服务器异常 + self.client.put(path="/api/agent/package/10000/", data={"is_ready": False}) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_destroy(self, *args, **kwargs): + gse_packages = GsePackages.objects.all() + self.assertEqual(len(gse_packages), 1) + self.client.delete(path=f"/api/agent/package/{gse_packages.first().id}/", data={"project": "gse_proxy"}) + self.assertEqual(len(GsePackages.objects.all()), 0) + + # 测试删除存在的id是否服务器异常 + self.client.delete(path="/api/agent/package/10000/") + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_quick_search_condition(self, *args, **kwargs): + result = self.client.get(path="/api/agent/package/quick_search_condition/", data={"project": "gse_proxy"}) + for condition in result["data"]: + if condition["id"] == "os_cpu_arch": + self.assertCountEqual( + condition["children"], + [ + {"id": "linux_x86_64", "name": "Linux_x86_64", "count": 1, "description": "### 1.0.1\nchange"}, + ], + ) + self.assertEqual(condition["count"], 1) + + elif condition["id"] == "version": + self.assertCountEqual( + condition["children"], + [ + {"id": "1.0.1", "name": "1.0.1", "count": 1, "description": "### 1.0.1\nchange"}, + ], + ) + self.assertEqual(condition["count"], 1) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_filter_condition_with_agent_pkg_manage(self, *args, **kwargs): + result = MetaHandler().filter_condition("agent_pkg_manage", params={"project": "gse_proxy"}) + self.assertEqual(len(GsePackages.objects.all()), 1) + gse_package = GsePackages.objects.first() + is_ready = gse_package.is_ready + for condition in result: + if condition["id"] == "version": + self.assertCountEqual( + condition["children"], + [ + {"id": gse_package.version, "name": gse_package.version}, + ], + ) + elif condition["id"] == "tags": + self.assertCountEqual( + condition["children"], + [ + {"id": "stable", "name": "稳定版本"}, + {"id": "latest", "name": "最新版本"}, + ], + ) + elif condition["id"] == "creator": + self.assertCountEqual( + condition["children"], + [ + {"id": gse_package.created_by, "name": gse_package.created_by}, + ], + ) + elif condition["id"] == "is_ready": + self.assertCountEqual( + condition["children"], + [ + {"id": is_ready, "name": GSE_PACKAGE_ENABLE_ALIAS_MAP.get(is_ready, is_ready)}, + ], + ) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_deployed_hosts_count(self, *args, **kwargs): + data = { + "items": [{"os_type": "linux", "cpu_arch": "x86_64"}, {"os_type": "windows", "cpu_arch": "x86_64"}], + "project": "gse_agent", + } + + # 100台主机都是LINUX,cpu_arch都为x86_64 + Host.objects.all().delete() + ProcessStatus.objects.all().delete() + create_host(100, os_type=constants.OsType.LINUX, node_type=constants.NodeType.AGENT) + result = self.client.post(path="/api/agent/package/deployed_hosts_count/", data=data) + self.assertEqual( + result["data"], + [ + {"os_type": "linux", "cpu_arch": "x86_64", "count": 100}, + {"os_type": "windows", "cpu_arch": "x86_64", "count": 0}, + ], + ) + + # 100台主机都是WINDOWS,cpu_arch都为x86_64 + Host.objects.all().delete() + ProcessStatus.objects.all().delete() + create_host(100, os_type=constants.OsType.WINDOWS, node_type=constants.NodeType.AGENT, start_idx=100) + result = self.client.post(path="/api/agent/package/deployed_hosts_count/", data=data) + self.assertEqual( + result["data"], + [ + {"os_type": "linux", "cpu_arch": "x86_64", "count": 0}, + {"os_type": "windows", "cpu_arch": "x86_64", "count": 100}, + ], + ) + + # 50台主机是WINDOWS,50台主机是LINUX,cpu_arch都为x86_64 + Host.objects.all().delete() + ProcessStatus.objects.all().delete() + create_host(50, os_type=constants.OsType.WINDOWS, node_type=constants.NodeType.AGENT, start_idx=200) + create_host(50, os_type=constants.OsType.LINUX, node_type=constants.NodeType.AGENT, start_idx=250) + result = self.client.post(path="/api/agent/package/deployed_hosts_count/", data=data) + self.assertEqual( + result["data"], + [ + {"os_type": "linux", "cpu_arch": "x86_64", "count": 50}, + {"os_type": "windows", "cpu_arch": "x86_64", "count": 50}, + ], + ) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_upload(self, *args, **kwargs): + self.assertEqual(UploadPackage.objects.count(), 1) + upload_package = UploadPackage.objects.first() + self.assertEqual(upload_package.module, "agent") + self.assertEqual(upload_package.file_path, self.storage_path) + self.assertEqual(upload_package.md5, self.md5) + self.assertEqual(upload_package.file_name, self.file_name) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + def test_parse(self, *args, **kwargs): + res = self.client.post( + path="/backend/api/agent/parse/", + data={ + "file_name": self.file_name, + }, + ) + self.assertEqual(res["result"], True) + self.assertIn("description", res["data"]) + self.assertIn("packages", res["data"]) + + for packages in res["data"]["packages"]: + self.assertIn(packages["os"].upper(), OS_TYPE.values()) + self.assertIn(packages["project"], GsePackageCode.get_member_value__alias_map()) + self.assertIn(packages["cpu_arch"], CPU_TUPLE) + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + @patch("apps.backend.sync_task.manager.AsyncTaskManager.delay", delay) + def test_create_register_task(self, *args, **kwargs): + file_name = kwargs.get("file_name") if "file_name" in kwargs else self.file_name + task_id = str(uuid.uuid4()) + self.task_map[task_id] = "PENDING" + + res = self.client.post( + path="/backend/api/sync_task/create/", + data={ + "task_name": SyncTaskType.REGISTER_GSE_PACKAGE.value, + "task_params": { + "file_name": file_name, + "tags": [], + }, + }, + ) + if res["result"] is True: + self.task_map[task_id] = "SUCCESS" + self.assertIn("task_id", res["data"]) + else: + self.task_map[task_id] = "FAILURE" + + return task_id + + @patch("apps.node_man.permissions.package_manage.PackageManagePermission.has_permission", return_value=True) + @patch("apps.backend.sync_task.manager.AsyncTaskManager.delay", delay) + def test_query_register_task(self, *args, **kwargs): + task_id = self.test_create_register_task() + self.assertEqual(self.task_map[task_id], "SUCCESS") + + wrong_task_id = self.test_create_register_task(file_name=self.file_name + "...") + self.assertEqual(self.task_map[wrong_task_id], "FAILURE") diff --git a/apps/node_man/tests/utils.py b/apps/node_man/tests/utils.py index 576f7883d..46a56d76c 100644 --- a/apps/node_man/tests/utils.py +++ b/apps/node_man/tests/utils.py @@ -27,6 +27,7 @@ from apps.node_man.models import ( AccessPoint, Cloud, + GsePackages, Host, IdentityData, InstallChannel, @@ -146,6 +147,7 @@ def create_host( login_ip=None, proc_type=None, os_type=None, + start_idx=0, ): # 若传了bk_host_id,number必须为1 host_to_create = [] @@ -156,8 +158,8 @@ def create_host( if number - index * max_count > max_count: # 若还要分批创建 host, process, identity = create_host_from_a_to_b( - index * max_count, - (index + 1) * max_count, + start_idx + index * max_count, + start_idx + (index + 1) * max_count, bk_host_id=bk_host_id, ip=ip, auth_type=auth_type, @@ -171,8 +173,8 @@ def create_host( ) else: host, process, identity = create_host_from_a_to_b( - index * max_count, - number, + start_idx + index * max_count, + start_idx + number, bk_host_id=bk_host_id, ip=ip, auth_type=auth_type, @@ -1267,3 +1269,40 @@ def ret_to_validate_data(data): login_ip_info = HostHandler.get_host_infos_gby_ip_key(login_ips, constants.CmdbIpVersion.V4.value) return biz_info, data, cloud_info, ap_id_name, inner_ip_info, outer_ip_info, login_ip_info, bk_biz_scope + + +def create_gse_package( + number, + start_id=1, + version=None, + project=None, + pkg_size=None, + is_ready=None, + pkg_path=None, + location=None, + created_by=None, + os=None, + cpu_arch=None, +): + gse_packages = [] + for i in range(start_id, number + start_id): + gse_package = GsePackages( + id=i, + pkg_name=f"pkg_name{i+1}", + version=version or random.choice(["version1", "version2", "version3"]), + project=project or random.choice(["agent", "proxy"]), + pkg_size=pkg_size or random.randint(1, 2000), + pkg_path=pkg_path or "/tmp/", + md5="", + location=location or "", + os=os or random.choice(constants.OS_TUPLE), + cpu_arch=cpu_arch or random.choice(constants.CPU_TUPLE), + is_ready=is_ready or random.choice(list(constants.GSE_PACKAGE_ENABLE_ALIAS_MAP.keys())), + created_by=created_by or "admin", + updated_by=created_by or "admin", + version_log="", + version_log_en="", + ) + gse_packages.append(gse_package) + gse_packages = GsePackages.objects.bulk_create(gse_packages) + return [gse_package.id for gse_package in gse_packages] diff --git a/apps/node_man/tools/gse_package.py b/apps/node_man/tools/gse_package.py new file mode 100644 index 000000000..15a96b4eb --- /dev/null +++ b/apps/node_man/tools/gse_package.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import hashlib +import os +import re +import tarfile +import time +from typing import Dict, List, Type + +from django.utils.translation import ugettext as _ + +from apps.backend.agent.artifact_builder import agent, proxy +from apps.backend.agent.artifact_builder.base import BaseArtifactBuilder +from apps.core.files.storage import get_storage +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.node_man import constants, exceptions, models +from apps.node_man.constants import CategoryType +from apps.node_man.models import GsePackageDesc, UploadPackage + + +class GsePackageTools: + @classmethod + def get_latest_upload_record(cls, file_name: str) -> models.UploadPackage: + """ + 获取最新的agent上传包记录 + :param file_name: agent包文件名 + """ + upload_package_obj: models.UploadPackage = ( + models.UploadPackage.objects.filter(file_name=file_name, module=TargetType.AGENT.value) + .order_by("-upload_time") + .first() + ) + if upload_package_obj is None: + raise exceptions.FileDoesNotExistError(_("找不到请求发布的文件,请确认后重试")) + + return upload_package_obj + + @classmethod + def distinguish_gse_package(cls, file_path: str) -> (str, Type[BaseArtifactBuilder]): + """ + 区分agent和proxy包 + :param file_path: agent包文件路径 + """ + storage = get_storage() + with storage.open(name=file_path) as fs: + with tarfile.open(fileobj=fs) as tf: + directory_members = tf.getmembers() + + for directory in directory_members: + if directory.name == "gse/server": + return constants.GsePackageCode.PROXY.value, proxy.ProxyArtifactBuilder + elif directory.name.startswith("gse/") and constants.AGENT_PATH_RE.match(directory.name[4:]): + return constants.GsePackageCode.AGENT.value, agent.AgentArtifactBuilder + + # 文件解析失败,将上传记录和包都干掉 + storage.delete(name=file_path) + UploadPackage.objects.filter(file_path=file_path).delete() + raise exceptions.GsePackageUploadError( + agent_name=os.path.basename(file_path), + error=_("该agent包无效," "gse_proxy的gse目录中应该包含server文件夹," "gse_agent的gse目录中应该包含agent_(os)_(cpu_arch)的文件夹"), + ) + + @classmethod + def generate_name_by_description(cls, description: str) -> str: + """ + 根据标签描述生成对应唯一的id + :param description: agent包标签描述 + """ + current_time: str = str(time.time()) + unique_string: str = description + current_time + return hashlib.md5(unique_string.encode("utf-8")).hexdigest() + + @classmethod + def create_agent_tags(cls, tag_descriptions, project): + """ + 根据agent包标签描述列表自动创建或返回已有的标签信息 + + :input + { + "project": "gse_agent", + "tag_descriptions": ["aaa", "bbb"] + } + + :return + [ + { + "name": "7188612c63753ec339500e72083fe8ac", + "description": "aaa" + }, + { + "name": "381b9dc36b32195acb53418b588bb99b", + "description": "bbb" + } + ] + + :params tag_descriptions: 标签描述列表 + :params project: gse_agent或gse_proxy + + """ + tags: List[Dict[str, str]] = [] + for tag_description in tag_descriptions: + gse_package_desc_obj, _ = GsePackageDesc.objects.get_or_create( + project=project, category=CategoryType.official + ) + + if tag_description in constants.BUILT_IN_TAG_DESCRIPTIONS + constants.BUILT_IN_TAG_NAMES: + # 内置标签,手动指定name和description + name: str = constants.TAG_DESCRIPTION__TAG_NAME[tag_description] + tag_description = constants.TAG_NAME__TAG_DESCRIPTION.get(tag_description, tag_description) + else: + # 自定义标签,自动生成name + name: str = GsePackageTools.generate_name_by_description(tag_description) + + tag_queryset = Tag.objects.filter( + description=tag_description, + target_id=gse_package_desc_obj.id, + target_type=TargetType.AGENT.value, + ) + + # 如果已存在标签,直接返回已存在的标签,否则创建一个新的标签 + if tag_queryset.exists(): + tag_obj: Tag = tag_queryset.first() + else: + tag_obj, _ = Tag.objects.update_or_create( + defaults={"description": tag_description}, + name=name, + target_id=gse_package_desc_obj.id, + target_type=TargetType.AGENT.value, + ) + + tags.append({"name": tag_obj.name, "description": tag_obj.description}) + + return tags + + @staticmethod + def extract_numbers(s): + """从字符串中提取所有的数字,并返回它们的整数列表""" + numbers = re.findall(r"\d+", s) + return [int(num) for num in numbers] + + @staticmethod + def compare_version(a, b): + return GsePackageTools.extract_numbers(a) > GsePackageTools.extract_numbers(b) + + @classmethod + def match_criteria(cls, pkg_version_info, validated_data, filter_keys): + for key in filter_keys: + if key == "os" and validated_data["os"] not in pkg_version_info["os_choices"]: + return False + elif key == "cpu_arch" and validated_data["cpu_arch"] not in pkg_version_info["cpu_arch_choices"]: + return False + return True diff --git a/apps/node_man/tools/package.py b/apps/node_man/tools/package.py new file mode 100644 index 000000000..0d8c6f6c7 --- /dev/null +++ b/apps/node_man/tools/package.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import json +import os +from typing import Any, Dict + +import requests +from django.conf import settings +from django.core.files.uploadedfile import InMemoryUploadedFile + +from apps.core.files import core_files_constants +from apps.core.files.storage import get_storage +from apps.node_man import exceptions +from apps.utils.files import md5sum +from apps.utils.local import get_request_username +from common.api import NodeApi + + +class PackageTools: + @staticmethod + def upload(package_file: InMemoryUploadedFile, module: str) -> Dict[str, Any]: + """ + 将文件上传至 + :param package_file: InMemoryUploadedFile + :param module: 所属模块 + :return: + { + "result": True, + "message": "", + "code": "00", + "data": { + "id": record.id, # 上传文件记录ID + "name": record.file_name, # 包名 + "pkg_size": record.file_size, # 大小, + } + } + """ + with package_file.open("rb") as tf: + + # 计算上传文件的md5 + md5 = md5sum(file_obj=tf, closed=False) + + base_params = {"module": module, "md5": md5} + + # 如果采用对象存储,文件直接上传至仓库,并将返回的目标路径传到后台,由后台进行校验并创建上传记录 + # TODO 后续应该由前端上传文件并提供md5 + if settings.STORAGE_TYPE in core_files_constants.StorageType.list_cos_member_values(): + storage = get_storage() + + try: + storage_path = storage.save(name=os.path.join(settings.UPLOAD_PATH, tf.name), content=tf) + except Exception as e: + raise exceptions.PluginUploadError(agent_name=tf.name, error=e) + + return NodeApi.upload( + { + **base_params, + # 最初文件上传的名称,后台会使用该文件名保存并覆盖同名文件 + "file_name": tf.name, + "file_path": storage_path, + "download_url": storage.url(storage_path), + } + ) + + else: + + response = requests.post( + url=settings.DEFAULT_FILE_UPLOAD_API, + data={ + **base_params, + "bk_app_code": settings.APP_CODE, + "bk_username": get_request_username(), + }, + # 本地文件系统仍通过上传文件到Nginx并回调后台 + files={"package_file": tf}, + ) + + return json.loads(response.content) diff --git a/apps/node_man/urls.py b/apps/node_man/urls.py index faa90fc48..d2f0724e2 100644 --- a/apps/node_man/urls.py +++ b/apps/node_man/urls.py @@ -40,6 +40,9 @@ ) from apps.node_man.views.healthz import HealthzViewSet from apps.node_man.views.host_v2 import HostV2ViewSet +from apps.node_man.views.package_manage import ( # AgentPackageDescViewSet, + PackageManageViewSet, +) from apps.node_man.views.plugin import GsePluginViewSet from apps.node_man.views.plugin_v2 import PluginV2ViewSet from apps.node_man.views.sync_task import SyncTaskViewSet @@ -67,6 +70,8 @@ router.register(r"v2/plugin", PluginV2ViewSet, basename="plugin_v2") router.register(r"healthz", HealthzViewSet, basename="healthz") router.register(r"sync_task", SyncTaskViewSet, basename="sync_task") +router.register(r"agent/package", PackageManageViewSet, basename="package_manage") +# router.register(r"agent/package_desc", AgentPackageDescViewSet, basename="package_desc") biz_dispatcher = DjangoBasicResourceApiDispatcher(iam, settings.BK_IAM_SYSTEM_ID) biz_dispatcher.register("biz", BusinessResourceProvider()) diff --git a/apps/node_man/views/meta.py b/apps/node_man/views/meta.py index 0a9f0c75b..3736d5bbf 100644 --- a/apps/node_man/views/meta.py +++ b/apps/node_man/views/meta.py @@ -29,6 +29,7 @@ class MetaViews(APIViewSet): @swagger_auto_schema( operation_summary="获取过滤条件", + query_serializer=FilterConditionSerializer, tags=META_VIEW_TAGS, methods=["GET", "POST"], ) diff --git a/apps/node_man/views/package_manage.py b/apps/node_man/views/package_manage.py new file mode 100644 index 000000000..bd70240f6 --- /dev/null +++ b/apps/node_man/views/package_manage.py @@ -0,0 +1,828 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from collections import defaultdict +from typing import Any, Dict, List + +import django_filters +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.db import transaction +from django.db.models import Min, Q, QuerySet +from django.http import JsonResponse +from django.utils.translation import get_language +from django.utils.translation import ugettext_lazy as _ +from django_filters.rest_framework import DjangoFilterBackend, FilterSet +from drf_yasg import openapi +from rest_framework import filters +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.status import HTTP_200_OK + +from apps.backend.sync_task.constants import SyncTaskType +from apps.core.files.storage import get_storage +from apps.core.ipchooser.tools.base import HostQuerySqlHelper +from apps.core.tag.constants import TargetType +from apps.core.tag.models import Tag +from apps.exceptions import ValidationError +from apps.generic import ApiMixinModelViewSet as ModelViewSet +from apps.generic import ValidationMixin +from apps.node_man import constants, exceptions, models +from apps.node_man.constants import STABLE_DESCRIPTION +from apps.node_man.handlers.gse_package import GsePackageHandler, gse_package_handler +from apps.node_man.models import GsePackageDesc, GsePackages, UploadPackage +from apps.node_man.permissions import package_manage as pkg_permission +from apps.node_man.serializers import package_manage as pkg_manage +from apps.node_man.tools.gse_package import GsePackageTools +from apps.node_man.tools.package import PackageTools +from common.api import NodeApi +from common.utils.drf_utils import swagger_auto_schema + +PACKAGE_MANAGE_VIEW_TAGS = ["PKG_Manager"] +PACKAGE_DES_VIEW_TAGS = ["PKG_Desc"] +logger = logging.getLogger("app") + + +class PackageManageOrderingFilterSet(filters.OrderingFilter): + def get_ordering(self, request, queryset, view): + # 这里由原来的request.params变为了request.data,其余不变 + params = request.data.get(self.ordering_param) + if params: + fields = [param.strip() for param in params.split(",")] + ordering = self.remove_invalid_fields(queryset, fields, view, request) + if ordering: + return ordering + + return self.get_default_ordering(view) + + def filter_queryset(self, request, queryset, view): + ordering = self.get_ordering(request, queryset, view) + if not ordering: + return queryset + + for field in ordering[::-1]: + reverse = field.startswith("-") + if field.lstrip("-") == "version": + # 版本按这样排 V2.1.6-beta.10 -> [2, 1, 5, 10] + queryset: List[GsePackages] = sorted( + queryset, key=lambda obj: GsePackageTools.extract_numbers(obj.version), reverse=reverse + ) + else: + queryset: List[GsePackages] = sorted( + queryset, key=lambda obj: getattr(obj, field.lstrip("-")), reverse=reverse + ) + + return queryset + + +class PackageManageFilterClass(FilterSet): + os = django_filters.BaseInFilter(field_name="os", lookup_expr="in") + cpu_arch = django_filters.BaseInFilter(field_name="cpu_arch", lookup_expr="in") + os_cpu_arch = django_filters.BaseInFilter(field_name="os_cpu_arch", method="filter_os_cpu_arch") + tag_names = django_filters.BaseInFilter(lookup_expr="in", method="filter_tag_names") + created_by = django_filters.BaseInFilter(field_name="created_by", lookup_expr="in") + is_ready = django_filters.BooleanFilter(field_name="is_ready") + version = django_filters.BaseInFilter(field_name="version", lookup_expr="in") + created_time = django_filters.DateTimeFromToRangeFilter() + condition = django_filters.Filter(method="filter_condition") + + def filter_tag_names(self, queryset, name, tag_names): + if "project" not in self.request.data: + raise ValidationError(_("筛选tag_names时必须传入project")) + return gse_package_handler.filter_tags(queryset, self.request.data["project"], tag_names=tag_names) + + def filter_os_cpu_arch(self, queryset, name, os_cpu_archs): + package_query = Q() + for os_cpu_arch in os_cpu_archs: + try: + os, cpu_arch = os_cpu_arch.split("_", 1) + except ValueError: + raise ValidationError(_("筛选格式应该为{os}_{cpu_arch}")) + + package_query |= Q(os=os, cpu_arch=cpu_arch) + + return queryset.filter(package_query) + + def filter_condition(self, queryset, name, query_list): + if not isinstance(query_list, list): + return queryset + + fields_to_search = ["os", "cpu_arch", "created_by", "is_ready", "version"] + + model_field_query, tag_query = Q(), Q() + tag_names: List[str] = [] + for query_info in query_list: + if not isinstance(query_info, dict) or query_info.get("key") != "query" or "value" not in query_info: + continue + + tag_names.append(query_info["value"]) + + for field in fields_to_search: + model_field_query |= Q(**{f"{field}__icontains": query_info["value"]}) + + if "project" in self.request.data and tag_names: + tag_query = Q( + id__in=gse_package_handler.filter_tags( + queryset, self.request.data["project"], tag_names=tag_names + ).values_list("id", flat=True) + ) + + return queryset.filter(model_field_query | tag_query) + + class Meta: + model = GsePackages + fields = ["tag_names", "project", "created_by", "is_ready", "version", "os", "cpu_arch"] + + +class PackageManageFilterBackend(DjangoFilterBackend): + def get_filterset_kwargs(self, request, queryset, view): + return { + "data": {**request.data, **dict(request.query_params.items())}, + "queryset": queryset, + "request": request, + } + + +class PackageManageViewSet(ValidationMixin, ModelViewSet): + serializer_class = pkg_manage.PackageSerializer + permission_classes = (pkg_permission.PackageManagePermission,) + filter_backends = (PackageManageFilterBackend, filters.SearchFilter, PackageManageOrderingFilterSet) + filter_class = PackageManageFilterClass + ordering_fields = ["version", "created_time"] + + def get_queryset(self): + if self.action not in ["search", "quick_search_condition"]: # noqa + self.filter_class = None + return models.GsePackages.objects.all().order_by("-is_ready") + + @swagger_auto_schema( + responses={200: pkg_manage.ListResponseSerializer}, + operation_summary="安装包列表", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["POST"]) + def search(self, request, *args, **kwargs): + """ + return: { + "total": 2, + "list": [ + { + "id": 1, + "pkg_name": "pkg_name", + "version": "1.1.1", + "os": "Linux", + "cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + }, + { + "id": 2, + "pkg_name": "pkg_name", + "version": "1.1.2", + "os": "Linux", + "os_cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + }, + ], + } + """ + return super().list(request, *args, **kwargs) + + def perform_update(self, serializer): + serializer.save() + + if "tags" not in serializer.validated_data: + return + + package_obj: GsePackages = self.get_object() + tags: QuerySet = gse_package_handler.get_tag_objs(package_obj.project, package_obj.version) + with transaction.atomic(): + tags.filter(name__in=["latest", "test"]).update(target_version="") + tags.exclude(name__in=["latest", "test", "stable"]).delete() + + package_desc_obj: GsePackageDesc = GsePackageDesc.objects.get(project=package_obj.project) + for tag_description in serializer.validated_data["tags"]: + GsePackageHandler.handle_add_tag( + tag_description=tag_description, + package_obj=package_obj, + package_desc_obj=package_desc_obj, + ) + + @swagger_auto_schema( + operation_summary="操作类动作:启用/停用/修改(新增, 删除)标签", + body_in=pkg_manage.OperateSerializer, + responses={200: pkg_manage.PackageSerializer}, + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + def update(self, request, validated_data, *args, **kwargs): + """ + return: { + "id": 1, + "pkg_name": "pkg_name", + "version": "1.1.1", + "os": "Linux", + "cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + } + """ + instance: GsePackages = self.get_object() + serializer = pkg_manage.OperateSerializer(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) + + updated_instance: GsePackages = self.get_object() + + return Response(pkg_manage.PackageSerializer(updated_instance).data) + + @swagger_auto_schema( + operation_summary="删除安装包", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + def destroy(self, request, *args, **kwargs): + gse_package_obj: GsePackages = self.get_object() + + # 如果最后一个版本的包被清除了,将标签的target_version置空,防止下次上传这个版本的包时留下以前的标签 + if GsePackages.objects.filter(version=gse_package_obj.version).count() == 1: + Tag.objects.filter(target_version=gse_package_obj.version).update(target_version=None) + + return super().destroy(request, *args, **kwargs) + + @swagger_auto_schema( + operation_summary="获取快速筛选信息", + manual_parameters=[ + openapi.Parameter( + "project", in_=openapi.TYPE_STRING, description="区分gse_agent, gse_proxy", type=openapi.TYPE_STRING + ) + ], + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["GET"]) + def quick_search_condition(self, request, *args, **kwargs): + """ + return: [ + { + "name": "操作系统/架构", + "id": "os_cpu_arch", + "children": [ + {"name": "Linux_x86_64", "id": "linux_x86_64", "count": 10}, + {"name": "Linux_x86", "id": "linux_x86", "count": 10}, + {"name": "ALL", "id": "ALL", "count": 20}, + ] + }, + { + "name": "版本号", + "id": "version", + "children": [ + {"name": "2.1.8", "id": "2.1.8", "count": 10}, + {"name": "2.1.7", "id": "2.1.7", "count": 10}, + {"name": "ALL", "id": "ALL", "count": 20}, + ] + }, + ] + """ + gse_packages = self.filter_queryset(self.get_queryset()).values("version", "os", "cpu_arch", "version_log") + + version__count_map: Dict[str, int] = defaultdict(int) + os_cpu_arch__count_map: Dict[str, int] = defaultdict(int) + + for package in gse_packages.values("version", "os", "cpu_arch", "version_log"): + version, os_cpu_arch = package["version"], f"{package['os']}_{package['cpu_arch']}" + + version__count_map[version] += 1 + os_cpu_arch__count_map[os_cpu_arch] += 1 + + return Response( + [ + { + "name": _("操作系统/架构"), + "id": "os_cpu_arch", + "children": [ + { + "id": os_cpu_arch, + "name": os_cpu_arch.capitalize(), + "count": count, + } + for os_cpu_arch, count in os_cpu_arch__count_map.items() + ], + "count": sum(os_cpu_arch__count_map.values()), + }, + { + "name": _("版本号"), + "id": "version", + "children": [ + { + "id": version, + "name": version.capitalize(), + "count": version__count_map[version], + } + # 版本按这样排 V2.1.6-beta.10 -> [2, 1, 5, 10] + for version in sorted(version__count_map, reverse=True, key=GsePackageTools.extract_numbers) + ], + "count": sum(version__count_map.values()), + }, + ] + ) + + @swagger_auto_schema( + operation_summary="Agent包上传", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.UploadResponseSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.UploadSerializer) + def upload(self, request): + """ + return: { + "id": 116, + "name": "HR7vt0c_gse_ce-v2.1.3-beta.13.tgz", + "pkg_size": "336252435" + } + """ + request_serializer = self.serializer_class(data=request.data) + request_serializer.is_valid(raise_exception=True) + validated_data = request_serializer.validated_data + + if validated_data.get("overload"): + storage = get_storage() + package_file: InMemoryUploadedFile = validated_data["package_file"] + + # 只选择最新上传的记录 + # 使用contains是因为上传之后得到的文件名前面带有前缀 + # gse_ce-v2.1.3-beta.13.tgz -> HR7vt0c_gse_ce-v2.1.3-beta.13.tgz + upload_package: UploadPackage = UploadPackage.objects.filter( + file_name__contains=package_file.name, module=TargetType.AGENT.value + ).first() + + # 如果需要覆盖,且数据库找得到该记录并且storage存在的情况下,将记录和相对应的包清掉 + if upload_package and storage.exists(name=upload_package.file_path): + storage.delete(name=upload_package.file_path) + upload_package.delete() + + res = PackageTools.upload(package_file=validated_data["package_file"], module=TargetType.AGENT.value) + + if "result" in res: + return JsonResponse(res) + else: + return Response(res) + + @swagger_auto_schema( + operation_summary="解析Agent包", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.ParseResponseSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.ParseSerializer) + def parse(self, request): + """ + return: { + "description": "test", + "packages": [ + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86_64.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86_64", + }, + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86", + }, + ], + } + """ + return Response(NodeApi.agent_parse(self.validated_data)) + + @swagger_auto_schema( + operation_summary="创建Agent包注册任务", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.AgentRegisterTaskSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.AgentRegisterSerializer) + def create_register_task(self, request): + """ + return: {"task_id": 1} + """ + validated_data = self.validated_data + + extra_tag_names: List[str] = [ + tag_info["name"] + for tag_info in GsePackageTools.create_agent_tags( + tag_descriptions=validated_data["tag_descriptions"], + project=validated_data["project"], + ) + ] + response = NodeApi.sync_task_create( + { + "task_name": SyncTaskType.REGISTER_GSE_PACKAGE.value, + "task_params": { + "file_name": validated_data["file_name"], + "tags": validated_data["tags"] + extra_tag_names, + }, + } + ) + + return Response({"task_id": response["task_id"]}) + + @swagger_auto_schema( + operation_summary="查询Agent包注册任务", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.AgentRegisterTaskResponseSerializer}, + ) + @action(detail=False, methods=["GET"], serializer_class=pkg_manage.AgentRegisterTaskSerializer) + def query_register_task(self, request): + """ + return: { + "is_finish": True, + "status": "SUCCESS", + "message": "", + } + """ + + validated_data = self.validated_data + + task_result = NodeApi.sync_task_status({"task_id": validated_data["task_id"]}) + + # 在celery任务中无法获取正确的用户名,当上传成功时,更新用户名 + if task_result["status"] == "SUCCESS": + GsePackages.objects.filter(version=validated_data["version"]).update(created_by=request.user.username) + + return Response(task_result) + + @swagger_auto_schema( + operation_summary="获取Agent包标签", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["GET"], serializer_class=pkg_manage.TagProjectSerializer) + def tags(self, request): + """ + return: [ + { + "name": "builtin", + "description": "内置标签", + "children": [ + { + "id": 95, + "name": "stable", + "description": "稳定版本" + }, + { + "id": 96, + "name": "latest", + "description": "最新版本" + }, + { + "id": 97, + "name": "test", + "description": "测试版本" + } + ] + }, + { + "name": "custom", + "description": "自定义标签", + "children": [ + { + "id": 145, + "name": "custom3", + "description": "自定义标签3" + }, + { + "id": 146, + "name": "custom4", + "description": "自定义标签4" + }, + { + "id": 147, + "name": "custom5", + "description": "自定义标签5" + } + ] + } + ] + """ + validated_data = self.validated_data + try: + return Response( + gse_package_handler.handle_tags( + tags=list( + Tag.objects.filter(target_id=GsePackageDesc.objects.get(project=validated_data["project"]).id) + .values("name") + .distinct() + .annotate( + id=Min("id"), + description=Min("description"), + ) + ), + tag_description=request.query_params.get("tag_description"), + enable_tag_separation=True, + ) + ) + except GsePackageDesc.DoesNotExist: + raise exceptions.ModelInstanceNotFoundError(model_name="GsePackageDesc") + + @swagger_auto_schema( + operation_summary="获取Agent包版本", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action( + detail=False, + methods=["POST"], + serializer_class=pkg_manage.VersionQuerySerializer, + permission_classes=[IsAuthenticated], + ) + def version(self, request): + """ + return: { + "total": 10, + "list": [ + { + "version": "2.1.2", + "tags": [{"id": "stable", "name": "稳定版本"}], + "is_ready": True, + "description": "", + "packages": [ + { + "pkg_name": "gseagent-2.1.2.tgz", + "tags": [{"id": "stable", "name": "稳定版本1"}, {"id": "latest", "name": "最新版本"}], + }, + { + "pkg_name": "gseagent-2.1.2.tgz", + "tags": [{"id": "stable", "name": "稳定版本2"}, {"id": "latest", "name": "最新版本"}], + } + ], + } + ], + } + """ + validated_data = self.validated_data + + gse_packages: QuerySet = ( + self.get_queryset() + .filter(project=validated_data["project"], is_ready=True) + .values("version", "project", "pkg_name", "os", "cpu_arch", "version_log", "version_log_en") + ) + + language = get_language() + + version__pkg_info_map: Dict[str, Dict[str, Any]] = {} + max_version_count: int = 0 + default_version: str = "" + for package in gse_packages: + version, project, pkg_name = package["version"], package["project"], package["pkg_name"] + tags: List[Dict[str, Any]] = gse_package_handler.get_tags( + version=version, + project=project, + enable_tag_separation=False, + ) + + # 获取默认标签 + if not default_version and any(tag["description"] == STABLE_DESCRIPTION for tag in tags): + default_version = version + + # 初始化某个版本的包 + if version not in version__pkg_info_map: + version__pkg_info_map[version] = { + "version": version, + "project": project, + "packages": [], + "tags": tags, + "description": package["version_log"] if language == "zh-hans" else package["version_log_en"], + "count": 0, + "os_choices": set(), + "cpu_arch_choices": set(), + } + + # 累加同个版本包的数量,并统计版本包最大数量 + version__pkg_info_map[version]["count"] += 1 + max_version_count = max(max_version_count, version__pkg_info_map[version]["count"]) + + # 聚合操作系统和cpu架构信息 + version__pkg_info_map[version]["os_choices"].add(package["os"]) + version__pkg_info_map[version]["cpu_arch_choices"].add(package["cpu_arch"]) + + # 添加小包包名和小包标签信息 + version__pkg_info_map[version]["packages"].append( + {"pkg_name": pkg_name, "tags": tags, "os": package["os"], "cpu_arch": package["cpu_arch"]} + ) + + # 将上一次的标签和这次的标签取共同的部分 + last_tags: List[Dict[str, Any]] = version__pkg_info_map[version]["tags"] + if last_tags != tags: + version__pkg_info_map[version]["tags"] = [tag for tag in last_tags if tag in tags] + + # 按版本排序 + version__pkg_info_map = dict( + sorted( + version__pkg_info_map.items(), + key=lambda version__pkg_info_tuple: GsePackageTools.extract_numbers(version__pkg_info_tuple[0]), + reverse=True, + ) + ) + + filter_keys = [key for key in ["os", "cpu_arch"] if validated_data.get(key)] + if filter_keys: + # 筛选 + version__pkg_version_info_map = { + version: pkg_version_info + for version, pkg_version_info in version__pkg_info_map.copy().items() + if GsePackageTools.match_criteria(pkg_version_info, validated_data, filter_keys) + } + else: + # 不筛选,默认为统一版本,统一版本需要各个系统的包都齐了才能算入 + # 如果count数量不等于包版本最大数量,则说明缺少某些系统的包 + version__pkg_version_info_map = { + version: pkg_version_info + for version, pkg_version_info in version__pkg_info_map.copy().items() + if pkg_version_info["count"] == max_version_count + } + + machine_latest_version: str = "" + if validated_data.get("versions", ""): + machine_latest_version = max(validated_data["versions"], key=GsePackageTools.extract_numbers) + package_latest_version = list(version__pkg_version_info_map.keys())[0] if version__pkg_version_info_map else "" + + return Response( + { + "machine_latest_version": machine_latest_version, + "package_latest_version": package_latest_version, + "default_version": default_version, + "pkg_info": list(version__pkg_version_info_map.values()), + "versions_count": len(validated_data["versions"]) if validated_data.get("versions") else 0, + } + ) + + @swagger_auto_schema( + operation_summary="Agent包版本比较", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action( + detail=False, + methods=["POST"], + serializer_class=pkg_manage.VersionCompareSerializer, + permission_classes=[IsAuthenticated], + ) + def version_compare(self, request): + validated_data = self.validated_data + + extracted_current_version: List[int] = GsePackageTools.extract_numbers(validated_data["current_version"]) + version_to_compares: List[str] = validated_data["version_to_compares"] + + upgrade_count, downgrade_count, no_change_count = 0, 0, 0 + + for version_to_compare in version_to_compares: + if GsePackageTools.extract_numbers(version_to_compare) > extracted_current_version: + downgrade_count += 1 + elif GsePackageTools.extract_numbers(version_to_compare) < extracted_current_version: + upgrade_count += 1 + else: + no_change_count += 1 + + return Response( + { + "upgrade_count": upgrade_count, + "downgrade_count": downgrade_count, + "no_change_count": no_change_count, + } + ) + + @swagger_auto_schema( + operation_summary="获取已部署主机数量", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.DeployedAgentCountSerializer) + def deployed_hosts_count(self, request): + """ + input: { + "items": [ + { + "os_type": "linux", + "version": "3.6.21" + }, + { + "os_type": "windows", + "version": "3.6.21" + }, + { + "os_type": "windows", + "version": "3.6.22" + } + ], + "project": "gse_agent" + } + + return: [ + { + "os_type": "linux", + "version": "3.6.21", + "count": 3 + }, + { + "os_type": "windows", + "version": "3.6.21", + "count": 2 + }, + { + "os_type": "windows", + "version": "3.6.22", + "count": 0 + } + ] + """ + validated_data = self.validated_data + + items = validated_data["items"] + project = validated_data["project"] + if not items: + return Response() + + # 划分维度到主机和进程 + dimensions: List[str] = list(items[0].keys()) + host_dimensions: List[str] = [d for d in dimensions if d in [field.name for field in models.Host._meta.fields]] + process_dimensions: List[str] = list(set(dimensions) - set(host_dimensions)) + + # 主机筛选条件 + host_kwargs: Dict[str, list] = { + f"{dimension}__in": [item[dimension] for item in items] for dimension in host_dimensions + } + + # 进程筛选条件 + process_params: Dict[str, list] = { + "conditions": [{"key": "status", "value": [constants.ProcStateType.RUNNING]}] + } + for dimension in process_dimensions: + process_params["conditions"].append({"key": dimension, "value": [item[dimension] for item in items]}) + + # 主机和进程连表查询 + host_queryset: QuerySet = HostQuerySqlHelper.multiple_cond_sql( + params=process_params, + biz_scope=[], + need_biz_scope=False, + is_proxy=False if project == constants.GsePackageCode.AGENT.value else True, + ).filter(**host_kwargs) + + # 分组统计数量 + dimension__count_map: Dict[str, int] = defaultdict(int) + for host in host_queryset.values(*dimensions): + dimension__count_map["|".join(host.get(d, "").lower() for d in dimensions)] += 1 + + # 填充count到item + for item in items: + item["count"] = dimension__count_map.get("|".join(item.get(d, "").lower() for d in dimensions), 0) + + return Response(items) + + @swagger_auto_schema( + operation_summary="批量编辑agent标签", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.TagCreateSerializer) + def create_agent_tags(self, request): + """ + input: { + "project": "gse_agent", + "tag_descriptions": ["stable", "恭喜", "发财"] + } + + return: [ + { + "name": "stable", + "description": "稳定版本" + }, + { + "name": "43f5242cbf2181dc8818a9b8c1c48da6", + "description": "恭喜" + }, + { + "name": "7d602b6a7e590b232c9c5d1f871601a4", + "description": "发财" + } + ] + """ + validated_data = self.validated_data + + return Response( + data=GsePackageTools.create_agent_tags( + tag_descriptions=validated_data["tag_descriptions"], + project=validated_data["project"], + ) + ) diff --git a/apps/node_man/views/plugin_v2.py b/apps/node_man/views/plugin_v2.py index 7df98ba81..78fb1a8bb 100644 --- a/apps/node_man/views/plugin_v2.py +++ b/apps/node_man/views/plugin_v2.py @@ -23,6 +23,7 @@ from apps.node_man.handlers.plugin_v2 import PluginV2Handler from apps.node_man.models import GsePluginDesc from apps.node_man.serializers import plugin_v2 +from apps.node_man.tools.package import PackageTools from apps.utils.local import get_request_username from common.api import NodeApi @@ -466,7 +467,7 @@ def upload(self, request): ser = self.serializer_class(data=request.data) ser.is_valid(raise_exception=True) data = ser.validated_data - result = PluginV2Handler.upload(package_file=data["package_file"], module=data["module"]) + result = PackageTools.upload(package_file=data["package_file"], module=data["module"]) if "result" in result: return JsonResponse(result) else: diff --git a/common/api/modules/bk_node.py b/common/api/modules/bk_node.py index f0f390b43..9e095833e 100644 --- a/common/api/modules/bk_node.py +++ b/common/api/modules/bk_node.py @@ -329,3 +329,12 @@ def __init__(self): before_request=add_esb_info_before_request, api_name="job_details", ) + self.agent_parse = DataAPI( + method="POST", + url=BK_NODE_APIGATEWAY_ROOT + "backend/api/agent/parse/", + module=self.MODULE, + simple_module=self.SIMPLE_MODULE, + description="解析agent包", + before_request=add_esb_info_before_request, + api_name="agent_parse", + ) diff --git a/common/utils/drf_utils.py b/common/utils/drf_utils.py new file mode 100644 index 000000000..bfb568a01 --- /dev/null +++ b/common/utils/drf_utils.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import copy +import functools +from collections import namedtuple +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union + +from django.conf import settings +from django.http.response import HttpResponseBase +from django.utils.module_loading import import_string +from rest_framework import status +from rest_framework.exceptions import ValidationError +from rest_framework.fields import empty +from rest_framework.serializers import BaseSerializer +from rest_framework.settings import api_settings +from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList + +if TYPE_CHECKING: + from rest_framework.request import Request + + +def stringify_validation_error(error: ValidationError) -> List[str]: + """Transform DRF's ValidationError into a list of error strings + + >>> stringify_validation_error(ValidationError({'foo': ErrorDetail('err')})) + ['foo: err'] + """ + results: List[str] = [] + + def traverse(err_detail: Any, keys: List[str]): + """Traverse error data to collect all error messages""" + + # Dig deeper when structure is list or dict + if isinstance(err_detail, (ReturnList, list, tuple)): + for err in err_detail: + traverse(err, keys) + elif isinstance(err_detail, (ReturnDict, dict)): + for key, err in err_detail.items(): + # Make a copy of keys so the inner loop won't affect outer scope + _keys = copy.copy(keys) + if key != api_settings.NON_FIELD_ERRORS_KEY: + _keys.append(str(key)) + traverse(err, _keys) + else: + if not keys: + results.append(str(err_detail)) + else: + results.append("{}: {}".format(".".join(keys), str(err_detail))) + + traverse(error.detail, []) + return sorted(results) + + +############# +# drf crown # +############# +class WearOptions: + is_unittest = False + skip_swagger_schema = False + + +try: + from drf_yasg.utils import swagger_auto_schema as drf_swagger_auto_schema + +except ImportError: + WearOptions.skip_swagger_schema = True + + +ResponseParams = namedtuple("ResponseParams", "data,params") + + +_DEFAULT_SETTINGS_PREFIX = "DRF_CROWN_" + + +def enable_unittest(): + """Call me when you running testing""" + WearOptions.is_unittest = True + + +@dataclass +class Config: + """Config for Injector, control the process of injecting""" + + return_validated_data: bool = True + remain_request: bool = False + # sometime return raw data instead of serializer + skip_out_cls: bool = False + default_return_status: status = status.HTTP_200_OK + + +@dataclass +class ViewCrown: + """A injector for injecting serializer as dependency""" + + body_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] + query_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] + out: Union[Type[BaseSerializer], BaseSerializer] + config_params: Optional[dict] = field(default_factory=dict) + valid_params: dict = field(default_factory=dict) + + def __post_init__(self): + if self.query_in and self.body_in: + raise ValueError("there should be only one param between in_body & in_query") + + self.valid_params = self.valid_params or {"raise_exception": True} + + # Priority decreases + # 1. config as parameter from decorator + # 2. config from django.settings + # 3. config from Config class(above) + _config = getattr(settings, _DEFAULT_SETTINGS_PREFIX + "DEFAULT_CONFIG", {}).copy() + _config.update(self.config_params or {}) + self.config = Config(**_config) + + # remain an entrance for custom response class + try: + self.resp_cls = import_string(getattr(settings, _DEFAULT_SETTINGS_PREFIX + "RESP_CLS")) + except AttributeError: + self.resp_cls = import_string("rest_framework.response.Response") + + def get_in_serializer_instance(self, request: Optional["Request"] = None) -> "BaseSerializer": + if not self.body_in and not self.query_in: + raise ValueError("should given at least one serializer input") + + _data = empty + if self.body_in: + _in = self.body_in + + if request is not None: + _data = getattr(request, "data") + else: + _in = self.query_in + + if request is not None: + _data = getattr(request, "query_params") + + if isinstance(_in, BaseSerializer): + # 由于传入的是全局对象,会残留上一次请求的结果 + # 这里需要手动清理一下 + if hasattr(_in, "_validated_data"): + delattr(_in, "_validated_data") + + _in.initial_data = _data + slz_obj = _in + elif issubclass(_in, BaseSerializer): + slz_obj = _in(data=_data) + else: + raise ValueError("unknown serializer input") + + return slz_obj + + def get_serializer_instance_by_request(self, request: "Request") -> "BaseSerializer": + """Get in serializer instance""" + slz_obj = self.get_in_serializer_instance(request) + slz_obj.is_valid(**self.valid_params) + return slz_obj + + def get_validated_data(self, request: "Request") -> dict: + """Get validated data via in_serializer""" + return self.get_serializer_instance_by_request(request).validated_data + + def get_in_params(self, request: "Request") -> dict: + """Get extra params before view logic""" + if WearOptions.is_unittest: + return {} + + if self.config.return_validated_data: + return {"validated_data": self.get_validated_data(request)} + else: + return {"serializer_instance": self.get_serializer_instance_by_request(request)} + + def get_response(self, data, out_params: dict) -> Any: + """Get Response data""" + if WearOptions.is_unittest: + return data + + if self.config.skip_out_cls: + return data + + if isinstance(data, (self.resp_cls, HttpResponseBase)): + return data + + if isinstance(self.out, BaseSerializer): + # 由于传入的是全局对象,会残留上一次请求的结果 + # 这里需要手动清理一下 + if hasattr(self.out, "_data"): + delattr(self.out, "_data") + + self.out.instance = data + _data = self.out.data + elif issubclass(self.out, BaseSerializer): + _data = self.out(data, **out_params).data + else: + raise ValueError("unknown serializer output") + + return self.resp_cls(data=_data, status=self.config.default_return_status) + + +def generate_swagger_params(crown: ViewCrown, swagger_params: dict) -> dict: + """ + assemble params for swagger_auto_schema by crown + """ + default_params = {} + if crown.body_in: + default_params = {"request_body": crown.get_in_serializer_instance()} + elif crown.query_in: + default_params = {"query_serializer": crown.get_in_serializer_instance()} + + if crown.out: + default_params.update({"responses": {crown.config.default_return_status: crown.out}}) + + default_params.update(swagger_params or {}) + return default_params + + +def swagger_auto_schema( + body_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + query_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + out: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + config: Optional[dict] = None, + **swagger_kwargs +): + """ + Sugar for simpling drf serializer specification + :param body_in: input serializer (request body) + :param query_in: input serializer (query) + :param out: output serializer + :param config: initial info of Config + :param swagger_kwargs: pass to swagger_auto_schema of drf-yasg + """ + + def decorator_serializer_inject(func): + crown = ViewCrown(body_in, query_in, out, config) + + if not WearOptions.skip_swagger_schema: + func = drf_swagger_auto_schema(**generate_swagger_params(crown, swagger_kwargs))(func) + + @functools.wraps(func) + def decorated(*args, **kwargs): + new_args = list(args) + in_content: Dict[str, Any] = {} + if body_in or query_in: + in_content.update(**crown.get_in_params(new_args[1])) + + if not crown.config.remain_request: + del new_args[1] + + original_data = func(*new_args, **kwargs, **in_content) + if not out: + return original_data + + # support runtime serializer params, like "context" + params = {} + if isinstance(original_data, ResponseParams): + params = original_data.params + original_data = original_data.data + + return crown.get_response(original_data, params) + + return decorated + + return decorator_serializer_inject diff --git a/config/default.py b/config/default.py index 9634e0e0b..64918bc46 100644 --- a/config/default.py +++ b/config/default.py @@ -262,6 +262,7 @@ "apps.node_man.periodic_tasks", # 避免 subscription.tools 循环导入,故单独引入 "apps.node_man.periodic_tasks.add_biz_to_gse2_gray_scope", + "apps.node_man.periodic_tasks.register_gse_package", ) BK_NODEMAN_CELERY_RESULT_BACKEND_BROKER_URL = "amqp://{user}:{passwd}@{host}:{port}/{vhost}".format( @@ -830,6 +831,8 @@ def get_standard_redis_mode(cls, config_redis_mode: str, default: Optional[str] if env.BKAPP_MONITOR_REPORTER_ENABLE: monitor_report_config() +DRF_CROWN_DEFAULT_CONFIG = {"remain_request": True} + # remove disabled apps if locals().get("DISABLED_APPS"): INSTALLED_APPS = locals().get("INSTALLED_APPS", []) diff --git a/locale/en/LC_MESSAGES/django.mo b/locale/en/LC_MESSAGES/django.mo index 5f0987a6c29a744ae50afa4cd51eb4f3cc3b1ca4..6245f31c089f5dec1e0158b0f299bd84e366a157 100644 GIT binary patch delta 31391 zcmZA91(;P;-}mu-sG+-aXc%DVp=;>w4(SeQ2|09kGjvIVNQy{zBS@nl9f~4dzTe+j zf9{L-+1IuCtggM*KIhEs^L(5i^ki|6|F?uuW_etTqk3LyOq|Q}CIoq2>?TU}yu#mj z-exR<74Zqi#ry+2F9(KW3hZu<#dO4%S^NNIA$|i3VdR0H7ly?!z32J8UIggVn~90> z04BuC$e6sxm{32>Vw=MnxlM?rR>*^;rGnhFs3jKRw1R`TG zOogQ|1vWMNqB@>}!5D#A@h41zcTn}-VmwST#F-iM5-*5pu>)4X(WsR@js8dkE)YnB zm#_prMm3y`(J8+qM!_(elT`hl!<*!6N>&=)JA0f}$i$2sn;t}f64KyZ%ZN1GHJApq+e1+mN}vYV2!pXRX2c1oa+^@)_MrC0RaD21QA_?3 z^+?{M9z`zy2)87CP)juqwb@pqM*f3&7_}l-P&2=0>G4OpnPx<7*8G?n8=+RNuf@Mb z?U4x71h!)`^dGapE%OcP7{(vvDr80tqzr0?^-vYtpq6?NYQSSKI!;4%5MlB4sCwH_ z$NUfm<8x$X{9dBbuA#JM2x{rVFbY=0$XE@vDQlq`YL8mlnW&j9Mh*NY)PT=nFy28u zk|<+byXjDyHxJgLf3FGwZHoCAgy&H+xr$oyKTtD$X1+(A>!7i2CDLLL@i2^vB~g!{ z9IB&M7##;;IUI>9e;o7DzjvO1mN4=-H{&=Mop?f2dMYf6nK3=KM0GG6wIYjAE3pmJ z;XaImw=f$1WxllZ52%%mIiB@bfp`RzkQ8-nQlVa{IZ#Vk4zpuJ)JhD);y4M5;R#g7 z(I&V}m<3g?Dysd4sFi7f+Wl=Y7S5i)`YW)EgqXMyRq#jbj(f2>hE8PmI2dE#GVF!x zu^pzHL?<{3wfSyhHhhK}X!6N=)}B`%wSwPaH9R(%^;bfYDXxP;n1Oh6Jc2{9Csv&5 z{1NrYa!hjrU5BxWZ$_=u9@NU6Hy>kG;*qB_3Cw{SSQD&^Bm4yTMD;GCMpSDCAGx>? z%i<+0f+=UZ7gasfCK`&7@Gxou$54CbF>1#DT6&&Yp4Xaq9gK(DFd-hsT z1SY{csD?Y5eNh#M;{u$8>LA}-_pHNFZ_av{AA4hRT!oo%pT+NC81YZ2_6p9EJj_Z2 z)L~Q9v+9UBa4J^A-B=!DM7URPP1I=^h$^=Nv*0<@0A8a8lw-cL3TkCKp$0S#HQ|k@ zPgw7uWjsX5;7hbmuUk$aU^#1h0CVGW#(mGB;FK%tA>fC`}| zRsl7E?x+b3SZtaIU<#ao`fOQg>D!n2U4ernXhi2M<0)#SZ&3q@vD`JB8dDOlh&iz}Dt`*L z!ethZvcj!EGSnjpMwQErYBwAMo84~-9nEi01;?UBKHpr8$%t>W{L`q0Z=p8RGgQM- zR=VAv*vy4$uY%dw?22mN@3+82%a~`bH}|4O`U`3aAEGKo`p$J24;4>`>L@2B$5NKw z2=xftq4vZ`)W9cUC!PNV1hn+gR=JD}s0Q<*-q|HlGoONLXccC}BdCrZqXrmxHJj1L zAd!Z>hHG3q`^__`cCKR@{1=0D{u8Wq1@oDuP&27!er4&sOh0O=Cz=be6!G;~9-pEn zkZ+wkEu}Cr@yZwttE2MkN&5F%5zq>BM|C{jTxfoes&Ek1z&VRwMh)aHs{RX9{rKyh z8Bz7ZQ1!~9%2&Z~Y=V9*;X(ph!tJPnr!Y2NKs~ct7!w~^`U^|R9GMEol{0z2+EL5wj=8B7KNC z9^(=B&$a^JS;7{JA4D~D4%P5w^D$~hA2Aaq-Rw4BG1QVbM0MN^Rd1j<$>?#l_R3>J_kfG1Mllff{I6)PyErdR&1y@CatW z=a@_9Kgm}2X;l_AqdurjGz`_jRIG+`P|Gi(-uJuEBDsfz(3{Y^k{#)xjonH|kY;9JM)bTl_t$ z{X{$5z_RS%{3juyI0+g^9n_MwMm0Fd(#KkSF6tG#(&A6dH)hnGZsii9I!cdfzW@ef z6-)1en(&aFtiLj*l8_8HV<9|lao>+_ASqBwpT*+2&Elv%Q5o}KJ5)js?XmPbsB(`{D--=EXA)HT3>GhFhNJ4&M0MN; z6XOijO0P!M^Y0{}nVqzZiS7 zU5xy*%WvrPd#wmeA)^bb!}nGo@g7$&Efyd>8|KI6*cqpx2KE6pqZoVLm(UcbnRP^+ zib1Gyi%{*XLrvgTAl>hI4*~%`J~0&;zI`@R)DnguWAbWZ2*%mZS8^m~4}U=I+ItvS5ez1t_@EnkLDY++IBEbL zF+I+}T(}K2@H>_s{g7)v1a*2UpjL7s`qjWb0vYiMs)2ZiT>~Xh18a?%;Sh^2L@nWu zs2S};&HS9jZ(96M^BrpQ1s!pZAT_Ff?IZU2--rY?&&{G549LQM>$_#qXN` zpjPY?>V1&%m|M}Rs3jj@@eNpv_;o*lA_S5hcQdbzTGFglXndbF;bMJdY`x?^|MKMg@|638zQqDzH+=0391h&C9sE(SQbxYP26(5co z*mQFvs+|+4&yq)21%2n-Ua5u}U;|V?Z7`G0e?Lo@Z*Ih3()Xh({DErtndQel?;1#t z$}fO9unfk+Zk9j5oQQf9i&2m05NhBzG4TEWttBM<#T5+2oD`^wDmWD5;Q~~L>rtEY zM~s2{Ed2z=A$}PX;yv>Xs$Q%M?zAPxio|nXVEt9F4+&Xu3@W|}Rp9`p!Lt}0|3nS! z6>2~kFS_#i%?hY`jZp(@kGb({%U^<8fi0-|r!KPo3fv$;oA9X>O!lj5IK(Vr)f7HIrqiW4p=XH&N|AM%DMOyK;$8@nFoZ^Piu93N}X_v)-tIOh*-5hRJc8#m|_x z%@=0S4Oc#;nG02~H0t;@w)hZKe{-bH{{{k@!9h%g=TIF#HUGmh#1r0hZD2vzS_^P%}3HLYx#p!*-~du0?Ii zpUtD@dGk7E_IMFlJl1VDuq3E{Qs4HwCC^Dh1rjQu8lGYw%Gzh$?t+?Fv3ruE`b}E7xUcVrGTw50UkH6V=>}?T0Hw*H_&33 zmh>7Jcm(D^EKB+ri=RYI>J7j|I{y<1X!9+>`gk5|W7Y?r*FyzSugXWL6?tg} zJ#??)6qteZvZ#jJV>BFqn!r$0$Kx$N2cr{T8c65)uMGtF+(*se5UQbzSP36kJkKLO zNQjq3Rf>fqZaZnKU@4PXgI$5ogUzekn3X+FmE^zXeUP#jY{bw3&#pf=HL z)Kad)_;}RfS1_3P3)IY${OJZ%81>AnT6_vtAifL(j|R1Q-(qY`_!sN1O_h#-5}KlB z*3sgFP!-2wN}P+D*%nl}U6>wEqaNWi%!t9y+^1o8|AugybK$_zm+Bs@^M9dmk|i2EA|t4@C{QBB%fEAcp+4K{$T_X6PSuxs+Fh?c3S+5c^fsL7Z{Ab$gW%lRC;05N|wih*a9_> znWzCC!{T@s^*%`-B~Z`r)g+LTgl4FY`(sj^goSYhrp246hCZ0dqxu5Rx&Uh6bx@xH z?N9@lgfs9UR>FqSe4dW+64XEsU|RN{cb$Mbe1{rgf*{vX4YM<<;%L;!SEC-$CDg0+ zJ_ci)=q^7L^AT@`+7nZ-5biRcqaJxk4BDlCuL*$=A3GNF5?>Y5E$LO%%Q3G5a+vg87uz>_E*%4HOZ>?a$IBplGL%ksz zpjKdz#iyY<*p8}q(Bd~x6L^H_@PnnNj_V72LCKDqQ1!Tex4G((pbEWF$7(cc)BTFt zbT69NxBgmV_=nzgZgfh^nLd@uMc-pKgJ5sQ17jEP+>1 zGfI}o%`g|LVhL1-wJ;4fM-6C@r4Ki!m&1%6QEM16=f#&BGS+GMv-FPLa4-HR;@^@XK6M#UDWP1+Xyg$XPo z&>1hIW)hytZLa#LJDt?Q40gXqEcpmE2yw=k9V`}2(EdNi`o9z>-!JsrgFA63=J^Q4nQ<4(3 zx$~J7Q9nqUrQ!T*sXLOOFOd^b&upi~&!YzL5jCL1Y2Bm9jCvu}#0J<0wX`QO5}rqO zcoD1NBh)~`(z*N^sDak@6VS*Tpq_P0%!h-mz$VMziuw@QgL*N&LcKte1iOJ1LM?eE z%!w^5eFEyyEkLdCM%1JG3AK{`g9Ks{xP{s@Pf#DnsnWX&VW_1li)yGQ>Y27e)$5De zLqkym`w{gic?C7VXc^oJW<#xDDO5YPkd^m)t*yXl)Gl9$dK5=dd*ff!BZ`*MH4uX8 zxEN~5D`0i3hV3x|)sZig8&E>jjPs%%NfEOK#@6|7L!c}fU!$JURx5A{b?$GY9!-+W z))4AFQ3Q3|TB2q&8r8vE)OW-U=26s&-A1+Z5;d^|S*WM;pNW7fhM|s8Rn&+Zqn5rE z>QraZbdY5SlWoQ&mg4yybOEP!`WE0rR|%{UXPe0J1n%ZGKbc!=L^ zwy7j&lkG;G|68b$zP0pdIoyatPz{8kKD0`s9$7`y($=^1CKhjH`R!51vkz)7j7RN- zMScQWfn8V+kC>@)`U1axzs69~=b@hIF;vG_P%HC0>f`zzs-29voVhV@N>Jr0-~z0M z-SH7-Mt`GFH=`jqn}o4A4%6lKd3A6NYIA--ZMFn?e1Shimc!P>hhjFogDo+3Ubmv% zP#w%e?WMhV9RI*xxIUlH8>I7}E`K24%}0$iSpm1H(x5h7PSmC=g?hoX!mKzHwaHdv zUi=LkWBh_XZyI*SM)(1hUq8$j_+P)rp!UdV%&qeusgOJW`B3lfei(uSP@852>f>?) z>X|*jkr=tKJNIKzn>GT=;18%r@dEY8ydv&BJ!hNAXZ4XN|rfq*(1irP%$P@8EE zYH1gsDjY@);5XD)uXm{9o4lwSKz`JURYiShwLs3%HGtHm+=>)Kb=(m3YVL_Sa4rVpAppM&e z)C=l3YUwXv7{)5+9#J@ICF`U1oPPj;qXdSdo@JHtK5rpT#U&V8!5x>gSebZ~itfcy z1GNboqh{3K(g&kfW(;azvrsFu2DKtPQIGB{GC;rg7XghZOC|ddLM{1l)Xe9gK3+ed zRv>O=w^B(_hiRx&orAMGvWDV*V?m)GF6oYjBZxT=g_fP|PfjUl6 zt2&dTc54pQOe(+95_ZLTsM8Rwn(HtaBNNYqqp$#KLc7qfSL+o5dQrTy zj3m`v#r)XR$9F$$NB-g(?%7AJ={8S%R7WXL?c~QySRb|Y15pzjgL;3=M(u%3mVTrr z=U>m}R}!?O4^T7zfb}q5EuYr`+oG246l%sdFe=_bZN5jS4*x=}#Cy~XgKE2YX4C-k zqL#i2s(j1ZoPQmIZX{?R{ZKO+i)vsQs=_|hqq&4C_Wj>E9hPtiHPc^E4gF#9x2TR|)U~hYsD{F@F@~dNxByju8|sl9 zMm>TDr~$k}O(=3bpBIe&6a>_8QPk$DhB}`uQ8Vu>!9{TBh(}6it4a0 zs(}&aRMfLyX!)yAOTOLWdra(F8>T`V@ z>e+vfA$SUN<7>>1S$SJk!luZFi?;$b;2WrkJVrf&PuLKnH{<-P!ns;K-nSPJ`_J1qaTpFn*w;Vo1;)mI0H3+H5T8GI_H;ByZSBW!gOD`JyIRDave|uo{Tzv3oY*7O5iIJ4x%<$ z#+GiT1yP$P9MxeR)TZl~NJv$+@bf;o+)@gb^%?5%xXODu(XaS3YXzn}*A0()TWHg3j)QJZ!-Dt|o|z(c4{ z!?zgt`~OD*IyR}>x(eA)E0Gs9(z2Fb7Xu%=sF{t%U|fSb&nHmN{vm3UMQ!Ka2l32I zsP{?{RDX>y9`7HoEdgz^KByPSAag2erYljK?=os{+{M256zgE?_AY%Js-2&45*|Wz zSf_)_?}<951F#~FM1Oe#rwDwF2|K!TIRbUeW@26Z9{qs5qQ#>KaFCBF7C(WWYiuwfvR{9i{Nvtk2$-# zV>KMr;cV2**PB~Vk75^=#LK8-n6#T~Hw5(v%Xjm;iXBPN=IDVsJ}WH#3+5vp)ZKlJ zE{cVS_r(0T23z4x)Jl}<;m-MBRJq-l2VbDt&-}If@!AA)6Cds;pbmdPJ=(2`4{3j%!7f4#v%!{LDSQT}CM_T@J3?;tb(*MR{#N&PA238gI>|3EWZ8y{-Sb|M( z6*j_11Kgu(ihgaPZwXYy>8KfAL+#GLQ4IzUbnk_ns2P?(J&MMtm0N<^W|t?5m*p+qxQyg)U$n$8d#lgoh?uk z>5MtikJ^OmQ5_vbwR;t{Cq7^nj5mbyuV)iB#2t$&s7KHm3t&&wUic2x!FJTMJ&F3I za~HKDF=!_nW=3^X!{R+rk1_&F;|0_J6AyL$l<^Z#2enYoswHZs9jw6DmalRe=oHk< zmZQEx?Lw`@N$iK0Q3I?o%yrZOwE{Cyn{+K|fV)wr!T&pfFai%y14%pFjVu(^VF}a= zqc&<_olq~FUZ??VMXk(XR0r2k^`4?$Y~BbL4@H#=w|GNjrTtzP0xCEV^=ww7mh>mo zJO47O;{+q!?oWy8pfc(m-yF3915uA|G-@ETP%E(tb-MOh{%@!i{1fBp`+t;CuH(dJ zR-8kDq8N_9U{*{r+8xIds5e+MY>FLF12~9U(zB>jaTzs`yB2?i`uZJtj9ZE57+2>% z4FP>vlDd*LEfM+wHdS8*ED%49`-SVc@QY*SDJT7!C& z+fgfZ2K`!!n*^fb3)Jp>huVx;$N2*PRZJ*qK%G!a-UszrFb!31KI)Mz!}_=f^@>eA z-nEwjwO4AQI&O*Dls(3C{*^Gw3e2(Ma ziYh-6butQW|vU|`WH2zkErvXV3PYG z5{B9%t*`)&!&0~hwL+gT8%CY%1`>*T)Ybh2v`agpmSh;_z&RM$RHy--MQyI%QA_&( zL+~YP1%s!!^xUX&B``OZN43)z_2D+mT!(rj{$B}{C2$|raL80=VbqK(pf*`8i#J9c z*EXn4H4W9_BGk%!hkAAI!Y+6N)lsEsZX%6Q6Yhj1bpD4E(7E4d1umdEzJnV1pQugx z){H;h%{VLSc!iKo|Grodq z@F8l+-=UT+@@%&W6Ju@Sc~CPLj;c2cb>3H^X1ELW#pMisi;3s>yn#3!hhv<%uHC8V zSB0en)ZjMMQXWI??nv|8Zq14sc?HxXsgCNfvBkTa15q;@jhe_z)C9gm9nT$D8PB5X zr-)=3(pT^R}KcQw&YN>0uKI(L|N3GDe z7J4}dHPC-iyFTN0u6`%fBN>Bw_A^oSSD_x+77YCNKYIwM;9=C#{DOK$f1ySi zd6oOzPG}}W?UB@|hQm>xiuFQJE1mXU(^66S^5H02dhy(MmJ$De2QAZlxy6_aVFHJtc`*H{;xFw z73_){Km=CA?@=ARF@x5+^PLnmuuQ0DpU2`g&8DbexhH=T;~QYFB4NHCP7q$m*arUw72z>u>4vQQv@e;0!#EIwkGa zyHnO}z2D7b5eZt-wWtOUpa$?8>W%gmHPfICuH!7IffPc$Xlh|cT#4FCaW}dZNQIhU z2GpL&g&IIUv#8$!;izX+#jJ;#VGGoX^gzvI2x{r)Vo6+q;dlqtaF$K>9zlIguZ(=m zdJ`}^9>=`+FKSQuGk@=%MN!lM+M#C916$)@)RO*=voZ2!w^9+P75V}7=uV@K?QPVq z{|~hZGyLGb(0qm36BDp7PDhTd-#bY_Z?N~M87JK0o@GYVamtC6us&*li?AuKMxFa8 zTit0XfGSrG3*sbHM+Z@l^aJWtCEw;wK{iaR&;R-ag30KNU2y_xsoq$6hVAY{q%7(M zH3J*sQPg`N-46F39+gM!iS?+JIE;E^cTg+x5w&8$J6(D)41E8uPar25Z7~?9p^o7X zs7G-O+v2a-5{v%m-hlJ55%CjP4m0j@E7TEnyn3NJT!?z%?8L@+6ZOh3^b_Y_BW*}P zGwF-Un2&m+{e;1I1-04U;(9E++ZXu%igyol5O4UiD>nkQaw{x;28$4XhI$0q_qbzM z9t#lfw}er1e~4N%AME7Xj;pxzU+ z%;n~0)C+7c>Xe*Ey$|l8+W*^6Kn=b@JuBZK`*1;>>u@ZKqfn>gII4kbr~%wF|FQg! zsJ#>Su=|rsCe)++1~q|MSO}M)_Kg1;0qxFoN8Ba}M;((6sJ+o2^#YoOdPFNwGyMtG z;BnLoU65TVW0211$X*>ec&~^6B4;eA+dT3Uy9% zqBciuRL70Y4yc*-MGbJ2In&aYqsni=7`WfkPny?JEAa$1;g1;j@BgBlaSf(JH4tj? z;;4~Uwe*%4nRs{9rtO7#mV;1_ZV_tWdr&KK1oPl!4D5}wZUXU8n>*cE&cDuKYZA1p zJDR;wn{5cH!C9!8FGOvk4X9V^dE}UT74;9-Xd2r@_}fVQ@sa!UwO*sP%V%L_Z_)q% z<(Yb@TzW=Jxf4)ep*5)VR~F7i`UuJvu<~z->nP>M5Xx*KeHI=jK1LO};?m1}2F0u2 zJ3zXwI+UAXZEF8{)wBe5Y1a9ubQ9U|-fsM!_;!53{rO5{gZLc3PuXmwCA5Ys(B?Ag z9iTuO>$v9^{X8d)gW~zyP^pg8sLbHiT=M*JnF=GK*u1|DO2iw`2dVjzcKPlWqWuOzpwo0RQIxzTu;JYCW8Uq1z= zTEc0pP&DBYb2WuMg_`IV1xp86(ruD@L0@CxpUl4saKwkmJp6h z`c>k|h)wMpcN4J;|I!<~i;l;-iQcq2L%Q|3q32 z!n!^YZ^u1{GFQ0ca{u3}C~2puA7mY`ApQ&CBec1#CmnCyO&sPBpB&9%68c$*kD%nT5qm=)X`^M)Ararg=A0`$bWF1bUjS*iA zz7TCxBX0m{^)&P@6qrZi0y-Mb-QGGpPx@sbfiPXeA@>#1b=4w`FH2qw%8Vm@1O7!?Kh*WkI#W0?`H{%e$9hB38`8#T;_rxmO*kX@ zv$b`~k@*9OKhele{Cs7!KxHcFds{;)=$cOa2=@!>MWW6>gnN>%pXU__>uOEjGdgKw zb$+MZ1@h96=Ob-A`CS5oX8i|{al>Z#7a420qjA3o6yWO^l|ER$8rM~wczYW9kMJDJ zdr3Gg;Yrl%!`CtzIu0g2 zmv)*`ZnI1FUQ&J?;jEh)TKwb)8=Kg&3rR-yF{rTZ*(pTcQ#J}OrVV!RyZSm&{ScSX> zD zCVe&GDpq$k?ew!U`^f);+kb|{dj#@PxDy3e5Y{ifR&*4N!dXb0Lzx9wk21qa*R`3j zuF=Fx;#lr|gdcMEr|fv@Mx#t=;(x24UG>T9VRb)g{<|rlD9?wtVHdZ0j z%t)cS)L2dYFeRUH*QJq9YQ?U?Uo_N`wCI$*hOh7_c?qrTMl09f;%g~WnzUgySnWS= z7MaVfbN*=Lr6n^t@$RH2rQ}2Cht$J0O_^yFYBxz@d1QilXjId z<%#PmLp&?>x}|JG~)3za0sK5(T)PT zN?|S1r&Dk+702K+)Rm6BeuVpz)|PNpOP^znW+bg0b$gNiiL^e1f5082|AM1vV=#9L zZhr`wlj;19Ro+ISbi^|fuSS8cx2`TUev5lJ@vXRsdmHyX z?ls&cN&i5(v2?7zD6XR2ch>eDOiF$a%IWdt{rRdvgFy^d*9gM3s22s#aeuxV5pF|dDS1<^Y#eN?BlXc5n@C4l2oIy+ zDeigPEl5jGrJ0n=OLzh{#Tr)UH}Z6?B)*SyT_YXd1pMC8&SE?|JVgDyq}Q@^|3e~k z)ffrQXl#HrNJ`*3N&GwVH&F2mW&S4Io3wm1_8;*O($f*Yj*E%MB0m9lGvd0wp>p<;UCBZ=Q5o|pW}gf~#3OrRvcpRJBX18k(s zAJi?2!K4iQqRjWCr=qQmHmGaZHjvNr-}+@Mgm+Q!Z!6fD#vT$+P3QeBO~og2uOVI6 z3rqivGO>u~py3D9yGr^~%R5h=uBX&lOI~S9_xmYa-ZJO9GTvX9z|xe|jc_u{>r4Yx zh`+}lurG}@!2uYDMz52enl_J9{w(3Lr2S-_<)ut(@M#P0(se7hz5CuzN@_{cg^R&FYvu(;Zv&wZKujCGcQ z{AuL-xocb6*W^bg{2li$s~bjII~CG(*HrjBI-QD#xL*ZE$q#GN8dzg3@CE4$RG4cf z9WEg4H|2BRrmaMj&qTBx<+9-@%GBoWN4N<07}8Ty#=nF}ME+&3ujFLx{oT;e{-os#@u8rQXucx;=wOW}1&;63SYsIRLS;Y^f&@1pfwO0Q-Dg1x0lQh(v^lCQpABd;0#$Q|ELgZ!AZT0KWXcTKWr3b=6!X^Lj z-^!#H;@)lLKW7}H&P>&H199s=&;{i%E}5xy_W@%iTbU z)LqBDpWxrrX^NZ3TSVG!*O`}?^qSmBh#w+P?;o!%>?Uy_PUF7AU6}i4t5hmbkOVG2 zjphF$uC!-X?gDKTRRULj?kki#NZ!*x3VVofcj~XfP}0JfKzyD51Xe)gA#o)IN|HX3 z!n!V*xk(ElUXwDqI^aV(ok965sB0i)ballir0ZHn{Y@_Dm8INs?(|mPUz^OC*1;eO zjiJ$@R2)VFB`9>Bw3ehjp-er}qLcOw_cZQoHjoI)453a@;;SgrkoX^zuT7cZq&Fcg zCgG{v<7`lMDYH=L|7Q|&kaz$;Ux#f#c}Tcr70Oy?&B-f3+D6LHrs2_6=@Mx}$!kmc zEYgb;|48@@`ID&o72%TB#y8}rB7cU?e@aU{M!}qfBT?}4)r{~>?ir+4v~tS3!d-+$ z>eAsH!e85f)0m6N`;&S*iRY)Ix0K0B{RgB?w=!Rm7I^z;(^v%QMXWKUWhQ^Il__lLQz#Re zdk7shr_2ZHbtW$gX}@rPO?qMKY$M%^8T`fdaMQ{yjr$)@vn9!_J4s`W9`)eojrW&~ zd9Y#6mRIXuM4It>W2QK{@6BKIXz0-3G9B7=@3STGrmj(<1`@Um*)ln8Q11JSw%=bg za?5|`=f#dW^#1JcAB-D&f5G}KvwnLKHQL?1OYaZcv}O3+s?lR@oA7YcEXzFk^i-5J9X`ot5f&jKCL?JC|=vQI`W+7^?bP!w98$tNB`~}dbMiX&W(6S^7_6+k$r7< zBV7Vg(ka~~C7^UmNl7CeqA2?P z?{__%%Wq%T_P3tZd#$zi3>?onzb%aYab9fy&7?7AdR((&dR|&QnaA_`#P+%s{-^;`5k|_$d~_;6a{O7^`DO&+~gDNQ9BG z43pt`Oo|VYIe8y2A9{m5FC6k?J8XlhcLY=774wzl#~b3x(;?5}<-_<`)T|KD?|F4d zB%z>%6?C_FFlHftBF4o{7!UVg5FW+Ec){X(s0qEYIL1)V3n5N|YM;X_V3xobjPF$< z5gn^z8mx_}u!}hwHSj_V#Wk26PhfIKtyB(Ms_oF6o4RhfG%TF@QZABhbc}dg&RZ&~g6t#j~ zF#-pm`dc@Q4sw(DiHvmk7FCdRxa%+sb=pgz@@t|d*cn4{FlNSisCxTR^-iPC#$(jL zA5cpkbA;QH_^7QY=_jEj8HHM^IjF<36E*Wg<|WjMJVHJBJIfCp>7KM8>ado_wAdN7 za-%JtggPT@P!DhvQ=tEfC0?3wM!5-vq8db?CQ=vmgzZrc`=FM30&2puF%B+94Y0=I zy{L9aP_Ow#48>@p11sbAGLuk8dClUerK^N7urWr*7N|qn8r4yM)Y2|PO>_flB~PFx zd>cdY4Qfl0jdA^kqYiHwti$>9nvu|<_ztyXcTi9A7`5cDQ4{>vj6c@B*C|jdkr&mj z62`<@s4ZxK8mJelzX@0YzsA^j74tK`cZYWkSJ3u1rNz-urI{$TMVEKKbC#`Ra!td2q2%+E+@ zkGf!PoPpYk{a6u$=D1I8J z`8rH5NN57d=DGK}1U4gXifM2w>dDWdmi#vAEqI99^H-PM*@VbsT${ zJN+rle5n4ank~#8sQyQo{wY>4-`rpxLe2CNY6*WsHH^934VV-aXF?4Wjw!L6kzn?h*wbWD0g;<(+16ITrs0WBx z zt{AFaB~<;ISPomFR&WvewS>D!sNxw^!=Et@-o<$M)Z+J6?yYt85}2vY>}Da1OS`gI z9jl`zINQqCpeC?mE$bgd;;a?iKvjH#+N+Nir&;In^P(D-#UN~k8mJTML3){Et$cyG z#@t~ZMzuS)j`i1}x^4yknnCN`0I5(blg;8Vi;H6h^2=gEY-9O7Q4<`1TJou=ffiZ3 z#^T-PVLu5i^=Z^Vmr+l46Vu{v7RTA(I!cau9W$U-su-$W4OF}OW^0U3+|3+>iHJv| z>dmy+zr->&qdMA;>iDR674<|1r=nh>j)RI?1b=U&cF3KEg`BN;OZ}A4y#P%Zn z`n{h>sG~cm3A{%w**~cFKF%gL^X#aBilFLMz+kM7s^7}Wqs;!O35~RPmbnzweq$hJ z|96l`M8;vOa1qt;4(j!KYjNVuu3=_Wc`no;Es2_FbJR)=#*8=vbK^G5gm*9uV{CCh zqzYh=-v4$a5@JuRfqvAPIB8x&y@oeY9X&=pLC{t=@nF;nW=5@0G0U%nd59ZXJQ9-- z&%mU(9Q{Q}>?BbfU!hha&o(!a5~zX3nNv{%%r=*yKCPQjhw_-kcTofUg~>7Dc4rn; z|HV-&Rc*Vy|Lv@xCu)FUs88x7i!YeB&1a|;`+yoK?hZFVS_~yFfXc6rdg6|jKM+$8 z&%q+NNq@1RjK^eXVBb!+^a)XM3Ns7pEab<0SQ9nCmslF7VM;uQItx!Mj=Rf!0W+b_ zN_W(XPDHhz;U}TjYK0Y?Kvg`Cn!ro*BdS5X-7Y_!nG4mv2x{OKm<)%YmU=3x-6GVP z+G6>8E#H5Vgl2vVHQ*m+%spzUoZ{%j{`CLA9KH3!lX#wUJ=ZN@30Yu9&qshOi#Q7HPAuKhNn?K<=&z4%N}$S zt%X|p&rtcjPy>I71#m7V_men9BCsSFO8hr!=IIW(4@ef&gz8{M9D@3gEI{r33Cn+p z8X)Np?(NBoTFJhs_Ny>6UO=^fkAdI+vmSO0tD&B-qs6096IqN}`ckhb%s6UPUeS1Jw5*+K+BY3!;|1 zjm0xDlK2NKhX0}-xY$o_MQiv;#3G{wYG&=sp5_;*Eg6scvAG&c;WpF{m$#^`2|eaI z&WDQ2oApsE*UsWW<~Y+o#}cc}U6`5%$52mr4>e=&IKSCKFlxoBnr$%)@eqp_p`LV? z#cxm#7<9rN!eCTf3LEI>e?^z@=As6Cff_LWN%z}odejrO!JOC&gK!?I-YU$1dr$*k z#~@68%DtYMurP5|)JhIRwOfc`jPGqG(TDls_&BQL%U0ocGwxYeo)&Xco)hC^L(~9m z%)Y3B#-g@pEo$ONEWTs$2UNX~bF6z|IqQZnM;LDYnfqbBkmlVa=(&eW)eVWdG)Gn$3W%BT*Sp$6`cVK~y_O{k9dp;qpq z%-kbFX;@^<=-Az8fwM zMNO z4OYTLw_U%D%qY~jLvQ=t-^Gq2Lo2Y^3QnLtNLMVrgUN`Wo6+yM4pN#~F)!r>uq?L0 z`nVMH<9qCgIe&G(piDKF_(?=ku+=hNqE;Zz2Lk-jh^Wf*G zdb7=Cn2~rBmcR>G8WaEK&Q2ZFN_N1+I9xH~d(%mT;(FARpFz#|HI~PiPhDIUD-pLq zJ=s#!q1}QyLnko^<2-ZqlcVAssQM9@8Y`k6s5SaEP$v?a`C!!E&Bx4m3UlErRK2v% z-QIQ7duoIqR-<_px8M0@F0ECzdYKx-=?uZn4gDjq6 zEzENX8#A1(7vd6#s=;FvWW}krLRKxFKpn$1yKnwm8-wZoFU&{P{ndWt2cQsI3BQVfno+9*xDw zpNG})5^7H~{^`z2F4UeEvN+PLh^ki`)qfMS1NyZmy)9!nYT(HhFTGILu|AA!a36`Rk@EX>|JE%`` z!GGKmHbQ-pTVXgZ#R7N%)p4wUUB@99N}L(B!WB>xuaC7b3e(_DKM6g_Mbzu_3^kFA zo-gp@vk9tvJyyjZQ5`1n`2xR?grZg|0yRJti$61ap`LUUhT=3-z0H>I|A~YKxP%e- z95sM(Y6U*O+%ONT{?3t@WfhI-P8<|<4_d>EOy z-}{w>zWMJ^6G#`s=gq+SSOp)V-rsyN-9+o5I_`oYI36{@#i)U9njcW@LSngz7e#GR zN6d!(F_iJW1y*4nYA>InzEBxr`vN};s+l8Dd%hdh@e|DDW54700>9%GiR)G-3biHu zQDI~dN)sLRQ7kI5g zQ28BDXJHhU#zp3B)Wky*x(6+m(C=nel?;8!T47n7gF1}YF)JoXoR2C2>!X$&AEg zQ7=>Eum8h-!9@Xv?YQWp5*Y6o>LjL$kT|rVagPF%HhU%b_#Z4{li0WXFkF@;p zs1Mm}%RgcH=gnIf`1Ai$tMD&s$zmmQ1BIC3*oF7J3Lc_-YjR)UZ$Jiw*pi|?RL@X{ zv~UV{_?n@9diJ+?5#}M@i{OpIwKLLrBB(#)WPz?uSFiu7-{Swr- z{}Af^e~B9K1FBtuH0}eG8a3gZs0S!+`E@ZZace980`*~=oQC&b9nK~r1};YJ{R-5R ztU;abL*`|SPy7_M)PG=ZOqxPKKl({% z$*-Ue$8#%4lfms>9@G+-LhW63)Kb>Rc-RwlW`>}C9Dj$Ze-yP+7f@&57HUghq1wd^ zb!W(*n1p6l6}88mQ8S!{TEd;EnVvy)bQ`twudO^Jqg&cA)Y3ObosE&Gdb3dNccTVA zfm-pOv8I0h|3ac88M!jKiqlXNT7r7wgQzVzX5K_S;TtTEUS_vN6;SnBpx*mlsI6I! zfqqcmiDRfw`b$irpZ_6QT*DlwUlL22jZsV13)RtR)RQent;AN;fJaep(RI{7k5Nni z3iV_8BkIY6vbs10>a1kLP`&?6NvOhL)O$G^)xk{E0Eq7@b`baNpv9N3Pxb5?CweWqTbV?s3jkb8fd!t zE$WP{K@E5iwE|aBdwd5q;6v2P#>nCNOOF+Zv*+ObSA(u(6vDo!l~|2>;;pC&>_WXx zhp;}L#M~H~(;c!JsGkizQ4<|!`Lj?H-i>O16!o*?ENaUx=k&XQ?pwhV)WEN-!tbcp zGe$0V7E+_mLS9tI)vy6JGQYzd#9nS+;Lm(HQCr#sHE?It%Je|}xbEjCp^mng`!VpE zpep{1^YIt#j)U^J)B6bZLP1xFJfCv9PTFS??a+B z8S_y~`Vq5Y&V24L)y8AQy|Fiz$nW!p;07~y0XNZ=s6(|5b?Ek@4&52l>--9{W8#AD zj1|TFdjGqVXiC8%oQ5B;2~I5Je)GAH6^TeJHAaRn)+3QT=_1dTr;U-j*F0`1}7eB=j2nY8C!OO(a<{cQ&%1KB4(=CzeIk z|A6{HB`NNcwUSR!TT~#D_h0X2MG|^#TA@BDT`?an!|Hem zYhlI`?uSNC)RV5p(s&*-VbYS$2+T~}#2kv6z#`O&>_?4zy(I6yzRB;%(BVs4icca| zK~;#t<~S1dA-RQm^1o4A5M0`=P+Btw>Wf(jwf9v~OFa_xzR$x@T#Y)c$4mR&lV2r6 z9X~)l*+Ekmu?TC9M(@N@hJ+hbH&SMM}x>wZQ(;6v1w1($Pg zS$01Oy~j0COWzm^;}p~$9YrnWRn!xI#2+zMdAFCRaRG6%3O;WUZbp4c>sNGtU+@*` zdvYFi2ydVc>tB}d_bR!iiI19DD%8@1p;n{>YVYc!CfFM_p*0qtMlE@)%I?Y2pkA+G zsFj+ITB*6HdMi=m>_xt;e(x*^Eztv1#rLQVVpMS}6N2%HLs1jTgX*X}s$CP*Cpro> zaX;!wC!-GKT2%cr<~>xsKQX@E|2S3MK*>-A=~0Iz4D}jDq6VmrdZIR{dQqqe3`A|! zICDPgux>y-=}F9qw^2U>;#G4GlmxpnzL$=KPWyP&fXgsCZpG2K6ZJ%;s=H5W3)F|? zOH}?GRJ$G6+{Z7K*nv26O}F>sP-o|B)Hn+;qv~Ve-~YKvLQDS->dE5QavzY?s3k9m z%CCmnnx?2F?SgvpVb}m?U?+TpnoynE?ulDtOyW+c!`BTpUhmqx|JqwW8G4clsCYH% z3$_il^ruk`?x9ZeE7U~(MmVRQ*b*t!aU(*A3OqkLqVUR>j$Mc>nc;cgWC( z;t$kJ6W4VWLr@dSj5-tHW=Yf&R!6mOZgFqafFn>_^fjve25gFZP!E>8o@<}iPeOZ9 z3bn+oQ8Vm;dXk|SiqlaYZ$TZZBdFK%D(cDpFr(LZ`3X@I2t!T03Th&aQ1#ndzQ3Dg z48aHr##q5#)RP@Y?eQJdA$p6N*hkc94sPJSWT{aPPzLpu)JA>l8>8CyK}}!`>H(%Z z{oV=^dg9%vhCiBTP&2=bS@FKb@f*6=H5Fp5}I)&>cdeT^_p}>txPw}f`d^@xDd4k-=PlcLDZ7|h?>ZC)PVO;?O&V! zqV_(hi7QWmf&czrR?EnP>bSVYbx^O-XP6y(pbph+)K+aqJ?U=L&x8x8ALs8;d!N3k z`vs&ph7)(df;bba;IXE>|N5blw3(YxHPn+d!N%AXHPB(ylb=LA*%j1)4>0ga<+Gc8e{Q0k~tB@EAQIG@m)2}7! z{ceMr$S~CDo`_nBnW%}bwEP{YAF~HgPxb^uF+n@`8s|mreGSxBMqx6&|9t`p&-)5B z-~!Zudr<@YggRtbP#=z4=3CU0#%=GmBocKts$f5?gZ1#J<)`Z4`pJls$j^c4_5N=s zp@x@G@8=DyjE}G)=IiL=-&Ei)5>c-E z^$+UsChN@mufvdugbq~|)DkwtGw8>{SUZZp0mU&GiLX&-An)g{T~#bb+z1=uRMcDb z05#wT)Cwl<;!J_siu7H0|4WgGBts1bqdK09+QZeTh9^;H;{xjSiPP1^g;7u31vSxc zu?Swmf|#J2&ufEaQ7f?&^_t&C)eG(JcfaR1>FzolhuJCEhvE1DHDE{&w?!3DXQBq` z^=g4S6J1bCIt;b+lP!M^mL*<~dXSf>v-KX;KDxiBd*Y<1_cjdmx>P`&(iW&6pB+&R z2BB8yE7U+sQE$U~RQ&^}`p3;1sFi$)88Nt*8z%xaA%6`LI(+pp6q}xC~ksYX| z+;8zI)Zx5@8t7Nlz(IZ8+mRN_5w}Ghw&kcT*<+qXt;{^G8Hkr#d<1ovuA?US4@O|<5O+2jq4u^F zYGT{XL#PKijd}1c>aZss%KNV;%0AR}ToQG9TVpotgW8(8sJCK0Y735HA-se-3-P{i z1EfanZ8+*TrYfiv>4rIQ9O?l!TYTvY-hb`oKV+1_B44^04nz&K95ujJ)a!B>^%k74 z@{3k}7d6p0s3(g}zuJ=Y*dN1D6I+2=f&Hilf9fZp(;01;n^`i{TTlcGV`WN#R z2JUM5aW?UE)G5w2%6+Q4pxX;6G$`Kt!Q@C+YpYLNQpqq`(KNMe%)@4 zT8Y-E8TZ2?I12U2-HRIFGU`cx#oqWkYM`hw?i1V(wK9IxFR5Rle#)*z^?x4qtJ*^h z{P+KSW8GdRL=Bt`wGt5+6KkLjO+C~h^kZ=xgPPDm)RLb>{S0`Fs`mzU`afVp4Ef4^ zS-Yb88;pJ(mX#zl@J`gJ{L$hYsPd;4e?$!sG|sI+2x_Zxp`Nff>h)@kIdCNEkghck zqw4>HdcB{Hs{mhv!G!bhkl3!mU7R2{V?4N>oX6z0Oo zm>qYaz87~;KmS7}x)o|Pk=IW%ZApeEG6uER%TcHF0BS|9Vs3nafkQRPO*lISURTr) ztiaMaYvKuPiK(W!FYEwpL%b82px;Y9-3^!*^>k zjT-Pe*2X8Ol`1~NO`r;D3!9-HtRIHrWYo%RK>hh*2gcIR|2-ts!6B@JXHid*@oTr# z1yM^@0(IzWVqJ_vt<)aWQl3V=-#1Yce}fe<_DrAm1=hsDcmhXYrCIdL_}(!RYH$tJ z;d9gy`ewV+9Em!dtugTaqqbrUYQX6huQa!y9%w)6K~A9_;1=q2e2G;t&NsaO>YyHp zuGk5+v={N9kDt@1C0jn%E$KQ`{F8YWOA%i|oskstoEb5KI5#%HHmI%Ij(WgY^W9sO zYQDYy;bdf|pcd-&>5uxLEJi)~k5~&Ipq?ONfs1RR-iG<8FX3|3mvN)z|Au<A6sQkSo^N^owWRgQ2pk&BN8AtV;R=iY#4^OW7P^^tL3KPF^)^gLt;}lF3U0!{ z_rvn9Vm0#bpw3X9MQ#hL_(^0SqXB9neQ`Ao!EnsF*nKD(qMqDu&PIKhHlSAM2x=wn zp}v&Qa3CgG;(oUrk9vFVqbB$owPk+aQrA&@EJ#L1bVu^es|LY;|X zs1949e(Lo`eZr@pevI!#eJ{R8tOF3VnrR2r zM0!~~2DJs#QA@bS@^_&IID+~^=1C00xGUX2MKBL>S=5>7Zux!;)B8V;geI^9E8{8D z0LfQ5v!LGRf~bj=MeTV_i@TZwQ4<-3`oc{>4LHTh7ooOnEouucqhBAMUrA`d-%)Qt z%+>C72}Lb+0n}-(i0be&)K>LC9l8mqL-)1i??nBEa}j6YbJW{0VvT#d#-ko&&l=u; zE$NSBsKZ;R3H*h6;*js$lV(8;Tpl%%2B;59H|&guP-iIjTDJm4QD>nvYTycH4YL92 z476D5cZrTxml^BkCf~lyPu0eh2wqrSbhw8ZKIyd1mSeCdsmc#{^6R%@_jJe*O zr6Q;;sDYZm06z)Md^onn$*84#g|jgE26yPzqE_ZGYRhh+Ub8o-)1G9bJA8#OH*qi2 zSy+HYaT)40y@C2*CD`O1*q?=j_AmnV8kNE-*a z7bqf7?#Gw``o_=P{*8)`cOT>?3iyq1N- zBWfabFbX@Oz9VN*1HMIVO^Ty#D?6YLrGFC%E!8>H1ny%ijQ*oLOzlvw;R4htKZly| z->AbF^pl%#QL{2?3mT!m7oE*-unX}|sEHRk=F?&Kdyypc#4WHTc0oPyF^qwiQA_^| zM&cvX)?_(uzYC(au%^X*uom$I%fE*D@OUR&{p6_jVVFhle<>0=9PLm8cQFT{o^&i~ zg0syPmcI>E{|Ls#3zmPwe1=+ykEkb3bkZHV5LAEpF#+RyWh|o>YNjn!fW0v~4nw^K zqfmP}3AJ|{Q4>FldZH_s4kFBCbSQXXhksE?D~pw9~bWfjd~=`h5RI zywc)?`rlv9OyfVvSZkSMt>bA_u1F`PX|U8feQ$n4-WKXdaW5n91$T1trebCCieWF} zeE6GY!SzP{6YEMsS>V$1m!}{XcVq4;4E~%-*|~#A*P*d#Xp2Unky&w`rFi z#}g0b-ovdcI(fP{4&Gi%E9M8J*U|d%2i`F5nwO!~Xw*C~hwRAi@$mFuy1T#is|9_;yDH=!E0E4Yi4P%q9D~g64 z=-fx0kCYe3bL4$V*<1XIvKyqkQGXzDN@;DEnU`o+i}ZKcl{&hHM0e-Un??ugDHuWK za9m7;F@77xK6v(F5COt zPvy@EmUAbeT;nU59&^da&zzJYel+*yOsC z#T9wqk^edM!YmeN8SFm!3CN#LJc9cAJnM>2{xR-sjIoe<>9zmSNkmZT4iyS$016{W ze@8kRomA%TPJD!W3Ae6AbP`FsvfR3~jcsVh@2G*RF#oJftV^Hdht$_Kl=9f*Zy;`^ z3_E{H2Cg6qo?3denVtsYNY}B>XH#bgab@n244i_AMp@nZq^nauEp_fwHs12rkgsbA z`8UYFOw4cep8qA86G#M+sp}vW0`iaPyLFEzn6&B)dz-`sMd(;ahSVdkK{Y+zU zUHz%A>!iaArF;|VQ1bPA!2{}MiOKVaQ=n@PCgT2y&W6!RQ7Vkb`81eib?T9qo_i4Y z4jbTG%1YBdCe9+SBxOBFm$5dt)j8KFi;Gex6a79RuhggaUloSaFdc($p+W^JKP8=r zTh~19=;U3sSbrc{N<5Id7f83mk(iu5bOkfO5$eV#PDWW7?p9XDA1ea?R_!76rTzbv zjHA}^0xE`E#g(MjlRtva;xpJj>hfz};QE`mBISMXD7UVf!~>|`jr1|@J=}3A8$g@U z^ih|x!B)2fb*~Wnds9i5k4lZnSVKWim*o|)!N!som-`#aFO#>F@?2I{p7aXxpV9FI z>g2EirjkCytt%$w^Qhm0dkpb-VqL%S`$ynUCJIVgL#5YI;T4rTkyp_we@i@t4trXs zs<(`L0p-7tmy9RW)s!~XshgO*lH4CDzvha(xK<}7<;}@E9{B%XATgYRjQ^Q{{?=og z4Lpf-N;=Zj9Dl;J)a`&dh)dI1ZSLIk5uG>}?fMhPqs}dyWc}o`{-@wZ+CJtE?Emjd zaPj9#uS_7tU&&HwtEH=w-cQ+WI)91R=&%#%Fz(LODQW}kByPd2Ydv>u^6#Vmq%+9s z8pf{Hc5ciPqdNm`oPx>JJ`==|@*N;+WL$ z!JW_AH6rf=<-IMgPpn^1xe|JP_~&6V^b2VI8F$(S`^VX^!^zdvm1g`B@BM=tu|Iju zaVBvp985eB4{(1=-Y)LW+}*inaDTe;FokWz{Dni{T0{S37^@YoqpY@j1V8^}Y35GB zQq1i#1OGmZTD7!T>GziYn+|qzzv6yp1HPl|OB-m>ewDpk*Q!|l=)YtW#m_LPjX=(E#Y5tbuf7f|^lCo2I zKld@~;1$J(i2qV!D!<3R4APOiA@?fsu5wQy|1fQyk*K|0S#*+cHpq?>YY=Kgfuw7Lh#SWlgi+@G#O z^w-hiH&$<eRD@S=L`q4F*@)X3nj!2BQpqq9iS zNaUxZR}_?|!AdHRAzgyXpOMzpfw(pG(kY;>uKeW1=k8#GD6b*+UGnn(Cod-H0<_h& zjyo3Vnb?8291ova|Mq3=n zOPrKSxyY=_oyG>a zhRG=Zok=|AZuXyc!z|s2It3X>*KzJ~#K&neiGst+*4_P74MK2Mw`>b?T9~JXJ}JF z`~QH%92;mA6;uAFMlm!G6w3>&N>^$L=o6`Roc3+v-2$~Tg(hQX9~r+gUkeJlTk zyphz;!u_*8(z*uNps{K6C-)%kMpou(a3;vp)yVPx{@I20Ik<~4NCq3IA8}GkkH*~$ z+}Qf~m)6DT-(Q)3{6t|!Dj%}}V_;tHSmc+s4kqGF8tLzqbY-HmYUEYtZbGMJh`*rC z66@nn>f|E74f&t0+Ke%u`e|$u2lbKGwSY!vC@4a~QR3pnmuUDkb|juherX!(ifaSx zrhX#J>w(Xx-Hd>$M19wk@e@*c1NU~DX?N1!lm3kXvSCSTM#Gt| zh4+k((sSRZ-URaM{iox;3{;Z4oAsrEt5Pqg#VM4*wSzk|dHUb2@yc=sGuTo(dibBt zx{{~s8m`4ZX;8#EI#1p->UOkAbtgZom8s1U2Caz2xU*9y2M)6SNP3B>`-nRJDs&I(|c3i1cDCN}Wq|p#Q(Q)c7}XW72W35^-AYtn`}$6OmWi`j~Bf zUZJdo{@kdm62SoOXMu==M@MC8aGA>U$=hl9?ai;qk4?w*u_tx=aQ{P`!Up*jQ!7ng z9PSvD>$*eUK<>ZE({+>o+``L1;YAmF6LBpSzQ#y8ugZOn^jRiX4Vz;c)Rl&I^)v|A z7V|Uef4cgSsLwz!PCVicb+Tp+CePcZ>po81wCBy^V zd2KKipZGKZ(s4)+qu!^hhNbhFg{|(dpDJKr{ST9vM#s9EQz;&k$xPgz^epn`a&NU6 zU!=3w)?sFRZtc`o*9nU&)AlWQTbJ#Pq|eW|uUp)}>Gw|1@i{6+r?4~kLOO^|oSb_o z_gfoS^%D~Jqr5nImr38{)^&~e2lD3_Nr$OybiukGL>#b;=Ua=(Tk^nY<{gH-O4{D62_Xaw{)^UsxwI>7xVXx(ZmES}wHp zC-|NQbE$O2Dn7D~UNi6}O^W<^Apv?u+52#a*vYYgC zi1N?KTZi8h*P)KTp*0vm=11;q7Vn@?*LdQIbdZL4I`8y27Z^}iJFD}GHo9JOpC>N{aUZL5gmeh)_7XoOoiI?7e|#GMDII73 zl!^C<-&6T>>$ItrRUtk@Syt=#Ir;5v0;-adbbIRT!YafyxeHPMSF5iddw&pD!D`g; z7o~DCGInz}A)_D-PFm$JaTs~J%2DrurEik{bp1e`1q6T5URML^JjC%9EALzK^q?c~ zsO6u+gZlk{lMSXX;Y{v++?%O<0i!5ugSr;sTTDt`W;}oiRKQixCbW+_qj<7rajSb2o4>#oBl4oOw7+b#CIx+p0q8BCxZ8>-ARm?4C?pKJjCG`36 zm|Nbz`8L*; Date: Tue, 12 Nov 2024 14:38:49 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20=E6=8F=90=E4=BE=9B=20Agent=20?= =?UTF-8?q?=E5=8C=85=E7=AE=A1=E7=90=86=E5=90=8E=E5=8F=B0=E5=9F=BA=E7=A1=80?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=20(closed=20#1683)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0084_merge_20241112_1438.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 apps/node_man/migrations/0084_merge_20241112_1438.py diff --git a/apps/node_man/migrations/0084_merge_20241112_1438.py b/apps/node_man/migrations/0084_merge_20241112_1438.py new file mode 100644 index 000000000..53cbda699 --- /dev/null +++ b/apps/node_man/migrations/0084_merge_20241112_1438.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.4 on 2024-11-12 06:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('node_man', '0083_merge_20240911_1050'), + ('node_man', '0083_subscription_operate_info'), + ] + + operations = [ + ] From cc349bd7f78b0c613804f33b6e8d7b05c089e8f0 Mon Sep 17 00:00:00 2001 From: gqp <446105468@qq.com> Date: Tue, 12 Nov 2024 14:39:07 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20=E6=8F=90=E4=BE=9B=20Agent=20?= =?UTF-8?q?=E5=8C=85=E7=AE=A1=E7=90=86=E5=90=8E=E5=8F=B0=E5=9F=BA=E7=A1=80?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=20(closed=20#1683)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/node_man/migrations/0084_merge_20241112_1438.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/node_man/migrations/0084_merge_20241112_1438.py b/apps/node_man/migrations/0084_merge_20241112_1438.py index 53cbda699..9a46c7744 100644 --- a/apps/node_man/migrations/0084_merge_20241112_1438.py +++ b/apps/node_man/migrations/0084_merge_20241112_1438.py @@ -6,9 +6,8 @@ class Migration(migrations.Migration): dependencies = [ - ('node_man', '0083_merge_20240911_1050'), - ('node_man', '0083_subscription_operate_info'), + ("node_man", "0083_merge_20240911_1050"), + ("node_man", "0083_subscription_operate_info"), ] - operations = [ - ] + operations = [] From 1fe08bef981eb15f296595e8330722efa4fd27af Mon Sep 17 00:00:00 2001 From: gqp <446105468@qq.com> Date: Tue, 12 Nov 2024 15:39:52 +0800 Subject: [PATCH 4/4] =?UTF-8?q?feat:=20=E6=8F=90=E4=BE=9B=20Agent=20?= =?UTF-8?q?=E5=8C=85=E7=AE=A1=E7=90=86=E5=90=8E=E5=8F=B0=E5=9F=BA=E7=A1=80?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=20(closed=20#1683)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0083_merge_20240911_1050.py | 13 ---- ...packages.py => 0084_auto_20241112_1539.py} | 66 +++++++++++++++---- .../migrations/0084_merge_20241112_1438.py | 13 ---- 3 files changed, 54 insertions(+), 38 deletions(-) delete mode 100644 apps/node_man/migrations/0083_merge_20240911_1050.py rename apps/node_man/migrations/{0082_gsepackagedesc_gsepackages.py => 0084_auto_20241112_1539.py} (60%) delete mode 100644 apps/node_man/migrations/0084_merge_20241112_1438.py diff --git a/apps/node_man/migrations/0083_merge_20240911_1050.py b/apps/node_man/migrations/0083_merge_20240911_1050.py deleted file mode 100644 index 1f98b8346..000000000 --- a/apps/node_man/migrations/0083_merge_20240911_1050.py +++ /dev/null @@ -1,13 +0,0 @@ -# Generated by Django 3.2.4 on 2024-09-11 02:50 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("node_man", "0082_gsepackagedesc_gsepackages"), - ("node_man", "0082_host_dept_name"), - ] - - operations = [] diff --git a/apps/node_man/migrations/0082_gsepackagedesc_gsepackages.py b/apps/node_man/migrations/0084_auto_20241112_1539.py similarity index 60% rename from apps/node_man/migrations/0082_gsepackagedesc_gsepackages.py rename to apps/node_man/migrations/0084_auto_20241112_1539.py index f10559a4b..fc82cf1c4 100644 --- a/apps/node_man/migrations/0082_gsepackagedesc_gsepackages.py +++ b/apps/node_man/migrations/0084_auto_20241112_1539.py @@ -1,13 +1,4 @@ -# -*- coding: utf-8 -*- -""" -TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. -Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at https://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" +# Generated by Django 3.2.4 on 2024-11-12 07:39 from django.db import migrations, models @@ -15,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("node_man", "0081_auto_20240307_1656"), + ("node_man", "0083_subscription_operate_info"), ] operations = [ @@ -62,7 +53,13 @@ class Migration(migrations.Migration): ( "os", models.CharField( - choices=[("windows", "windows"), ("linux", "linux"), ("aix", "aix"), ("solaris", "solaris")], + choices=[ + ("windows", "windows"), + ("linux", "linux"), + ("aix", "aix"), + ("solaris", "solaris"), + ("darwin", "darwin"), + ], db_index=True, default="linux", max_length=32, @@ -94,4 +91,49 @@ class Migration(migrations.Migration): "verbose_name_plural": "Gse包(GsePackages)", }, ), + migrations.AlterField( + model_name="job", + name="job_type", + field=models.CharField( + choices=[ + ("INSTALL_AGENT", "INSTALL_AGENT"), + ("RESTART_AGENT", "RESTART_AGENT"), + ("REINSTALL_AGENT", "REINSTALL_AGENT"), + ("UNINSTALL_AGENT", "UNINSTALL_AGENT"), + ("REMOVE_AGENT", "REMOVE_AGENT"), + ("UPGRADE_AGENT", "UPGRADE_AGENT"), + ("DOWNGRADE_AGENT", "DOWNGRADE_AGENT"), + ("IMPORT_AGENT", "IMPORT_AGENT"), + ("RESTART_AGENT", "RESTART_AGENT"), + ("RELOAD_AGENT", "RELOAD_AGENT"), + ("ACTIVATE_AGENT", "ACTIVATE_AGENT"), + ("MAIN_START_PLUGIN", "MAIN_START_PLUGIN"), + ("MAIN_STOP_PLUGIN", "MAIN_STOP_PLUGIN"), + ("MAIN_RESTART_PLUGIN", "MAIN_RESTART_PLUGIN"), + ("MAIN_RELOAD_PLUGIN", "MAIN_RELOAD_PLUGIN"), + ("MAIN_DELEGATE_PLUGIN", "MAIN_DELEGATE_PLUGIN"), + ("MAIN_UNDELEGATE_PLUGIN", "MAIN_UNDELEGATE_PLUGIN"), + ("MAIN_INSTALL_PLUGIN", "MAIN_INSTALL_PLUGIN"), + ("MAIN_STOP_AND_DELETE_PLUGIN", "MAIN_STOP_AND_DELETE_PLUGIN"), + ("DEBUG_PLUGIN", "DEBUG_PLUGIN"), + ("STOP_DEBUG_PLUGIN", "STOP_DEBUG_PLUGIN"), + ("PUSH_CONFIG_PLUGIN", "PUSH_CONFIG_PLUGIN"), + ("REMOVE_CONFIG_PLUGIN", "REMOVE_CONFIG_PLUGIN"), + ("PACKING_PLUGIN", "PACKING_PLUGIN"), + ("INSTALL_PROXY", "INSTALL_PROXY"), + ("RESTART_PROXY", "RESTART_PROXY"), + ("REINSTALL_PROXY", "REINSTALL_PROXY"), + ("REPLACE_PROXY", "REPLACE_PROXY"), + ("UNINSTALL_PROXY", "UNINSTALL_PROXY"), + ("UPGRADE_PROXY", "UPGRADE_PROXY"), + ("IMPORT_PROXY", "IMPORT_PROXY"), + ("RESTART_PROXY", "RESTART_PROXY"), + ("RELOAD_PROXY", "RELOAD_PROXY"), + ], + db_index=True, + default="INSTALL_PROXY", + max_length=45, + verbose_name="作业类型", + ), + ), ] diff --git a/apps/node_man/migrations/0084_merge_20241112_1438.py b/apps/node_man/migrations/0084_merge_20241112_1438.py deleted file mode 100644 index 9a46c7744..000000000 --- a/apps/node_man/migrations/0084_merge_20241112_1438.py +++ /dev/null @@ -1,13 +0,0 @@ -# Generated by Django 3.2.4 on 2024-11-12 06:38 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("node_man", "0083_merge_20240911_1050"), - ("node_man", "0083_subscription_operate_info"), - ] - - operations = []