feature: Agent 配置重载功能优化 (closed #1769)

app.yml

@@ -8,7 +8,7 @@ introduction: 通过节点管理，可以对蓝鲸体系中的gse agent进行管
 introduction_en: NodeMan can be used to manage the gse agent in the BlueKing system.
   Its functions include agent installation, status query, version update, plugin management,
   health check, process control, and so on.
-version: 2.4.1
+version: 2.4.3
 category: 运维工具
 language_support: 英语,中文
 desktop:

apps/backend/agent/artifact_builder/base.py

@@ -259,10 +259,23 @@ def _inject_dependencies(self, extract_dir: str):
                     # TODO 后续如果 1.0 Agent 包也纳入管理，此处需要区分
                     #  1.0 Agent / Proxy 的 gsectl 采用固定的 rclocal 模式
                     if gsectl_filename == "gsectl" and "{{ AUTO_TYPE }}" in gsectl_file_content:
-                        auto_type: str = models.GlobalSettings.get_config(
+                        auto_type_strategy: typing.Union[str, typing.Dict[str, str]] = models.GlobalSettings.get_config(
                             models.GlobalSettings.KeyEnum.GSE2_LINUX_AUTO_TYPE.value,
                             constants.GseLinuxAutoType.RCLOCAL.value,
                         )
+                        logger.info(f"get auto_type_strategy -> {auto_type_strategy} from global settings")
+
+                        if isinstance(auto_type_strategy, dict):
+                            try:
+                                auto_type = auto_type_strategy[self.NAME]
+                            except KeyError:
+                                auto_type = constants.GseLinuxAutoType.RCLOCAL.value
+                        else:
+                            if auto_type_strategy not in constants.GseLinuxAutoType.list_member_values():
+                                auto_type = constants.GseLinuxAutoType.RCLOCAL.value
+                            else:
+                                auto_type = auto_type_strategy
+
                         logger.info(f"apply auto_type -> {auto_type} to gsectl")
                         gsectl_file_content = gsectl_file_content.replace("{{ AUTO_TYPE }}", auto_type)
                     if "{{ AUTO_TYPE }}" in gsectl_file_content:

apps/backend/agent/solution_maker.py

@@ -29,6 +29,7 @@
 from apps.core.script_manage.base import ScriptHook
 from apps.node_man import constants, models
 from apps.utils import basic
+from apps.utils.files import PathHandler
 
 
 class ExecutionSolutionStepContent:
@@ -101,6 +102,14 @@ def choose_script_file(cls, host: models.Host, is_execute_on_target: bool) -> st
         script_file_name = constants.SCRIPT_FILE_NAME_MAP[host.os_type]
         return script_file_name
 
+    @staticmethod
+    def get_gse_extra_config_dir(os_type: str):
+        extra_config_sub_dir: str = "user_conf"
+        if os_type.upper() == constants.OsType.WINDOWS:
+            return json.dumps(PathHandler(os_type).join(settings.GSE_ENVIRON_WIN_DIR, extra_config_sub_dir))[1:-1]
+        else:
+            return PathHandler(os_type).join(settings.GSE_ENVIRON_DIR, extra_config_sub_dir)
+
 
 class BaseExecutionSolutionMaker(metaclass=abc.ABCMeta):
     # 是否直接在目标机器上执行
@@ -349,11 +358,19 @@ def get_create_pre_dirs_step(self, is_shell_adapter: bool = False) -> ExecutionS
         """
         # 目前依赖文件路径相关配置分两类：1-文件名路径，创建上级目录，2-目录路径，暂无需求
         filepath_config_names: typing.List[str] = []
+        filepath_necessary_names: typing.List[str] = []
 
         if self.host.os_type != constants.OsType.WINDOWS:
             filepath_config_names.extend(["dataipc"])
 
+        if not self.agent_setup_info.is_legacy:
+            # GSE 1.0 不需要创建额外配置目录
+            filepath_necessary_names.append(ExecutionSolutionTools.get_gse_extra_config_dir(self.host.os_type))
+
         dirs_to_be_created: typing.Set[str] = {self.dest_dir}
+        for filepath_necessary_name in filepath_necessary_names:
+            dirs_to_be_created.add(filepath_necessary_name)
+
         # 获取到相应操作系统
         agent_config: typing.Dict[str, typing.Any] = self.host_ap.get_agent_config(self.host.os_type)
         for filepath_config_name in filepath_config_names:

apps/backend/agent/tools.py

@@ -257,7 +257,7 @@ def gen_commands(
     # 批量场景请传入Optional所需对象，以避免 n+1 查询，提高执行效率
     host_ap = host_ap or host.ap
     identity_data = identity_data or host.identity
-    install_channel = install_channel or host.install_channel
+    install_channel = install_channel or host.install_channel(ap_id=host_ap.id)
     proxies = proxies if proxies is not None else fetch_proxies(host, host_ap)
     # TODO 如果是额外安装场景，这里的 jumpserver 应该选取当前作业平台全业务集可用的机器
     gse_servers_info: Dict[str, Any] = fetch_gse_servers_info(

apps/backend/components/collections/agent_new/add_or_update_hosts.py

@@ -97,7 +97,7 @@ def static_ip_selector(
         except_bk_biz_id: int = sub_inst.instance_info["host"]["bk_biz_id"]
         if except_bk_biz_id != cmdb_host["bk_biz_id"]:
             self.move_insts_to_failed(
-                sub_inst.id,
+                [sub_inst.id],
                 log_content=_("主机期望注册到业务【ID：{except_bk_biz_id}】，但实际存在于业务【ID: {actual_biz_id}】，请前往该业务进行安装").format(
                     except_bk_biz_id=except_bk_biz_id, actual_biz_id=cmdb_host["bk_biz_id"]
                 ),
@@ -132,7 +132,7 @@ def dynamic_ip_selector(
                     break
                 else:
                     self.move_insts_to_failed(
-                        sub_inst.id,
+                        [sub_inst.id],
                         log_content=_(
                             "主机期望注册到业务【ID：{except_bk_biz_id}】，但实际存在于业务【ID: {actual_biz_id}】，请前往该业务进行安装"
                         ).format(except_bk_biz_id=except_bk_biz_id, actual_biz_id=cmdb_host["bk_biz_id"]),

apps/backend/components/collections/agent_new/base.py

@@ -29,11 +29,25 @@
 from apps.backend.subscription.steps.agent_adapter.adapter import AgentStepAdapter
 from apps.node_man import constants, models
 from apps.node_man.exceptions import AliveProxyNotExistsError
+from apps.prometheus import metrics
+from apps.prometheus.helper import SetupObserve
 
 from .. import job
 from ..base import BaseService, CommonData
 
 
+@dataclass
+class AgentCommonData(CommonData):
+    # 默认接入点
+    default_ap: models.AccessPoint
+    # 主机ID - 接入点 映射关系
+    host_id__ap_map: Dict[int, models.AccessPoint]
+    # AgentStep 适配器
+    agent_step_adapter: AgentStepAdapter
+    # 注入AP_ID
+    injected_ap_id: int
+
+
 class AgentBaseService(BaseService, metaclass=abc.ABCMeta):
     """
     AGENT安装基类
@@ -47,6 +61,7 @@ def sub_inst_failed_handler(self, sub_inst_ids: Union[List[int], Set[int]]):
         pass
 
     @classmethod
+    @SetupObserve(histogram=metrics.app_task_engine_get_common_data_duration_seconds, labels={"step_type": "AGENT"})
     def get_common_data(cls, data):
         """
         初始化常用数据，注意这些数据不能放在 self 属性里，否则会产生较大的 process snap shot，
@@ -145,15 +160,14 @@ def get_agent_pkg_name(
 
         return (agent_pkg_name.format(cpu_arch=host.cpu_arch), agent_pkg_name)[return_name_with_cpu_tmpl]
 
-    @classmethod
-    def get_agent_pkg_dir(cls, common_data: "AgentCommonData", host: models.Host) -> str:
+    def get_agent_pkg_dir(self, common_data: "AgentCommonData", host: models.Host) -> str:
         """
         获取 Agent 安装包目录
         :param common_data: AgentCommonData
         :param host: models.Host
         :return:
         """
-        host_ap = common_data.host_id__ap_map[host.bk_host_id]
+        host_ap: models.AccessPoint = self.get_host_ap(common_data=common_data, host=host)
         download_path = host_ap.nginx_path or settings.DOWNLOAD_PATH
         if common_data.agent_step_adapter.is_legacy:
             # 旧版本 Agent 安装包位于下载目录
@@ -169,12 +183,12 @@ def get_host_id__install_channel_map(
         hosts: List[models.Host],
         host_id__sub_inst_id: Dict[int, int],
         cloud_id__proxies_map: Dict[int, List[models.Host]],
+        common_data: AgentCommonData,
     ) -> Dict[int, Tuple[Optional[models.Host], Dict[str, List]]]:
         install_channel_ids: List[int] = list({host.install_channel_id for host in hosts})
         install_channel_id__jump_servers_map: Dict[
             int, List[models.Host]
         ] = models.InstallChannel.install_channel_id__host_objs_map(install_channel_ids)
-
         # 建立通道ID - 通道的映射关系
         id__install_channel_obj_map: Dict[int, models.InstallChannel] = {}
         for install_channel_obj in models.InstallChannel.objects.filter(id__in=install_channel_ids):
@@ -188,6 +202,7 @@ def get_host_id__install_channel_map(
 
         host_id__install_channel_map: Dict[int, Tuple[Optional[models.Host], Dict[str, List]]] = {}
         for host in hosts:
+            host_ap: models.AccessPoint = self.get_host_ap(common_data=common_data, host=host)
             sub_inst_id = host_id__sub_inst_id[host.bk_host_id]
             install_channel_obj = id__install_channel_obj_map.get(host.install_channel_id)
             if install_channel_obj:
@@ -198,7 +213,10 @@ def get_host_id__install_channel_map(
                         [sub_inst_id], log_content=_("所选安装通道「{name}」 没有可用跳板机".format(name=install_channel_obj.name))
                     )
                 else:
-                    host_id__install_channel_map[host.bk_host_id] = (jump_server, install_channel_obj.upstream_servers)
+                    host_id__install_channel_map[host.bk_host_id] = (
+                        jump_server,
+                        install_channel_obj.get_upstream_servers_by_ap_id(host_ap.id),
+                    )
             elif host.bk_cloud_id != constants.DEFAULT_CLOUD and host.node_type != constants.NodeType.PROXY:
                 # 仅校验
                 alive_proxies = cloud_id__alive_proxies_map.get(host.bk_cloud_id, [])
@@ -233,6 +251,7 @@ def get_host_id__installation_tool_map(
             hosts=hosts_need_gen_commands,
             host_id__sub_inst_id=common_data.host_id__sub_inst_id_map,
             cloud_id__proxies_map=cloud_id__proxies_map,
+            common_data=common_data,
         )
 
         id__sub_inst_obj_map: Dict[int, models.SubscriptionInstanceRecord] = {
@@ -305,17 +324,13 @@ def maintain_agent_proc_status_uniqueness(self, bk_host_ids: Set[int]) -> None:
             proc_statuses_to_be_created.append(models.ProcessStatus(bk_host_id=host_id, **self.agent_proc_common_data))
         models.ProcessStatus.objects.bulk_create(proc_statuses_to_be_created, batch_size=self.batch_size)
 
-
-@dataclass
-class AgentCommonData(CommonData):
-    # 默认接入点
-    default_ap: models.AccessPoint
-    # 主机ID - 接入点 映射关系
-    host_id__ap_map: Dict[int, models.AccessPoint]
-    # AgentStep 适配器
-    agent_step_adapter: AgentStepAdapter
-    # 注入AP_ID
-    injected_ap_id: int
+    def get_host_ap(self, common_data: AgentCommonData, host: models.Host) -> models.AccessPoint:
+        # 优先使用注入的AP ID
+        if common_data.injected_ap_id:
+            host_ap: models.AccessPoint = common_data.ap_id_obj_map[common_data.injected_ap_id]
+        else:
+            host_ap: models.AccessPoint = common_data.host_id__ap_map[host.bk_host_id]
+        return host_ap
 
 
 class RetryHandler:

apps/backend/components/collections/agent_new/check_agent_ability.py

@@ -31,7 +31,8 @@ def get_script_content(self, data, common_data: AgentCommonData, host: models.Ho
         path_handler: PathHandler = PathHandler(host.os_type)
         ctl_exe_name: str = ("gse_agent", "gse_agent.exe")[host.os_type == constants.OsType.WINDOWS]
         general_node_type: str = self.get_general_node_type(host.node_type)
-        setup_path: str = common_data.host_id__ap_map[host.bk_host_id].get_agent_config(host.os_type)["setup_path"]
+        host_ap: models.AccessPoint = self.get_host_ap(common_data=common_data, host=host)
+        setup_path: str = host_ap.get_agent_config(host.os_type)["setup_path"]
         agent_path: str = path_handler.join(setup_path, general_node_type, "bin", ctl_exe_name)
 
         return f"{agent_path} --version"