fix: 服务实例并发请求数量过大

Huayeaaa · May 10, 2024 · 2730668 · 2730668
1 parent 921636f
commit 2730668
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 10 deletions.
diff --git a/apps/backend/subscription/tools.py b/apps/backend/subscription/tools.py
@@ -305,7 +305,7 @@ def get_service_instances(
             filter_field_name.value: filter_id_list,
         }
         if filter_field_name.needs_batch_request:
-            result = batch_request(CCApi.list_service_instance_detail, params)
+            result = batch_request(CCApi.list_service_instance_detail, params, sort="id")
         else:
             params["page"] = {
                 "start": 0,
@@ -378,7 +378,9 @@ def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
     if not module_ids:
         return []
 
-    if len(module_ids) <= constants.QUERY_MODULE_ID_THRESHOLD:
+    if len(module_ids) <= models.GlobalSettings.get_config(
+        models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value, constants.QUERY_MODULE_ID_THRESHOLD
+    ):
         params = [
             {
                 "func": CCApi.list_service_instance_detail,
@@ -492,7 +494,7 @@ def get_service_instances_by_template(bk_obj_id, template_info_list: list, bk_bi
         params = dict(bk_set_template_ids=template_ids, bk_biz_id=int(bk_biz_id), fields=("bk_host_id", "bk_cloud_id"))
     host_info_result = batch_request(call_func, params)
     bk_host_ids = [inst["bk_host_id"] for inst in host_info_result]
-    all_service_instances = batch_request(client_v2.cc.list_service_instance_detail, params)
+    all_service_instances = batch_request(CCApi.list_service_instance_detail, params, sort="id")
     service_instances = [instance for instance in all_service_instances if instance["bk_host_id"] in bk_host_ids]
 
     return service_instances
@@ -1049,7 +1051,7 @@ def _add_process_info_to_host_instances(bk_biz_id: int, instances: List[Dict]):
     """
     bk_host_list = [instance["host"]["bk_host_id"] for instance in instances]
     host_processes = get_process_by_biz_id(bk_biz_id, bk_host_list)
-    logging.info(
+    logger.info(
         f"[add_process_info_to_host_instances] instance_hosts_count -> {len(bk_host_list)}, "
         f"cc_query_count -> {len(host_processes)}"
     )

diff --git a/apps/node_man/models.py b/apps/node_man/models.py
@@ -158,6 +158,8 @@ class KeyEnum(Enum):
         JOB_TASK_POLICY = "JOB_TASK_POLICY"
         # 插件进程启动检查时间
         PLUGIN_PROC_START_CHECK_SECS = "PLUGIN_PROC_START_CHECK_SECS"
+        # 查询服务实例时module_id阈值，当小于该阈值时以单module_id并发查询
+        SERVICE_INSTANCE_MODULE_ID_THRESHOLD = "SERVICE_INSTANCE_MODULE_ID_THRESHOLD"
 
     key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True)
     v_json = JSONField(_("值"))

diff --git a/apps/utils/batch_request.py b/apps/utils/batch_request.py
@@ -69,21 +69,28 @@ def batch_request(
     if not get_count:
         return sync_batch_request(func, params, get_data, limit)
 
+    start = 0
+    data = []
     if not split_params:
-        final_request_params = [
-            {"count": get_count(func(dict(page={"start": 0, "limit": 1}, **params))), "params": params}
-        ]
+        request_params = dict(page={"start": 0, "limit": limit}, **params)
+        if sort:
+            request_params["page"]["sort"] = sort
+        query_res = func(request_params)
+        final_request_params = [{"count": get_count(query_res), "params": params}]
+        data = get_data(query_res)
+        # 如果count小于等于limit，直接返回
+        if final_request_params[0]["count"] <= limit:
+            return data
+
+        start = limit
     else:
         final_request_params = format_params(params, get_count, func)
 
-    data = []
-
     # 根据请求总数并发请求
     pool = ThreadPool(20)
     futures = []
 
     for req in final_request_params:
-        start = 0
         while start < req["count"]:
             request_params = {"page": {"limit": limit, "start": start}}
             if sort: