fix: 同步主机差量逻辑修复 (closed TencentBlueKing#2167)

apps/node_man/constants.py

@@ -66,6 +66,7 @@ class TimeUnit:
 
 SYNC_CMDB_BIZ_TOPO_TASK_INTERVAL = 1 * TimeUnit.DAY
 SYNC_CMDB_HOST_INTERVAL = 1 * TimeUnit.DAY
+CLEAR_NEED_DELETE_HOST_IDS_INTERVAL = 1 * TimeUnit.MINUTE
 
 ########################################################################################################
 # 第三方系统相关配置
@@ -588,6 +589,9 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
 LIST_SERVICE_INSTANCE_DETAIL_LIMIT = 1000
 LIST_SERVICE_INSTANCE_DETAIL_INTERVAL = 0.2
 
+# redis键名模板
+REDIS_NEED_DELETE_HOST_IDS_KEY_TPL = f"{settings.APP_CODE}:node_man:need_delete_host_ids:list"
+
 
 class ProxyFileFromType(Enum):
     """Proxy上传文件名称来源"""

apps/node_man/periodic_tasks/sync_cmdb_host.py

@@ -17,6 +17,7 @@
 from django.db import transaction
 
 from apps.backend.celery import app
+from apps.backend.utils.redis import REDIS_INST
 from apps.component.esbclient import client_v2
 from apps.core.gray.tools import GrayTools
 from apps.exceptions import ComponentCallError
@@ -434,6 +435,8 @@ def sync_cmdb_host(bk_biz_id=None, task_id=None):
     """
     同步cmdb主机
     """
+    from apps.backend.views import LPUSH_AND_EXPIRE_FUNC
+
     logger.info(f"[sync_cmdb_host] start: task_id -> {task_id}, bk_biz_id -> {bk_biz_id}")
 
     ap_map_config: SyncHostApMapConfig = get_sync_host_ap_map_config()
@@ -464,14 +467,29 @@ def sync_cmdb_host(bk_biz_id=None, task_id=None):
     # 节点管理需要删除的host_id
     need_delete_host_ids = set(node_man_host_ids) - set(cc_bk_host_ids)
     if need_delete_host_ids:
-        models.Host.objects.filter(bk_host_id__in=need_delete_host_ids).delete()
-        models.IdentityData.objects.filter(bk_host_id__in=need_delete_host_ids).delete()
-        models.ProcessStatus.objects.filter(bk_host_id__in=need_delete_host_ids).delete()
-        logger.info(f"[sync_cmdb_host] task_id -> {task_id}, need_delete_host_ids -> {need_delete_host_ids}")
+        name = constants.REDIS_NEED_DELETE_HOST_IDS_KEY_TPL
+        LPUSH_AND_EXPIRE_FUNC(keys=[name], args=[constants.TimeUnit.DAY] + list(need_delete_host_ids))
+        logger.info(
+            f"[sync_cmdb_host] task_id -> {task_id}, store need_delete_host_ids -> {need_delete_host_ids} into redis"
+        )
 
     logger.info(f"[sync_cmdb_host] complete: task_id -> {task_id}, bk_biz_ids -> {bk_biz_ids}")
 
 
+def clear_need_delete_host_ids(task_id=None):
+    name = constants.REDIS_NEED_DELETE_HOST_IDS_KEY_TPL
+    report_data_len = REDIS_INST.llen(name)
+    report_data = REDIS_INST.lrange(name, -report_data_len, -1)
+    # 后使用ltrim保留剩下的，可以保证report_log中新push的值不会丢失
+    REDIS_INST.ltrim(name, 0, -report_data_len - 1)
+    report_data.reverse()
+    need_delete_host_ids: typing.Set[int] = set([int(data) for data in report_data])
+    models.Host.objects.filter(bk_host_id__in=need_delete_host_ids).delete()
+    models.IdentityData.objects.filter(bk_host_id__in=need_delete_host_ids).delete()
+    models.ProcessStatus.objects.filter(bk_host_id__in=need_delete_host_ids).delete()
+    logger.info(f"[clear_need_delete_host_ids] task_id -> {task_id}, need_delete_host_ids -> {need_delete_host_ids}")
+
+
 @periodic_task(
     queue="default",
     options={"queue": "default"},
@@ -492,3 +510,16 @@ def sync_cmdb_host_task(bk_biz_id=None):
     """
     task_id = sync_cmdb_host_task.request.id
     sync_cmdb_host(bk_biz_id, task_id)
+
+
+@periodic_task(
+    queue="default",
+    options={"queue": "default"},
+    run_every=constants.CLEAR_NEED_DELETE_HOST_IDS_INTERVAL,
+)
+def clear_need_delete_host_ids_task():
+    """
+    被动周期删除cmdb不存在的主机
+    """
+    task_id = clear_need_delete_host_ids_task.request.id
+    clear_need_delete_host_ids(task_id)

apps/node_man/tests/test_pericdic_tasks/test_sync_cmdb_host.py

@@ -13,7 +13,10 @@
 
 from apps.node_man import constants
 from apps.node_man.models import Host
-from apps.node_man.periodic_tasks.sync_cmdb_host import sync_cmdb_host_periodic_task
+from apps.node_man.periodic_tasks.sync_cmdb_host import (
+    clear_need_delete_host_ids_task,
+    sync_cmdb_host_periodic_task,
+)
 from apps.utils.unittest.testcase import CustomBaseTestCase
 
 from .mock_data import MOCK_BK_BIZ_ID, MOCK_HOST, MOCK_HOST_NUM
@@ -55,4 +58,5 @@ def test_sync_cmdb_host(self):
         self.assertEqual(itf_results.sort(key=lambda t: t[0]), list(db_result).sort(key=lambda t: t[0]))
 
         # 验证主机信息是否删除成功
+        clear_need_delete_host_ids_task()
         self.assertEqual(Host.objects.filter(bk_host_id=-1).count(), 0)