feat: local data source import & export (#1228)

.github/workflows/bk-user.yml

@@ -64,6 +64,8 @@ jobs:
         run: |
           # random secret
           export BK_APP_SECRET="fod6MKVTVi_3M5HgGoj-qI7b3l0dgCzTBwGypnDz4vg="
+          # random secret key
+          export BKKRILL_ENCRYPT_SECRET_KEY="tttHSBLiVdQPItrfy7n9dV7AxAUMZpYVkD6IHMbL0VE="
           export BK_USER_URL=""
           export BK_COMPONENT_API_URL=""
           export MYSQL_PASSWORD=root_pwd

src/bk-user/Dockerfile

@@ -31,6 +31,7 @@ RUN poetry config virtualenvs.create false && poetry install --only main
 
 COPY src/bk-user/bkuser /app/bkuser
 COPY src/bk-user/bin /app/bin
+COPY src/bk-user/media /app/media
 COPY src/bk-user/manage.py /app
 
 COPY --from=StaticBuilding /dist /app/staticfiles

src/bk-user/bin/start_celery.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# 设置环境变量
+CELERY_CONCURRENCY=${CELERY_CONCURRENCY:-8}
+CELERY_LOG_LEVEL=${CELERY_LOG_LEVEL:-info}
+
+# Run!
+celery -A bkuser.celery worker -l ${CELERY_LOG_LEVEL} --concurrency ${CELERY_CONCURRENCY}

src/bk-user/bkuser/apis/web/data_source/mixins.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from bkuser.apis.web.mixins import CurrentUserTenantMixin
+from bkuser.apps.data_source.models import DataSource
+
+
+class CurrentUserTenantDataSourceMixin(CurrentUserTenantMixin):
+    """获取当前用户所在租户下属数据源"""
+
+    lookup_url_kwarg = "id"
+
+    def get_queryset(self):
+        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())

src/bk-user/bkuser/apis/web/data_source/serializers.py

@@ -17,9 +17,9 @@
 from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 
-from bkuser.apps.data_source.constants import DataSourcePluginEnum, FieldMappingOperation
+from bkuser.apps.data_source.constants import FieldMappingOperation
 from bkuser.apps.data_source.models import DataSource, DataSourcePlugin
-from bkuser.apps.data_source.plugins.constants import DATA_SOURCE_PLUGIN_CONFIG_CLASS_MAP
+from bkuser.plugins.constants import DATA_SOURCE_PLUGIN_CONFIG_CLASS_MAP, DataSourcePluginEnum
 from bkuser.utils.pydantic import stringify_pydantic_error
 
 logger = logging.getLogger(__name__)
@@ -61,7 +61,7 @@ def get_updated_at(self, obj: DataSource) -> str:
 class DataSourceFieldMappingSLZ(serializers.Serializer):
     """
     单个数据源字段映射
-    FIXME (su) 动态字段实现后，需要检查：target_field 需是租户定义的，source_field 需是插件允许的
+    FIXME (su) 自定义字段实现后，需要检查：target_field 需是租户定义的，source_field 需是插件允许的
     """
 
     source_field = serializers.CharField(help_text="数据源原始字段")
@@ -185,5 +185,21 @@ class DataSourceTestConnectionOutputSLZ(serializers.Serializer):
     """数据源连通性测试"""
 
     error_message = serializers.CharField(help_text="错误信息")
-    user = serializers.CharField(help_text="用户")
-    department = serializers.CharField(help_text="部门")
+    user = RawDataSourceUserSLZ(help_text="用户")
+    department = RawDataSourceDepartmentSLZ(help_text="部门")
+
+
+class LocalDataSourceImportInputSLZ(serializers.Serializer):
+    """本地数据源导入"""
+
+    file = serializers.FileField(help_text="数据源用户信息文件（Excel 格式）")
+    overwrite = serializers.BooleanField(help_text="允许对同名用户覆盖更新", default=False)
+    incremental = serializers.BooleanField(help_text="是否使用增量同步", default=False)
+
+
+class LocalDataSourceImportOutputSLZ(serializers.Serializer):
+    """本地数据源导入结果"""
+
+    task_id = serializers.CharField(help_text="任务 ID")
+    status = serializers.CharField(help_text="任务状态")
+    summary = serializers.CharField(help_text="任务执行结果概述")

src/bk-user/bkuser/apis/web/data_source/views.py

@@ -8,11 +8,17 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
+import logging
+
+import openpyxl
+from django.conf import settings
 from django.db import transaction
+from django.utils.translation import gettext_lazy as _
 from drf_yasg.utils import swagger_auto_schema
 from rest_framework import generics, status
 from rest_framework.response import Response
 
+from bkuser.apis.web.data_source.mixins import CurrentUserTenantDataSourceMixin
 from bkuser.apis.web.data_source.serializers import (
     DataSourceCreateInputSLZ,
     DataSourceCreateOutputSLZ,
@@ -24,15 +30,24 @@
     DataSourceSwitchStatusOutputSLZ,
     DataSourceTestConnectionOutputSLZ,
     DataSourceUpdateInputSLZ,
+    LocalDataSourceImportInputSLZ,
+    LocalDataSourceImportOutputSLZ,
 )
 from bkuser.apis.web.mixins import CurrentUserTenantMixin
 from bkuser.apps.data_source.constants import DataSourceStatus
 from bkuser.apps.data_source.models import DataSource, DataSourcePlugin
-from bkuser.apps.data_source.plugins.constants import DATA_SOURCE_PLUGIN_CONFIG_SCHEMA_MAP
 from bkuser.apps.data_source.signals import post_create_data_source, post_update_data_source
+from bkuser.apps.sync.constants import SyncTaskTrigger
+from bkuser.apps.sync.data_models import DataSourceSyncOptions
+from bkuser.apps.sync.managers import DataSourceSyncManager
 from bkuser.biz.data_source_plugin import DefaultPluginConfigProvider
+from bkuser.biz.exporters import DataSourceUserExporter
 from bkuser.common.error_codes import error_codes
+from bkuser.common.response import convert_workbook_to_response
 from bkuser.common.views import ExcludePatchAPIViewMixin, ExcludePutAPIViewMixin
+from bkuser.plugins.constants import DATA_SOURCE_PLUGIN_CONFIG_SCHEMA_MAP
+
+logger = logging.getLogger(__name__)
 
 
 class DataSourcePluginListApi(generics.ListAPIView):
@@ -127,13 +142,11 @@ def post(self, request, *args, **kwargs):
         )
 
 
-class DataSourceRetrieveUpdateApi(CurrentUserTenantMixin, ExcludePatchAPIViewMixin, generics.RetrieveUpdateAPIView):
+class DataSourceRetrieveUpdateApi(
+    CurrentUserTenantDataSourceMixin, ExcludePatchAPIViewMixin, generics.RetrieveUpdateAPIView
+):
     pagination_class = None
     serializer_class = DataSourceRetrieveOutputSLZ
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
 
     @swagger_auto_schema(
         tags=["data_source"],
@@ -175,14 +188,10 @@ def put(self, request, *args, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class DataSourceTestConnectionApi(CurrentUserTenantMixin, generics.RetrieveAPIView):
+class DataSourceTestConnectionApi(CurrentUserTenantDataSourceMixin, generics.RetrieveAPIView):
     """数据源连通性测试"""
 
     serializer_class = DataSourceTestConnectionOutputSLZ
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
 
     @swagger_auto_schema(
         tags=["data_source"],
@@ -215,14 +224,10 @@ def get(self, request, *args, **kwargs):
         return Response(DataSourceTestConnectionOutputSLZ(instance=mock_data).data)
 
 
-class DataSourceSwitchStatusApi(CurrentUserTenantMixin, ExcludePutAPIViewMixin, generics.UpdateAPIView):
+class DataSourceSwitchStatusApi(CurrentUserTenantDataSourceMixin, ExcludePutAPIViewMixin, generics.UpdateAPIView):
     """切换数据源状态（启/停）"""
 
     serializer_class = DataSourceSwitchStatusOutputSLZ
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
 
     @swagger_auto_schema(
         tags=["data_source"],
@@ -242,29 +247,93 @@ def patch(self, request, *args, **kwargs):
         return Response(DataSourceSwitchStatusOutputSLZ(instance={"status": data_source.status.value}).data)
 
 
-class DataSourceTemplateApi(generics.ListAPIView):
+class DataSourceTemplateApi(CurrentUserTenantDataSourceMixin, generics.ListAPIView):
+    """获取本地数据源数据导入模板"""
+
+    pagination_class = None
+
+    @swagger_auto_schema(
+        tags=["data_source"],
+        operation_description="下载数据源导入模板",
+        responses={status.HTTP_200_OK: "org_tmpl.xlsx"},
+    )
     def get(self, request, *args, **kwargs):
         """数据源导出模板"""
-        # TODO (su) 实现代码逻辑
-        return Response()
+        # 获取数据源信息，用于后续填充模板中的自定义字段
+        data_source = self.get_object()
+        if not data_source.is_local:
+            raise error_codes.DATA_SOURCE_OPERATION_UNSUPPORTED.f(_("仅本地数据源类型有提供导入模板"))
+
+        workbook = DataSourceUserExporter(data_source).get_template()
+        return convert_workbook_to_response(workbook, f"{settings.EXPORT_EXCEL_FILENAME_PREFIX}_org_tmpl.xlsx")
 
 
-class DataSourceExportApi(generics.ListAPIView):
+class DataSourceExportApi(CurrentUserTenantDataSourceMixin, generics.ListAPIView):
     """本地数据源用户导出"""
 
+    pagination_class = None
+
+    @swagger_auto_schema(
+        tags=["data_source"],
+        operation_description="下载本地数据源用户数据",
+        responses={status.HTTP_200_OK: "org_data.xlsx"},
+    )
     def get(self, request, *args, **kwargs):
         """导出指定的本地数据源用户数据（Excel 格式）"""
-        # TODO (su) 实现代码逻辑，注意：仅本地数据源可以导出
-        return Response()
+        data_source = self.get_object()
+        if not data_source.is_local:
+            raise error_codes.DATA_SOURCE_OPERATION_UNSUPPORTED.f(_("仅能导出本地数据源数据"))
 
+        workbook = DataSourceUserExporter(data_source).export()
+        return convert_workbook_to_response(workbook, f"{settings.EXPORT_EXCEL_FILENAME_PREFIX}_org_data.xlsx")
 
-class DataSourceImportApi(generics.CreateAPIView):
+
+class DataSourceImportApi(CurrentUserTenantDataSourceMixin, generics.CreateAPIView):
     """从 Excel 导入数据源用户数据"""
 
+    @swagger_auto_schema(
+        tags=["data_source"],
+        operation_description="本地数据源用户数据导入",
+        request_body=LocalDataSourceImportInputSLZ(),
+        responses={status.HTTP_200_OK: LocalDataSourceImportOutputSLZ()},
+    )
     def post(self, request, *args, **kwargs):
         """从 Excel 导入数据源用户数据"""
-        # TODO (su) 实现代码逻辑，注意：仅本地数据源可以导入
-        return Response()
+        slz = LocalDataSourceImportInputSLZ(data=request.data)
+        slz.is_valid(raise_exception=True)
+        data = slz.validated_data
+
+        data_source = self.get_object()
+        if not data_source.is_local:
+            raise error_codes.DATA_SOURCE_OPERATION_UNSUPPORTED.f(_("仅本地数据源支持导入功能"))
+
+        # Request file 转换成 openpyxl.workbook
+        try:
+            workbook = openpyxl.load_workbook(data["file"])
+        except Exception:  # pylint: disable=broad-except
+            logger.exception("本地数据源导入失败")
+            raise error_codes.DATA_SOURCE_IMPORT_FAILED.f(_("文件格式异常"))
+
+        options = DataSourceSyncOptions(
+            operator=request.user.username,
+            overwrite=data["overwrite"],
+            incremental=data["incremental"],
+            async_run=False,
+            trigger=SyncTaskTrigger.MANUAL,
+        )
+
+        try:
+            plugin_init_extra_kwargs = {"workbook": workbook}
+            task = DataSourceSyncManager(data_source, options).execute(plugin_init_extra_kwargs)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception("本地数据源导入失败")
+            raise error_codes.DATA_SOURCE_IMPORT_FAILED.f(str(e))
+
+        return Response(
+            LocalDataSourceImportOutputSLZ(
+                instance={"task_id": task.id, "status": task.status, "summary": task.summary}
+            ).data
+        )
 
 
 class DataSourceSyncApi(generics.CreateAPIView):

src/bk-user/bkuser/apis/web/tenant/serializers.py

@@ -18,17 +18,17 @@
 from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 
-from bkuser.apps.data_source.constants import DataSourcePluginEnum
 from bkuser.apps.data_source.models import DataSourceUser
-from bkuser.apps.data_source.plugins.local.constants import PasswordGenerateMethod
-from bkuser.apps.data_source.plugins.local.models import LocalDataSourcePluginConfig, NotificationConfig
 from bkuser.apps.tenant.constants import TENANT_ID_REGEX
 from bkuser.apps.tenant.models import Tenant, TenantUser
 from bkuser.biz.data_source import DataSourceSimpleInfo
 from bkuser.biz.data_source_plugin import DefaultPluginConfigProvider
 from bkuser.biz.tenant import TenantUserWithInheritedInfo
 from bkuser.biz.validators import validate_data_source_user_username
 from bkuser.common.passwd import PasswordValidator
+from bkuser.plugins.constants import DataSourcePluginEnum
+from bkuser.plugins.local.constants import PasswordGenerateMethod
+from bkuser.plugins.local.models import LocalDataSourcePluginConfig, NotificationConfig
 from bkuser.utils.pydantic import stringify_pydantic_error
 
 

src/bk-user/bkuser/apis/web/tenant/views.py

@@ -25,7 +25,6 @@
     TenantUserSearchInputSLZ,
     TenantUserSearchOutputSLZ,
 )
-from bkuser.apps.data_source.plugins.local.models import PasswordInitialConfig
 from bkuser.apps.tenant.models import Tenant, TenantUser
 from bkuser.biz.data_source import DataSourceHandler
 from bkuser.biz.tenant import (
@@ -36,6 +35,7 @@
     TenantManagerWithoutID,
 )
 from bkuser.common.views import ExcludePatchAPIViewMixin
+from bkuser.plugins.local.models import PasswordInitialConfig
 
 logger = logging.getLogger(__name__)
 

src/bk-user/bkuser/apps/data_source/constants.py

@@ -23,16 +23,6 @@ class DataSourceStatus(str, StructuredEnum):
     DISABLED = EnumField("disabled", label=_("未启用"))
 
 
-class DataSourcePluginEnum(str, StructuredEnum):
-    """数据源插件枚举"""
-
-    LOCAL = EnumField("local", label=_("本地数据源"))
-    GENERAL = EnumField("general", label=_("通用数据源"))
-    WECOM = EnumField("wecom", label=_("企业微信"))
-    LDAP = EnumField("ldap", label=_("OpenLDAP"))
-    MAD = EnumField("mad", label=_("MicrosoftActiveDirectory"))
-
-
 class FieldMappingOperation(str, StructuredEnum):
     """字段映射关系"""