feat: data source base api (#1207)

.github/workflows/bk-user.yml

@@ -1,11 +1,11 @@
 name: bkuser_ci_check
 on:
   push:
-    branches: [master, ft_tenant, ft_tenant_manage]
+    branches: [master, ft_tenant]
     paths:
       - "src/bk-user/**"
   pull_request:
-    branches: [master, ft_tenant, ft_tenant_manage]
+    branches: [master, ft_tenant]
     paths:
       - "src/bk-user/**"
 jobs:
@@ -35,28 +35,38 @@ jobs:
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Start MySQL Container
+        uses: samin/[email protected]
+        with:
+          mysql version: "8.0"
+          mysql database: bk-user
+          mysql user: root
+          mysql password: root_pwd
+          mysql root password: root_pwd
+      - name: Start Redis Container
+        uses: supercharge/[email protected]
+        with:
+          redis-version: "3.2.0"
+      - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
       - name: Set up Poetry
         uses: abatilo/[email protected]
         with:
           poetry-version: "1.5.1"
-      - name: Start Redis Container
-        uses: supercharge/[email protected]
-        with:
-          redis-version: "3.2.0"
       - name: Install dependencies
         working-directory: src/bk-user
         run: poetry install
       - name: Run unittest
         working-directory: src/bk-user
-        # TODO 使用更合适的方式解决“必须的”配置项问题
         run: |
-          export BK_APP_SECRET=""
+          # random secret
+          export BK_APP_SECRET="fod6MKVTVi_3M5HgGoj-qI7b3l0dgCzTBwGypnDz4vg="
           export BK_USER_URL=""
           export BK_COMPONENT_API_URL=""
+          export MYSQL_PASSWORD=root_pwd
+          export MYSQL_HOST="127.0.0.1"
           export DJANGO_SETTINGS_MODULE=bkuser.settings
           poetry run pytest ./tests

.gitignore

@@ -216,4 +216,4 @@ pre_commit_hooks
 # local settings
 cliff.toml
 .codecc
-.idea
+.idea

src/bk-user/Makefile

@@ -15,7 +15,7 @@ dj-settings: ## 设置 DJANGO_SETTINGS_MODULE
 	export DJANGO_SETTINGS_MODULE=bkuser.settings
 
 test: dj-settings ## 执行项目单元测试（pytest）
-	pytest --maxfail=5 -l --reuse-db bkuser
+	pytest --maxfail=1 -l --reuse-db tests
 
 i18n-po: dj-settings ## 将源代码 & 模版中的 message 采集到 django.po
 	python manage.py makemessages -d django -l en -e html,part -e py

src/bk-user/bkuser/apis/web/data_source/serializers.py

@@ -9,176 +9,176 @@
 specific language governing permissions and limitations under the License.
 """
 import logging
-from typing import Dict, List
+from typing import Any, Dict, List
 
-from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 from drf_yasg.utils import swagger_serializer_method
+from pydantic import ValidationError as PDValidationError
 from rest_framework import serializers
+from rest_framework.exceptions import ValidationError
 
-from bkuser.apps.data_source.models import (
-    DataSourceDepartment,
-    DataSourceDepartmentUserRelation,
-    DataSourceUser,
-)
-from bkuser.biz.validators import validate_data_source_user_username
-from bkuser.common.validators import validate_phone_with_country_code
+from bkuser.apps.data_source.constants import DataSourcePluginEnum, FieldMappingOperation
+from bkuser.apps.data_source.models import DataSource, DataSourcePlugin
+from bkuser.apps.data_source.plugins.constants import DATA_SOURCE_PLUGIN_CONFIG_CLASS_MAP
+from bkuser.utils.pydantic import stringify_pydantic_error
 
 logger = logging.getLogger(__name__)
 
 
-class UserSearchInputSLZ(serializers.Serializer):
-    username = serializers.CharField(required=False, help_text="用户名", allow_blank=True)
+class DataSourceSearchInputSLZ(serializers.Serializer):
+    keyword = serializers.CharField(help_text="搜索关键字", required=False)
 
 
-class DataSourceSearchDepartmentsOutputSLZ(serializers.Serializer):
-    id = serializers.CharField(help_text="部门ID")
-    name = serializers.CharField(help_text="部门名称")
+class DataSourceSearchOutputSLZ(serializers.Serializer):
+    id = serializers.IntegerField(help_text="数据源 ID")
+    name = serializers.CharField(help_text="数据源名称")
+    owner_tenant_id = serializers.CharField(help_text="数据源所属租户 ID")
+    plugin_name = serializers.SerializerMethodField(help_text="数据源插件名称")
+    cooperation_tenants = serializers.SerializerMethodField(help_text="协作公司")
+    status = serializers.CharField(help_text="数据源状态")
+    updater = serializers.CharField(help_text="更新者")
+    updated_at = serializers.SerializerMethodField(help_text="更新时间")
+
+    def get_plugin_name(self, obj: DataSource) -> str:
+        return self.context["data_source_plugin_map"].get(obj.plugin_id, "")
+
+    @swagger_serializer_method(
+        serializer_or_field=serializers.ListField(
+            help_text="协作公司",
+            child=serializers.CharField(),
+            allow_empty=True,
+        )
+    )
+    def get_cooperation_tenants(self, obj: DataSource) -> List[str]:
+        # TODO 目前未支持数据源跨租户协作，因此该数据均为空
+        return []
+
+    def get_updated_at(self, obj: DataSource) -> str:
+        return obj.updated_at_display
+
 
+class DataSourceFieldMappingSLZ(serializers.Serializer):
+    """
+    单个数据源字段映射
+    FIXME (su) 动态字段实现后，需要检查：target_field 需是租户定义的，source_field 需是插件允许的
+    """
 
-class UserSearchOutputSLZ(serializers.Serializer):
-    id = serializers.CharField(help_text="用户ID")
-    username = serializers.CharField(help_text="用户名")
-    full_name = serializers.CharField(help_text="全名")
-    phone = serializers.CharField(help_text="手机号")
-    email = serializers.CharField(help_text="邮箱")
-    departments = serializers.SerializerMethodField(help_text="用户部门")
-
-    # FIXME:考虑抽象一个函数 获取数据后传递到context
-    @swagger_serializer_method(serializer_or_field=DataSourceSearchDepartmentsOutputSLZ(many=True))
-    def get_departments(self, obj: DataSourceUser):
-        return [
-            {"id": department_user_relation.department.id, "name": department_user_relation.department.name}
-            for department_user_relation in DataSourceDepartmentUserRelation.objects.filter(user=obj)
-        ]
-
-
-class UserCreateInputSLZ(serializers.Serializer):
-    username = serializers.CharField(help_text="用户名", validators=[validate_data_source_user_username])
-    full_name = serializers.CharField(help_text="姓名")
-    email = serializers.EmailField(help_text="邮箱")
-    phone_country_code = serializers.CharField(
-        help_text="手机号国际区号", required=False, default=settings.DEFAULT_PHONE_COUNTRY_CODE
+    source_field = serializers.CharField(help_text="数据源原始字段")
+    mapping_operation = serializers.ChoiceField(help_text="映射关系", choices=FieldMappingOperation.get_choices())
+    target_field = serializers.CharField(help_text="目标字段")
+    expression = serializers.CharField(help_text="表达式", required=False)
+
+
+class DataSourceCreateInputSLZ(serializers.Serializer):
+    name = serializers.CharField(help_text="数据源名称", max_length=128)
+    plugin_id = serializers.CharField(help_text="数据源插件 ID")
+    plugin_config = serializers.JSONField(help_text="数据源插件配置")
+    field_mapping = serializers.ListField(
+        help_text="用户字段映射", child=DataSourceFieldMappingSLZ(), allow_empty=True, required=False, default=list
     )
-    phone = serializers.CharField(help_text="手机号")
-    logo = serializers.CharField(help_text="用户 Logo", required=False)
-    department_ids = serializers.ListField(help_text="部门ID列表", child=serializers.IntegerField(), default=[])
-    leader_ids = serializers.ListField(help_text="上级ID列表", child=serializers.IntegerField(), default=[])
-
-    def validate(self, data):
-        validate_phone_with_country_code(phone=data["phone"], country_code=data["phone_country_code"])
-        return data
-
-    def validate_department_ids(self, department_ids):
-        diff_department_ids = set(department_ids) - set(
-            DataSourceDepartment.objects.filter(
-                id__in=department_ids, data_source=self.context["data_source"]
-            ).values_list("id", flat=True)
-        )
-        if diff_department_ids:
-            raise serializers.ValidationError(_("传递了错误的部门信息: {}").format(diff_department_ids))
-        return department_ids
-
-    def validate_leader_ids(self, leader_ids):
-        diff_leader_ids = set(leader_ids) - set(
-            DataSourceUser.objects.filter(id__in=leader_ids, data_source=self.context["data_source"]).values_list(
-                "id", flat=True
-            )
-        )
-        if diff_leader_ids:
-            raise serializers.ValidationError(_("传递了错误的上级信息: {}").format(diff_leader_ids))
-        return leader_ids
 
+    def validate_name(self, name: str) -> str:
+        if DataSource.objects.filter(name=name).exists():
+            raise ValidationError(_("同名数据源已存在"))
 
-class UserCreateOutputSLZ(serializers.Serializer):
-    id = serializers.CharField(help_text="数据源用户ID")
+        return name
 
+    def validate_plugin_id(self, plugin_id: str) -> str:
+        if not DataSourcePlugin.objects.filter(id=plugin_id).exists():
+            raise ValidationError(_("数据源插件不存在"))
 
-class LeaderSearchInputSLZ(serializers.Serializer):
-    keyword = serializers.CharField(help_text="搜索关键字", required=False)
+        return plugin_id
 
+    def validate(self, attrs: Dict[str, Any]) -> Dict[str, Any]:
+        # 除本地数据源类型外，都需要配置字段映射
+        if attrs["plugin_id"] != DataSourcePluginEnum.LOCAL and not attrs["field_mapping"]:
+            raise ValidationError(_("当前数据源类型必须配置字段映射"))
 
-class LeaderSearchOutputSLZ(serializers.Serializer):
-    id = serializers.CharField(help_text="上级ID")
-    username = serializers.CharField(help_text="上级名称")
+        PluginConfigCls = DATA_SOURCE_PLUGIN_CONFIG_CLASS_MAP.get(attrs["plugin_id"])  # noqa: N806
+        # 自定义插件，可能没有对应的配置类，不需要做格式检查
+        if not PluginConfigCls:
+            return attrs
 
+        try:
+            PluginConfigCls(**attrs["plugin_config"])
+        except PDValidationError as e:
+            raise ValidationError(_("插件配置不合法：{}").format(stringify_pydantic_error(e)))
 
-class DepartmentSearchInputSLZ(serializers.Serializer):
-    name = serializers.CharField(required=False, help_text="部门名称", allow_blank=True)
+        return attrs
 
 
-class DepartmentSearchOutputSLZ(serializers.Serializer):
-    id = serializers.CharField(help_text="部门ID")
-    name = serializers.CharField(help_text="部门名称")
+class DataSourceCreateOutputSLZ(serializers.Serializer):
+    id = serializers.IntegerField(help_text="数据源 ID")
 
 
-class UserDepartmentOutputSLZ(serializers.Serializer):
-    id = serializers.IntegerField(help_text="部门ID")
-    name = serializers.CharField(help_text="部门名称")
+class DataSourcePluginOutputSLZ(serializers.Serializer):
+    id = serializers.CharField(help_text="数据源插件唯一标识")
+    name = serializers.CharField(help_text="数据源插件名称")
+    description = serializers.CharField(help_text="数据源插件描述")
+    logo = serializers.CharField(help_text="数据源插件 Logo")
 
 
-class UserLeaderOutputSLZ(serializers.Serializer):
-    id = serializers.IntegerField(help_text="上级ID")
-    username = serializers.CharField(help_text="上级用户名")
+class DataSourceRetrieveOutputSLZ(serializers.Serializer):
+    id = serializers.IntegerField(help_text="数据源 ID")
+    name = serializers.CharField(help_text="数据源名称")
+    owner_tenant_id = serializers.CharField(help_text="数据源所属租户 ID")
+    status = serializers.CharField(help_text="数据源状态")
+    plugin = DataSourcePluginOutputSLZ(help_text="数据源插件")
+    plugin_config = serializers.JSONField(help_text="数据源插件配置")
+    sync_config = serializers.JSONField(help_text="数据源同步任务配置")
+    field_mapping = serializers.JSONField(help_text="用户字段映射")
 
 
-class UserRetrieveOutputSLZ(serializers.Serializer):
-    username = serializers.CharField(help_text="用户名")
-    full_name = serializers.CharField(help_text="全名")
-    email = serializers.CharField(help_text="邮箱")
-    phone_country_code = serializers.CharField(help_text="手机区号")
-    phone = serializers.CharField(help_text="手机号")
-    logo = serializers.SerializerMethodField(help_text="用户Logo")
+class DataSourceUpdateInputSLZ(serializers.Serializer):
+    plugin_config = serializers.JSONField(help_text="数据源插件配置")
+    field_mapping = serializers.ListField(
+        help_text="用户字段映射", child=DataSourceFieldMappingSLZ(), allow_empty=True, required=False, default=list
+    )
 
-    departments = serializers.SerializerMethodField(help_text="部门信息")
-    leaders = serializers.SerializerMethodField(help_text="上级信息")
+    def validate_plugin_config(self, plugin_config: Dict[str, Any]) -> Dict[str, Any]:
+        PluginConfigCls = DATA_SOURCE_PLUGIN_CONFIG_CLASS_MAP.get(self.context["plugin_id"])  # noqa: N806
+        # 自定义插件，可能没有对应的配置类，不需要做格式检查
+        if not PluginConfigCls:
+            return plugin_config
 
-    def get_logo(self, obj: DataSourceUser) -> str:
-        return obj.logo or settings.DEFAULT_DATA_SOURCE_USER_LOGO
+        try:
+            PluginConfigCls(**plugin_config)
+        except PDValidationError as e:
+            raise ValidationError(_("插件配置不合法：{}").format(stringify_pydantic_error(e)))
 
-    @swagger_serializer_method(serializer_or_field=UserDepartmentOutputSLZ(many=True))
-    def get_departments(self, obj: DataSourceUser) -> List[Dict]:
-        user_departments_map = self.context["user_departments_map"]
-        departments = user_departments_map.get(obj.id, [])
-        return [{"id": dept.id, "name": dept.name} for dept in departments]
+        return plugin_config
 
-    @swagger_serializer_method(serializer_or_field=UserLeaderOutputSLZ(many=True))
-    def get_leaders(self, obj: DataSourceUser) -> List[Dict]:
-        user_leaders_map = self.context["user_leaders_map"]
-        leaders = user_leaders_map.get(obj.id, [])
-        return [{"id": leader.id, "username": leader.username} for leader in leaders]
+    def validate_field_mapping(self, field_mapping: List[Dict]) -> List[Dict]:
+        # 除本地数据源类型外，都需要配置字段映射
+        if self.context["plugin_id"] == DataSourcePluginEnum.LOCAL:
+            return field_mapping
 
+        if not field_mapping:
+            raise ValidationError(_("当前数据源类型必须配置字段映射"))
 
-class UserUpdateInputSLZ(serializers.Serializer):
-    full_name = serializers.CharField(help_text="姓名")
-    email = serializers.CharField(help_text="邮箱")
-    phone_country_code = serializers.CharField(help_text="手机国际区号")
-    phone = serializers.CharField(help_text="手机号")
-    logo = serializers.CharField(help_text="用户 Logo", allow_blank=True, required=False, default="")
+        return field_mapping
 
-    department_ids = serializers.ListField(help_text="部门ID列表", child=serializers.IntegerField())
-    leader_ids = serializers.ListField(help_text="上级ID列表", child=serializers.IntegerField())
 
-    def validate(self, data):
-        validate_phone_with_country_code(phone=data["phone"], country_code=data["phone_country_code"])
-        return data
+class DataSourceSwitchStatusOutputSLZ(serializers.Serializer):
+    status = serializers.CharField(help_text="数据源状态")
 
-    def validate_department_ids(self, department_ids):
-        diff_department_ids = set(department_ids) - set(
-            DataSourceDepartment.objects.filter(
-                id__in=department_ids, data_source=self.context["data_source"]
-            ).values_list("id", flat=True)
-        )
-        if diff_department_ids:
-            raise serializers.ValidationError(_("传递了错误的部门信息: {}").format(diff_department_ids))
-        return department_ids
-
-    def validate_leader_ids(self, leader_ids):
-        diff_leader_ids = set(leader_ids) - set(
-            DataSourceUser.objects.filter(id__in=leader_ids, data_source=self.context["data_source"]).values_list(
-                "id", flat=True
-            )
-        )
-        if diff_leader_ids:
-            raise serializers.ValidationError(_("传递了错误的上级信息: {}").format(diff_leader_ids))
-        return leader_ids
+
+class RawDataSourceUserSLZ(serializers.Serializer):
+    id = serializers.CharField(help_text="用户 ID")
+    properties = serializers.JSONField(help_text="用户属性")
+    leaders = serializers.ListField(help_text="用户 leader ID 列表", child=serializers.CharField())
+    departments = serializers.ListField(help_text="用户部门 ID 列表", child=serializers.CharField())
+
+
+class RawDataSourceDepartmentSLZ(serializers.Serializer):
+    id = serializers.CharField(help_text="部门 ID")
+    name = serializers.CharField(help_text="部门名称")
+    parent = serializers.CharField(help_text="父部门 ID")
+
+
+class DataSourceTestConnectionOutputSLZ(serializers.Serializer):
+    """数据源连通性测试"""
+
+    error_message = serializers.CharField(help_text="错误信息")
+    user = serializers.CharField(help_text="用户")
+    department = serializers.CharField(help_text="部门")