Merge pull request #7600 from Yuanruitao/master

feat: codecc 支持使用github账号登录态和鉴权 #7363
TencentBlueKing · Sep 27, 2022 · 1a0642a · 1a0642a
2 parents 9307a28 + 9da0239
commit 1a0642a
Show file tree

Hide file tree

Showing 128 changed files with 1,065 additions and 178 deletions.
diff --git a/helm-charts/core/codecc/templates/_helpers.tpl b/helm-charts/core/codecc/templates/_helpers.tpl
@@ -21,12 +21,16 @@ Create a default fully qualified mongodb subchart.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 */}}
 {{- define "codecc.mongodb.fullname" -}}
+{{- if eq .Values.mongodb.enabled true -}}
 {{- if .Values.mongodb.fullnameOverride -}}
 {{- .Values.mongodb.fullnameOverride | trunc 63 | trimSuffix "-" -}}
 {{- else -}}
 {{- $name := default "mongodb" .Values.mongodb.nameOverride -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+{{- else -}}
+{{- .Values.externalMongodb.host -}}
+{{- end -}}
 {{- end -}}
 
 {{- define "codecc.redis.fullname" -}}
@@ -48,34 +52,97 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end -}}
 {{- end -}}
 
+
+{{/*
+Return the mongodb username
+*/}}
+{{- define "codecc.mongodb.username" -}}
+{{- if eq .Values.mongodb.enabled true -}}
+{{- .Values.mongodb.auth.username -}}
+{{- else -}}
+{{- .Values.externalMongodb.username -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the mongodb password
+*/}}
+{{- define "codecc.mongodb.password" -}}
+{{- if eq .Values.mongodb.enabled true -}}
+{{- .Values.mongodb.auth.password -}}
+{{- else -}}
+{{- .Values.externalMongodb.password -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the mongodb port
+*/}}
+{{- define "codecc.mongodb.port" -}}
+{{- if eq .Values.mongodb.enabled true -}}
+27017
+{{- else -}}
+{{- .Values.externalMongodb.port -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Return the mongodb connection uri
 */}}
 {{- define "codecc.defect.mongodbUri" -}}
 {{- if eq .Values.mongodb.enabled true -}}
-{{- printf "mongodb://%s:%s@%s:27017/db_defect" .Values.mongodb.auth.username .Values.mongodb.auth.password (include "codecc.mongodb.fullname" .) -}}
+{{- printf "mongodb://%s:%s@%s:%s/db_defect" .Values.mongodb.auth.username .Values.mongodb.auth.password (include "codecc.mongodb.fullname" .) (include "codecc.mongodb.port" .) -}}
 {{- else -}}
-{{- .Values.externalMongodb.defectUrl -}}
+{{- printf "mongodb://%s:%s@%s:%s/db_defect?%s" .Values.externalMongodb.username (.Values.externalMongodb.password | urlquery) (include "codecc.mongodb.fullname" .) (include "codecc.mongodb.port" .) .Values.externalMongodb.extraUrlParams -}}
 {{- end -}}
 {{- end -}}
 
 
 {{- define "codecc.task.mongodbUri" -}}
 {{- if eq .Values.mongodb.enabled true -}}
-{{- printf "mongodb://%s:%s@%s:27017/db_task" .Values.mongodb.auth.username .Values.mongodb.auth.password (include "codecc.mongodb.fullname" .) -}}
+{{- printf "mongodb://%s:%s@%s:%s/db_task" .Values.mongodb.auth.username .Values.mongodb.auth.password (include "codecc.mongodb.fullname" .) (include "codecc.mongodb.port" .) -}}
 {{- else -}}
-{{- .Values.externalMongodb.taskUrl -}}
+{{- printf "mongodb://%s:%s@%s:%s/db_task?%s" .Values.externalMongodb.username (.Values.externalMongodb.password | urlquery) (include "codecc.mongodb.fullname" .) (include "codecc.mongodb.port" .) .Values.externalMongodb.extraUrlParams -}}
 {{- end -}}
 {{- end -}}
 
 {{- define "codecc.quartz.mongodbUri" -}}
 {{- if eq .Values.mongodb.enabled true -}}
-{{- printf "mongodb://%s:%s@%s:27017/db_quartz" .Values.mongodb.auth.username .Values.mongodb.auth.password (include "codecc.mongodb.fullname" .) -}}
+{{- printf "mongodb://%s:%s@%s:%s/db_quartz" .Values.mongodb.auth.username .Values.mongodb.auth.password (include "codecc.mongodb.fullname" .) (include "codecc.mongodb.port" .) -}}
+{{- else -}}
+{{- printf "mongodb://%s:%s@%s:%s/db_quartz?%s" .Values.externalMongodb.username (.Values.externalMongodb.password | urlquery) (include "codecc.mongodb.fullname" .) (include "codecc.mongodb.port" .) .Values.externalMongodb.extraUrlParams -}}
+{{- end -}}
+{{- end -}}
+
+
+{{/*
+Return the mongodb auth ab
+*/}}
+{{- define "codecc.mongodb.defectAuthDB" -}}
+{{- if eq .Values.mongodb.enabled true -}}
+db_defect
 {{- else -}}
-{{- .Values.externalMongodb.quartzUrl -}}
+{{- .Values.externalRedis.authDB -}}
 {{- end -}}
 {{- end -}}
 
+{{- define "codecc.mongodb.taskAuthDB" -}}
+{{- if eq .Values.mongodb.enabled true -}}
+db_task
+{{- else -}}
+{{- .Values.externalRedis.authDB -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "codecc.mongodb.quartzAuthDB" -}}
+{{- if eq .Values.mongodb.enabled true -}}
+db_quartz
+{{- else -}}
+{{- .Values.externalRedis.authDB -}}
+{{- end -}}
+{{- end -}}
+
+
 {{- define "codecc.redis.host" -}}
 {{- if eq .Values.redis.enabled true -}}
 {{- (include "codecc.redis.fullname" .) -}}

diff --git a/helm-charts/core/codecc/templates/job/init-mongodb.yaml b/helm-charts/core/codecc/templates/job/init-mongodb.yaml
@@ -44,10 +44,18 @@ spec:
           command: ['/bin/sh','-c','/data/workspace/init-mongodb.sh']
           env:
             - name: CODECC_USERNAME
-              value: {{ .Values.mongodb.auth.username }}
+              value: {{ include "codecc.mongodb.username" . }}
             - name: CODECC_PASSWORD
-              value: {{ .Values.mongodb.auth.password }}
+              value: {{ include "codecc.mongodb.password" . }}
             - name: CODECC_HOST
               value: {{ include "codecc.mongodb.fullname" . }}
+            - name: CODECC_PORT
+              value: {{ include "codecc.mongodb.port" . | quote }}
+            - name: CODECC_AUTH_DB_DEFECT
+              value: {{ include "codecc.mongodb.defectAuthDB" . }}
+            - name: CODECC_AUTH_DB_TASK
+              value: {{ include "codecc.mongodb.taskAuthDB" . }}
+            - name: CODECC_AUTH_DB_QUARTZ
+              value: {{ include "codecc.mongodb.quartzAuthDB" . }}
       restartPolicy: OnFailure
 {{- end }}
diff --git a/helm-charts/core/codecc/values.yaml b/helm-charts/core/codecc/values.yaml
@@ -88,10 +88,14 @@ mongodb:
       EOF
 ## 如果mongodb.enabled为false，bkrepo将使用下面的参数连接外部mongodb
 externalMongodb:
-  ## mongodb 标准连接字符串
-  defectUrl: mongodb://codecc:codecc@localhost:27017/db_defect
-  taskUrl: mongodb://codecc:codecc@localhost:27017/db_task
-  quartzUrl: mongodb://codecc:codecc@localhost:27017/db_task
+  ## mongodb 帐号密码与
+  host: localhost
+  username: codecc
+  password: codecc
+  port: 27017
+  extraUrlParams: ""
+  authDB: admin
+
 
 redis:
   enabled: true

diff --git a/src/backend/codecc/build.gradle.kts b/src/backend/codecc/build.gradle.kts
@@ -153,6 +153,44 @@ allprojects {
                 entry("jersey-media-multipart")
             }
 
+            /**
+             * 蓝盾依赖
+             */
+            dependencySet("com.tencent.bk.devops.ci.common:${Versions.devopsVersion}") {
+                entry("common-auth-api")
+                entry("common-auth-v3")
+                entry("common-redis")
+                entry("common-pipeline")
+            }
+            dependencySet("com.tencent.bk.devops.ci.auth:${Versions.devopsVersion}") {
+                entry("api-auth")
+            }
+            dependencySet("com.tencent.bk.devops.ci.project:${Versions.devopsVersion}") {
+                entry("api-project")
+            }
+            dependencySet("com.tencent.bk.devops.ci.process:${Versions.devopsVersion}") {
+                entry("api-process")
+            }
+            dependencySet("com.tencent.bk.devops.ci.log:${Versions.devopsVersion}") {
+                entry("api-log")
+            }
+            dependencySet("com.tencent.bk.devops.ci.quality:${Versions.devopsVersion}") {
+                entry("api-quality")
+            }
+            dependencySet("com.tencent.bk.devops.ci.repository:${Versions.devopsVersion}") {
+                entry("api-repository")
+            }
+            dependencySet("com.tencent.bk.devops.ci.notify:${Versions.devopsVersion}") {
+                entry("api-notify")
+            }
+            dependencySet("com.tencent.bk.devops.ci.image:${Versions.devopsVersion}") {
+                entry("api-image")
+            }
+            dependencySet("com.tencent.bk.devops.ci.plugin:${Versions.devopsVersion}") {
+                entry("api-plugin")
+                entry("api-codecc")
+                entry("common-codecc")
+            }
         }
     }
 

diff --git a/src/backend/codecc/core/apiquery/biz-apiquery/build.gradle.kts b/src/backend/codecc/core/apiquery/biz-apiquery/build.gradle.kts
@@ -8,7 +8,7 @@ dependencies {
     api(project(":core:apiquery:api-apiquery"))
     api(project(":core:defect:api-defect"))
     api(project(":core:task:api-task"))
-    api(project(":core:common:common-auth-api"))
+    api(project(":core:common:common-auth:common-auth-api"))
     api(project(":core:schedule:api-schedule"))
     api("io.jsonwebtoken:jjwt")
     api(group = "net.sf.json-lib", name = "json-lib", classifier = "jdk15")

diff --git a/...uery/src/main/java/com/tencent/bk/codecc/apiquery/resources/OpDefectRestResourceImpl.java b/...uery/src/main/java/com/tencent/bk/codecc/apiquery/resources/OpDefectRestResourceImpl.java
@@ -27,7 +27,6 @@
 import com.tencent.devops.common.api.exception.CodeCCException;
 import com.tencent.devops.common.api.pojo.Page;
 import com.tencent.devops.common.api.pojo.Result;
-import com.tencent.devops.common.auth.api.external.AuthExPermissionApi;
 import com.tencent.devops.common.auth.api.util.AuthApiUtils;
 import com.tencent.devops.common.constant.ComConstants;
 import com.tencent.devops.common.constant.CommonMessageCode;

diff --git a/...iquery/biz-apiquery/src/main/kotlin/com/tencent/bk/codecc/apiquery/DefectDBMongoConfig.kt b/...iquery/biz-apiquery/src/main/kotlin/com/tencent/bk/codecc/apiquery/DefectDBMongoConfig.kt
@@ -1,6 +1,12 @@
 package com.tencent.bk.codecc.apiquery
 
+import com.fasterxml.jackson.core.JsonProcessingException
+import com.mongodb.DBObjectCodecProvider
+import com.mongodb.DBRefCodecProvider
+import com.mongodb.client.gridfs.codecs.GridFSFileCodecProvider
+import com.mongodb.client.model.geojson.codecs.GeoJsonCodecProvider
 import com.tencent.bk.codecc.apiquery.pojo.CodeCCMongoProperties
+import org.bson.conversions.Bson
 import org.springframework.boot.context.properties.ConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -10,6 +16,12 @@ import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver
 import org.springframework.data.mongodb.core.convert.MappingMongoConverter
 import org.springframework.data.mongodb.core.mapping.MongoMappingContext
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories
+import java.io.IOException
+import com.tencent.devops.common.api.codecc.util.JsonUtil
+import org.bson.Document
+import org.bson.codecs.*
+import org.bson.codecs.configuration.CodecRegistries
+import org.bson.codecs.configuration.CodecRegistry
 
 @Configuration
 @EnableMongoRepositories(
@@ -20,6 +32,20 @@ class DefectDBMongoConfig {
 
     companion object {
         const val MONGO_TEMPLATE = "defectMongoTemplate"
+        private val DEFAULT_REGISTRY: CodecRegistry = CodecRegistries.fromProviders(
+            listOf(
+                ValueCodecProvider(),
+                BsonValueCodecProvider(),
+                DocumentCodecProvider(),
+                DBRefCodecProvider(),
+                DBObjectCodecProvider(),
+                BsonValueCodecProvider(),
+                GeoJsonCodecProvider(),
+                GridFSFileCodecProvider()
+            )
+        )
+        private val DEFAULT_BSON_TYPE_CLASS_MAP = BsonTypeClassMap()
+        val documentCodec = DocumentCodec(DEFAULT_REGISTRY,DEFAULT_BSON_TYPE_CLASS_MAP)
     }
 
     @ConfigurationProperties(prefix = "spring.data.mongodb.defectdb")
@@ -39,11 +65,29 @@ class DefectDBMongoConfig {
         mongoMappingContext: MongoMappingContext
     ): MappingMongoConverter {
         val dbRefResolver = DefaultDbRefResolver(defectMongoDbFactory)
-        return MappingMongoConverter(dbRefResolver, mongoMappingContext)
+        return object : MappingMongoConverter(dbRefResolver, mongoMappingContext) {
+            override fun <S : Any?> read(clazz: Class<S>, bson: Bson): S {
+                val string = (bson as Document).toJson(documentCodec)
+                try {
+                    return JsonUtil.getObjectMapper().readValue(string, clazz)
+                } catch (e: IOException) {
+                    throw RuntimeException(string, e)
+                }
+            }
+
+            override fun write(obj: Any, bson: Bson) {
+                var string: String? = null
+                try {
+                    string = JsonUtil.getObjectMapper().writeValueAsString(obj)
+                } catch (e: JsonProcessingException) {
+                    throw RuntimeException(string, e)
+                }
+                (bson as Document).putAll(Document.parse(string, documentCodec))
+            }
+        }
     }
 
     @Bean(MONGO_TEMPLATE)
-    fun mongoTemplate(defectMongoDbFactory: SimpleMongoClientDatabaseFactory,
-                      defectMappingMongoConverter: MappingMongoConverter) =
+    fun mongoTemplate(defectMongoDbFactory: SimpleMongoClientDatabaseFactory, defectMappingMongoConverter: MappingMongoConverter) =
         MongoTemplate(defectMongoDbFactory, defectMappingMongoConverter)
 }
diff --git a/...apiquery/biz-apiquery/src/main/kotlin/com/tencent/bk/codecc/apiquery/TaskDBMongoConfig.kt b/...apiquery/biz-apiquery/src/main/kotlin/com/tencent/bk/codecc/apiquery/TaskDBMongoConfig.kt
@@ -1,6 +1,17 @@
 package com.tencent.bk.codecc.apiquery
 
+import com.fasterxml.jackson.core.JsonProcessingException
+import com.mongodb.DBObjectCodecProvider
+import com.mongodb.DBRefCodecProvider
+import com.mongodb.client.gridfs.codecs.GridFSFileCodecProvider
+import com.mongodb.client.model.geojson.codecs.GeoJsonCodecProvider
 import com.tencent.bk.codecc.apiquery.pojo.CodeCCMongoProperties
+import com.tencent.devops.common.api.codecc.util.JsonUtil
+import org.bson.Document
+import org.bson.codecs.*
+import org.bson.codecs.configuration.CodecRegistries
+import org.bson.codecs.configuration.CodecRegistry
+import org.bson.conversions.Bson
 import org.springframework.boot.context.properties.ConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -10,6 +21,8 @@ import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver
 import org.springframework.data.mongodb.core.convert.MappingMongoConverter
 import org.springframework.data.mongodb.core.mapping.MongoMappingContext
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories
+import java.io.IOException
+
 
 @Configuration
 @EnableMongoRepositories(
@@ -20,8 +33,23 @@ class TaskDBMongoConfig {
 
     companion object {
         const val MONGO_TEMPLATE = "taskMongoTemplate"
+        private val DEFAULT_REGISTRY: CodecRegistry = CodecRegistries.fromProviders(
+            listOf(
+                ValueCodecProvider(),
+                BsonValueCodecProvider(),
+                DocumentCodecProvider(),
+                DBRefCodecProvider(),
+                DBObjectCodecProvider(),
+                BsonValueCodecProvider(),
+                GeoJsonCodecProvider(),
+                GridFSFileCodecProvider()
+            )
+        )
+        private val DEFAULT_BSON_TYPE_CLASS_MAP = BsonTypeClassMap()
+        val documentCodec = DocumentCodec(DEFAULT_REGISTRY,DEFAULT_BSON_TYPE_CLASS_MAP)
     }
 
+
     @Bean
     @ConfigurationProperties(prefix = "spring.data.mongodb.taskdb")
     fun taskMongoProperties() = CodeCCMongoProperties()
@@ -36,11 +64,29 @@ class TaskDBMongoConfig {
         mongoMappingContext: MongoMappingContext
     ): MappingMongoConverter {
         val dbRefResolver = DefaultDbRefResolver(taskMongoDbFactory)
-        return MappingMongoConverter(dbRefResolver, mongoMappingContext)
+        return object : MappingMongoConverter(dbRefResolver, mongoMappingContext) {
+            override fun <S : Any?> read(clazz: Class<S>, bson: Bson): S {
+                val string = (bson as Document).toJson(documentCodec)
+                try {
+                    return JsonUtil.getObjectMapper().readValue(string, clazz)
+                } catch (e: IOException) {
+                    throw RuntimeException(string, e)
+                }
+            }
+
+            override fun write(obj: Any, bson: Bson) {
+                var string: String? = null
+                try {
+                    string = JsonUtil.getObjectMapper().writeValueAsString(obj)
+                } catch (e: JsonProcessingException) {
+                    throw RuntimeException(string, e)
+                }
+                (bson as Document).putAll(Document.parse(string, documentCodec))
+            }
+        }
     }
 
     @Bean(MONGO_TEMPLATE)
-    fun mongoTemplate(taskMongoDbFactory: SimpleMongoClientDatabaseFactory,
-                      taskMappingMongoConverter: MappingMongoConverter) =
+    fun mongoTemplate(taskMongoDbFactory: SimpleMongoClientDatabaseFactory, taskMappingMongoConverter: MappingMongoConverter) =
         MongoTemplate(taskMongoDbFactory, taskMappingMongoConverter)
 }