Merge pull request #8438 from stubenhuang/issue-ssl-8434

feat: 支持redis和elasticsearch的加密模式 #8434

helm-charts/core/ci/build_chart.py

@@ -62,7 +62,9 @@
     'bkCiDevopsToken': 'devops',
     'bkCiAppToken': 'test',
     'bkCiNotifyEmailSendChannel': 'blueking',
-    'initLocale': 'zh-CN',
+    'bkCiInitLocale': 'zh-CN',
+    'bkCiRedisSsl': 'false',
+    'bkCiEsHttps': 'false'
 }
 
 if os.path.isfile(default_value_json):
@@ -101,7 +103,11 @@
 for line in env_file:
     if line.startswith('BK_'):
         # 排除掉数据库的相关值
-        if ('_MYSQL_' in line) or ('_REDIS_' in line and 'DB' not in line and 'SENTINEL' not in line) or ('_ES_' in line and 'CLUSTER' not in line) or ('_RABBITMQ_' in line) or ('_INFLUXDB_' in line and 'DB' not in line):
+        if ('_MYSQL_' in line) \
+                or ('_REDIS_' in line and 'DB' not in line and 'SENTINEL' not in line and 'SSL' not in line) \
+                or ('_ES_' in line and 'CLUSTER' not in line and 'HTTPS' not in line) \
+                or ('_RABBITMQ_' in line) \
+                or ('_INFLUXDB_' in line and 'DB' not in line):
             continue
         datas = line.split("=")
         key = datas[0]
@@ -156,7 +162,7 @@
 
 # 生成网关的configmap
 gateway_envs = set(["__BK_CI_PUBLIC_URL__", "__BK_CI_DOCS_URL__", "__BK_CI_PAAS_LOGIN_URL__",
-                    "__BK_CI_VERSION__", "__BK_CI_BADGE_URL__", "__BK_REPO_HOST__"])  # frondend需要的变量
+                    "__BK_CI_VERSION__", "__BK_CI_BADGE_URL__", "__BK_REPO_HOST__","__BK_CI_INIT_LOCALE__"])  # frondend需要的变量
 for file in os.listdir(config_parent):
     if file.startswith('gateway'):
         for line in open(config_parent+file, 'r'):

helm-charts/core/ci/templates/dispatchDocker/deployment.yaml

@@ -97,7 +97,7 @@ spec:
             httpGet:
               path: /management/health/livenessState
               port: http
-            initialDelaySeconds: 90
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 20

helm-charts/core/ci/templates/init/init.plugin.yaml

@@ -37,9 +37,6 @@ spec:
                 echo "{{ include "bkci.names.fullname" . }}-$service is available";
               done 
 
-              touch atom.sql
-              touch atom.redis
-
               {{ range .Values.init.plugin.atoms }}
               atom_code={{ .code }}
               check_atom_url=http://{{ include "bkci.names.fullname" $ }}-store.{{ $.Release.Namespace }}.svc.cluster.local/api/service/pipeline/atom/codes/$atom_code/versions/1.0.0
@@ -54,12 +51,7 @@ spec:
               else
                   echo "atomCode exists"
               fi
-
-              echo 'UPDATE devops_ci_store.T_ATOM SET DEFAULT_FLAG=true WHERE ATOM_CODE="'$atom_code'";' >> atom.sql
-              echo 'SADD STORE_PUBLIC_FLAG_KEY:ATOM '$atom_code >> atom.redis
+              curl -XPOST "http://{{ include "bkci.names.fullname" $ }}-store.{{ $.Release.Namespace }}.svc.cluster.local/api/op/pipeline/atom/default/atomCodes/$atom_code" -H "X-DEVOPS-UID: admin"
               {{ end }}
-
-              mysql -u{{- include "bkci.mysqlUsername" . }} -p{{- include "bkci.mysqlPassword" . }} -h{{ $mysqlData._0 }} -P{{ $mysqlData._1 }} < atom.sql
-              redis-cli -h {{ include "bkci.redisHost" . }} -a {{ include "bkci.redisPassword" . }} -p {{ include "bkci.redisPort" . }} < atom.redis
       restartPolicy: OnFailure
 {{- end -}}

helm-charts/core/ci/templates/metrics/deployment.yaml

@@ -95,7 +95,7 @@ spec:
             httpGet:
               path: /management/health/livenessState
               port: http
-            initialDelaySeconds: 90
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 20

helm-charts/core/ci/templates/notify/deployment.yaml

@@ -95,7 +95,7 @@ spec:
             httpGet:
               path: /management/health/livenessState
               port: http
-            initialDelaySeconds: 90
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 20

helm-charts/core/ci/templates/openapi/deployment.yaml

@@ -95,7 +95,7 @@ spec:
             httpGet:
               path: /management/health/livenessState
               port: http
-            initialDelaySeconds: 270
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 20
@@ -104,7 +104,7 @@ spec:
             httpGet:
               path: /management/health/readinessState
               port: http
-            initialDelaySeconds: 270
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 20

helm-charts/core/ci/templates/plugin/deployment.yaml

@@ -95,7 +95,7 @@ spec:
             httpGet:
               path: /management/health/livenessState
               port: http
-            initialDelaySeconds: 90
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 20

helm-charts/core/ci/templates/store/deployment.yaml

@@ -95,7 +95,7 @@ spec:
             httpGet:
               path: /management/health/livenessState
               port: http
-            initialDelaySeconds: 90
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 30
@@ -104,7 +104,7 @@ spec:
             httpGet:
               path: /management/health/readinessState
               port: http
-            initialDelaySeconds: 90
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 30

helm-charts/core/ci/templates/stream/deployment.yaml

@@ -95,7 +95,7 @@ spec:
             httpGet:
               path: /management/health/livenessState
               port: http
-            initialDelaySeconds: 90
+            initialDelaySeconds: 60
             periodSeconds: 15
             timeoutSeconds: 10
             failureThreshold: 20

scripts/bkenv.properties

@@ -146,7 +146,7 @@ BK_CI_PAAS_DIALOG_LOGIN_URL=$BK_PAAS_PUBLIC_URL/login/plain/?c_url=
 # BK_CI_PAAS_LOGIN_URL默认为$BK_PAAS_PUBLIC_URL/login/?c_url=. 无需修改. 跳转到蓝鲸登录服务主页. CI会在结尾直接追加登录前的地址.
 BK_CI_PAAS_LOGIN_URL=$BK_PAAS_PUBLIC_URL/login/?c_url=
 # 默认语言配置，默认为zh-CN，中文
-INIT_LOCALE=zh-CN
+BK_CI_INIT_LOCALE=zh-CN
 # BK_CI_RABBITMQ_ADDR默认为127.0.0.1:5672. 按需修改. CI专用的RabbitMQ地址, HOST:PORT形式.
 BK_CI_RABBITMQ_ADDR=127.0.0.1:5672
 # BK_CI_RABBITMQ_PASSWORD无默认值. 按需修改.
@@ -163,6 +163,8 @@ BK_CI_REDIS_HOST=127.0.0.1
 BK_CI_REDIS_PASSWORD=
 # BK_CI_REDIS_PORT默认为6379. 按需修改.
 BK_CI_REDIS_PORT=6379
+# BK_CI_REDIS_SSL
+BK_CI_REDIS_SSL=false
 # BK_CI_REDIS_SENTINEL_ADDR无默认值. 无需修改. CI专用的Redis集群地址. 目前暂无法使用集群协议接入.
 BK_CI_REDIS_SENTINEL_ADDR=
 # BK_CI_REDIS_SENTINEL_MASTER_NAME无默认值. 无需修改.
@@ -333,6 +335,8 @@ BK_CI_DOCKER_URL=
 BK_CI_ES_CLUSTER_NAME=bkee-es
 # BK_CI_ES_PASSWORD无默认值. 按需修改. ElasticSearch密码, 如设置则启用认证.
 BK_CI_ES_PASSWORD=
+# BK_CI_ES_HTTPS
+BK_CI_ES_HTTPS=false
 # BK_CI_ES_REST_ADDR默认为127.0.0.1. 按需修改. ElasticSearch REST入口. HOST:PORT
 BK_CI_ES_REST_ADDR=127.0.0.1
 # BK_CI_ES_REST_PORT默认为9200. 按需修改. ElasticSearch REST API端口

src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/OpAtomResource.kt

@@ -211,4 +211,16 @@ interface OpAtomResource {
         @FormDataParam("file")
         disposition: FormDataContentDisposition
     ): Result<Boolean>
+
+    @ApiOperation("设置插件为默认插件")
+    @POST
+    @Path("/default/atomCodes/{atomCode}")
+    fun setDefault(
+        @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE)
+        @HeaderParam(AUTH_HEADER_USER_ID)
+        userId: String,
+        @ApiParam("atomCode", required = true)
+        @PathParam("atomCode")
+        atomCode: String
+    ): Result<Boolean>
 }

src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/AtomFeatureUpdateRequest.kt

@@ -33,5 +33,7 @@ import io.swagger.annotations.ApiModelProperty
 @ApiModel("插件插件市场-插件插件特性信息更新报文体")
 data class AtomFeatureUpdateRequest(
     @ApiModelProperty("插件插件代码库地址", required = false)
-    val repositoryUrl: String?
+    val repositoryUrl: String? = null,
+    @ApiModelProperty("默认插件", required = false)
+    val defaultFlag: Boolean? = null
 )

src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/dao/atom/AtomDao.kt

@@ -761,16 +761,24 @@ class AtomDao : AtomBaseDao() {
                 queryFitAgentBuildLessAtomFlag = queryFitAgentBuildLessAtomFlag
             ) // 开发者测试插件查询条件组装
             // 默认插件和普通插件需排除初始化项目下面有处于测试中或者审核中的插件
-            defaultAtomCondition.add(ta.ATOM_CODE.notIn(dslContext.select(ta.ATOM_CODE)
-                .from(ta).join(tspr).on(ta.ATOM_CODE.eq(tspr.STORE_CODE))
-                .leftJoin(taf).on(ta.ATOM_CODE.eq(taf.ATOM_CODE))
-                .leftJoin(tsst).on(ta.ATOM_CODE.eq(tsst.STORE_CODE))
-                .where(initTestAtomCondition)))
-            normalAtomConditions.add(ta.ATOM_CODE.notIn(dslContext.select(ta.ATOM_CODE)
-                .from(ta).join(tspr).on(ta.ATOM_CODE.eq(tspr.STORE_CODE))
-                .leftJoin(taf).on(ta.ATOM_CODE.eq(taf.ATOM_CODE))
-                .leftJoin(tsst).on(ta.ATOM_CODE.eq(tsst.STORE_CODE))
-                .where(initTestAtomCondition)))
+            defaultAtomCondition.add(
+                ta.ATOM_CODE.notIn(
+                    dslContext.select(ta.ATOM_CODE)
+                        .from(ta).join(tspr).on(ta.ATOM_CODE.eq(tspr.STORE_CODE))
+                        .leftJoin(taf).on(ta.ATOM_CODE.eq(taf.ATOM_CODE))
+                        .leftJoin(tsst).on(ta.ATOM_CODE.eq(tsst.STORE_CODE))
+                        .where(initTestAtomCondition)
+                )
+            )
+            normalAtomConditions.add(
+                ta.ATOM_CODE.notIn(
+                    dslContext.select(ta.ATOM_CODE)
+                        .from(ta).join(tspr).on(ta.ATOM_CODE.eq(tspr.STORE_CODE))
+                        .leftJoin(taf).on(ta.ATOM_CODE.eq(taf.ATOM_CODE))
+                        .leftJoin(tsst).on(ta.ATOM_CODE.eq(tsst.STORE_CODE))
+                        .where(initTestAtomCondition)
+                )
+            )
             queryNormalAtomStep.join(tspr).on(ta.ATOM_CODE.eq(tspr.STORE_CODE))
             queryInitTestAtomStep.join(tspr).on(ta.ATOM_CODE.eq(tspr.STORE_CODE))
         }
@@ -863,9 +871,11 @@ class AtomDao : AtomBaseDao() {
         }
         if (!os.isNullOrBlank() && !KEY_ALL.equals(os, true)) {
             if (fitOsFlag == false) {
-                conditions.add((ta.OS.notLike("%$os%")
-                    .and(ta.BUILD_LESS_RUN_FLAG.ne(true).or(ta.BUILD_LESS_RUN_FLAG.isNull)))
-                    .and(ta.CATEGROY.eq(AtomCategoryEnum.TASK.category.toByte())))
+                conditions.add(
+                    (ta.OS.notLike("%$os%")
+                        .and(ta.BUILD_LESS_RUN_FLAG.ne(true).or(ta.BUILD_LESS_RUN_FLAG.isNull)))
+                        .and(ta.CATEGROY.eq(AtomCategoryEnum.TASK.category.toByte()))
+                )
             } else {
                 conditions.add(ta.OS.contains(os).or(ta.BUILD_LESS_RUN_FLAG.eq(true)))
             }
@@ -961,10 +971,14 @@ class AtomDao : AtomBaseDao() {
             fitOsFlag = fitOsFlag,
             queryFitAgentBuildLessAtomFlag = queryFitAgentBuildLessAtomFlag
         )
-        conditions.add(ta.ATOM_STATUS.`in`(listOf(
-            AtomStatusEnum.TESTING.status.toByte(),
-            AtomStatusEnum.AUDITING.status.toByte()
-        ))) // 只查测试中和审核中的插件
+        conditions.add(
+            ta.ATOM_STATUS.`in`(
+                listOf(
+                    AtomStatusEnum.TESTING.status.toByte(),
+                    AtomStatusEnum.AUDITING.status.toByte()
+                )
+            )
+        ) // 只查测试中和审核中的插件
         conditions.add(tspr.PROJECT_CODE.eq(projectCode))
         conditions.add(tspr.TYPE.`in`(listOf(StoreProjectTypeEnum.TEST.type.toByte()))) // 调试项目
         conditions.add(tspr.STORE_TYPE.eq(StoreTypeEnum.ATOM.type.toByte()))
@@ -1056,6 +1070,9 @@ class AtomDao : AtomBaseDao() {
             if (!atomFeatureUpdateRequest.repositoryUrl.isNullOrBlank()) {
                 baseStep.set(CODE_SRC, atomFeatureUpdateRequest.repositoryUrl)
             }
+            if (atomFeatureUpdateRequest.defaultFlag != null) {
+                baseStep.set(DEFAULT_FLAG, atomFeatureUpdateRequest.defaultFlag)
+            }
             baseStep.set(UPDATE_TIME, LocalDateTime.now())
                 .set(MODIFIER, userId)
                 .where(ATOM_CODE.eq(atomCode))
@@ -1117,8 +1134,10 @@ class AtomDao : AtomBaseDao() {
         )
             .from(ta)
             .join(t)
-            .on(ta.ATOM_CODE.eq(t.field(KEY_ATOM_CODE, String::class.java))
-                .and(ta.CREATE_TIME.eq(t.field(KEY_CREATE_TIME, LocalDateTime::class.java))))
+            .on(
+                ta.ATOM_CODE.eq(t.field(KEY_ATOM_CODE, String::class.java))
+                    .and(ta.CREATE_TIME.eq(t.field(KEY_CREATE_TIME, LocalDateTime::class.java)))
+            )
             .join(tc)
             .on(ta.CLASSIFY_ID.eq(tc.ID))
             .join(tspr)

src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/OpAtomResourceImpl.kt

@@ -136,4 +136,8 @@ class OpAtomResourceImpl @Autowired constructor(
             disposition = disposition
         )
     }
+
+    override fun setDefault(userId: String, atomCode: String): Result<Boolean> {
+        return Result(opAtomService.setDefault(userId = userId, atomCode = atomCode))
+    }
 }

src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/OpAtomService.kt

@@ -81,4 +81,9 @@ interface OpAtomService {
         inputStream: InputStream,
         disposition: FormDataContentDisposition
     ): Result<Boolean>
+
+    /**
+     * 将插件设置为默认
+     */
+    fun setDefault(userId: String, atomCode: String): Boolean
 }

src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/OpAtomServiceImpl.kt

@@ -56,6 +56,7 @@ import com.tencent.devops.store.dao.common.ClassifyDao
 import com.tencent.devops.store.dao.common.LabelDao
 import com.tencent.devops.store.pojo.atom.ApproveReq
 import com.tencent.devops.store.pojo.atom.Atom
+import com.tencent.devops.store.pojo.atom.AtomFeatureUpdateRequest
 import com.tencent.devops.store.pojo.atom.AtomReleaseRequest
 import com.tencent.devops.store.pojo.atom.AtomResp
 import com.tencent.devops.store.pojo.atom.MarketAtomCreateRequest
@@ -82,6 +83,7 @@ import com.tencent.devops.store.utils.AtomReleaseTxtAnalysisUtil
 import com.tencent.devops.store.utils.StoreUtils
 import org.glassfish.jersey.media.multipart.FormDataContentDisposition
 import org.jooq.DSLContext
+import org.jooq.impl.DSL
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Service
@@ -405,8 +407,10 @@ class OpAtomServiceImpl @Autowired constructor(
                 )
             }
             val relativePath = logoUrlAnalysisResult.data
-            val logoFile = File("$atomPath${File.separator}file" +
-                    "${File.separator}${relativePath?.removePrefix(File.separator)}")
+            val logoFile = File(
+                "$atomPath${File.separator}file" +
+                        "${File.separator}${relativePath?.removePrefix(File.separator)}"
+            )
             if (logoFile.exists()) {
                 val result = storeLogoService.uploadStoreLogo(
                     userId = userId,
@@ -512,4 +516,18 @@ class OpAtomServiceImpl @Autowired constructor(
         // 确认测试通过
         return atomReleaseService.passTest(userId, atomId)
     }
+
+    override fun setDefault(userId: String, atomCode: String): Boolean {
+        return try {
+            dslContext.transaction { t ->
+                val context = DSL.using(t)
+                atomDao.updateAtomByCode(context, userId, atomCode, AtomFeatureUpdateRequest(defaultFlag = true))
+                redisOperation.delete(StoreUtils.getStorePublicFlagKey(StoreTypeEnum.ATOM.name)) // 直接删除重建
+            }
+            true
+        } catch (e: Exception) {
+            logger.error("set default atom failed , userId:$userId , atomCode:$atomCode")
+            false
+        }
+    }
 }

src/frontend/devops-nav/src/index.html

@@ -89,7 +89,7 @@
       var PERM_URL_PREFIX = '__BK_PAAS_PUBLIC_URL__/o/bk_iam_app' // EE PERM URL PREFIX
       var DOCS_URL_PREFIX = '__BK_CI_DOCS_URL__' // 文档中心域名
       var LOGIN_SERVICE_URL = '__BK_CI_PAAS_LOGIN_URL__' // 蓝鲸PaaS登录入口
-      var INIT_LOCALE = '__INIT_LOCALE__' // 国际化配置占位符
+      var INIT_LOCALE = '__BK_CI_INIT_LOCALE__' // 国际化配置占位符
       var DEVOPS_LS_VERSION =
         '<%- htmlWebpackPlugin.options.DEVOPS_LS_VERSION%>'
       var X_DEVOPS_PROJECT_ID = 'X-DEVOPS-PROJECT-ID'

src/gateway/core/lua/util/redis_util.lua

@@ -31,7 +31,7 @@ function _M:new()
         return nil
     end
     red:set_timeout(2000) -- 2 second
-    local res, err = red:connect(redisConfig['host'], redisConfig['port'], {backlog = redisConfig['backlog']})
+    local res, err = red:connect(redisConfig['host'], redisConfig['port'], {backlog = redisConfig['backlog'] , ssl = redisConfig['ssl']})
     if not res then
         ngx.log(ngx.ERR, "red connect error:", redisConfig['host'], ",", redisConfig['port'], " ", err)
         return nil

support-files/templates/#etc#ci#application-log.yml

@@ -83,7 +83,7 @@ log:
     username: __BK_CI_ES_USER__
     password: __BK_CI_ES_PASSWORD__
     #  ES集群如果要求HTTPS协议请求，则需要需要打开以下5个配置项， 设置keystore文件和truststore文件以及密码
-#    https: true
+    https: __BK_CI_ES_HTTPS__
 #    keystore:
 #      filePath:
 #      password:

support-files/templates/#etc#ci#common.yml

@@ -36,6 +36,7 @@ spring:
     port: __BK_CI_REDIS_PORT__
     password: __BK_CI_REDIS_PASSWORD__
     database: __BK_CI_REDIS_DB__
+    ssl: __BK_CI_REDIS_SSL__
     pool:
       max-active: 16
       max-wait:  2000