diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/SubtaskPoller.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/SubtaskPoller.kt index 8773f0e455..9248a7c853 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/SubtaskPoller.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/SubtaskPoller.kt @@ -69,9 +69,9 @@ open class SubtaskPoller( open fun dispatch() { executionClusterService.list().forEach { executor.execute { - logger.info("cluster [${it.name}] start to dispatch subtask") + logger.debug("cluster [${it.name}] start to dispatch subtask") dispatcherCache.get(it.name).dispatch() - logger.info("cluster [${it.name}] dispatch finished") + logger.debug("cluster [${it.name}] dispatch finished") } } } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/SubtaskPushDispatcher.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/SubtaskPushDispatcher.kt index d33538a120..321e9f499f 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/SubtaskPushDispatcher.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dispatcher/SubtaskPushDispatcher.kt @@ -36,7 +36,7 @@ abstract class SubtaskPushDispatcher( override fun dispatch() { if (scanService.peek(executionCluster.name) == null) { - logger.info("cluster [${executionCluster.name}] has no subtask to dispatch") + logger.debug("cluster [${executionCluster.name}] has no subtask to dispatch") return } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt index aaae9fe479..0c39a5fcb3 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/TemporaryScanTokenServiceImpl.kt @@ -234,7 +234,7 @@ class TemporaryScanTokenServiceImpl( .build() ) if (nodes.records.isEmpty()) { - throw SystemErrorException(RESOURCE_NOT_FOUND, sha256) + throw ErrorCodeException(RESOURCE_NOT_FOUND, "file[$sha256] of [$projectId:$repoName] not found") } val fullPath = nodes.records[0][NodeDetail::fullPath.name].toString() return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))?.let { node -> diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt index 4df366536d..f51ce3bb52 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt @@ -43,6 +43,7 @@ import com.tencent.bkrepo.auth.pojo.oauth.JsonWebKey import com.tencent.bkrepo.auth.pojo.oauth.JsonWebKeySet import com.tencent.bkrepo.auth.pojo.oauth.OauthToken import com.tencent.bkrepo.auth.dao.repository.OauthTokenRepository +import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.auth.pojo.oauth.OidcConfiguration import com.tencent.bkrepo.auth.pojo.oauth.UserInfo import com.tencent.bkrepo.auth.service.OauthAuthorizationService @@ -146,17 +147,16 @@ class OauthAuthorizationServiceImpl( with(generateTokenRequest) { Preconditions.checkNotNull(clientId, this::clientId.name) Preconditions.checkNotNull(refreshToken, this::refreshToken.name) - val token = oauthTokenRepository.findFirstByAccountIdAndRefreshToken(clientId!!, refreshToken!!) + var token = oauthTokenRepository.findFirstByAccountIdAndRefreshToken(clientId!!, refreshToken!!) ?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, refreshToken!!) - val idToken = generateOpenIdToken( - clientId = clientId!!, + val client = accountDao.findById(clientId!!) + ?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, clientId!!) + token = buildOauthToken( userId = token.userId, - nonce = OauthUtils.generateRandomString(10) + nonce = OauthUtils.generateRandomString(10), + client = client, + openId = token.idToken != null ) - token.accessToken = idToken.toJwtToken() - token.issuedAt = Instant.now(Clock.systemDefaultZone()) - token.idToken?.let { token.idToken = idToken } - oauthTokenRepository.save(token) responseToken(transfer(token)) } } @@ -201,7 +201,7 @@ class OauthAuthorizationServiceImpl( ): TOauthToken { val idToken = generateOpenIdToken(client.id!!, userId, nonce) val tOauthToken = TOauthToken( - accessToken = idToken.toJwtToken(), + accessToken = idToken.toJwtToken(client.scope), refreshToken = OauthUtils.generateRefreshToken(), expireSeconds = oauthProperties.expiredDuration.seconds, type = "Bearer", @@ -300,7 +300,7 @@ class OauthAuthorizationServiceImpl( accessToken = tOauthToken.accessToken, tokenType = tOauthToken.type, scope = if (tOauthToken.scope == null) "" else tOauthToken.scope!!.joinToString(StringPool.COMMA), - idToken = tOauthToken.idToken?.toJwtToken(), + idToken = tOauthToken.idToken?.toJwtToken(tOauthToken.scope), refreshToken = tOauthToken.refreshToken, expiresIn = if (tOauthToken.expireSeconds == null) { null @@ -310,12 +310,15 @@ class OauthAuthorizationServiceImpl( } ) - private fun IdToken.toJwtToken(): String { + private fun IdToken.toJwtToken(scope: Set?): String { + val claims = mutableMapOf() + claims["scope"] = scope.orEmpty() + claims.putAll(JsonUtils.objectMapper.convertValue(this, jacksonTypeRef())) return JwtUtils.generateToken( signingKey = RsaUtils.stringToPrivateKey(cryptoProperties.privateKeyStr2048PKCS8), expireDuration = oauthProperties.expiredDuration, subject = sub, - claims = JsonUtils.objectMapper.convertValue(this, jacksonTypeRef()), + claims = claims, header = mapOf(KEY_ID_NAME to KEY_ID_VALUE), algorithm = SignatureAlgorithm.RS256 ) diff --git a/src/backend/build.gradle.kts b/src/backend/build.gradle.kts index 2813fe9475..0bd3c7fc73 100644 --- a/src/backend/build.gradle.kts +++ b/src/backend/build.gradle.kts @@ -87,6 +87,7 @@ allprojects { exclude(group = "log4j", module = "log4j") exclude(group = "org.slf4j", module = "slf4j-log4j12") exclude(group = "commons-logging", module = "commons-logging") + exclude(group = "org.springframework.boot", module = "spring-boot-starter-tomcat") } tasks.withType { diff --git a/src/backend/common/common-analysis/src/main/kotlin/com/tencent/bkrepo/common/analysis/pojo/scanner/utils/DockerUtils.kt b/src/backend/common/common-analysis/src/main/kotlin/com/tencent/bkrepo/common/analysis/pojo/scanner/utils/DockerUtils.kt index b424947e50..3a4324b16f 100644 --- a/src/backend/common/common-analysis/src/main/kotlin/com/tencent/bkrepo/common/analysis/pojo/scanner/utils/DockerUtils.kt +++ b/src/backend/common/common-analysis/src/main/kotlin/com/tencent/bkrepo/common/analysis/pojo/scanner/utils/DockerUtils.kt @@ -65,9 +65,9 @@ object DockerUtils { password: String?, ) { val images = listImagesCmd().exec() - val exists = images.any { image -> - image.repoTags.any { it == tag } - } + val exists = images?.any { image -> + image.repoTags?.any { it == tag } ?: false + } ?: false if (exists) { return } diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt index b38e19b921..6f4344a61c 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt @@ -167,3 +167,9 @@ const val BKREPO_TRACE = "X-BKREPO-RID" */ const val CODE_PROJECT_PREFIX = "CODE_" const val CLOSED_SOURCE_PREFIX = "CLOSED_SOURCE_" + + +const val AUDITED_UID = "audited_uid" +const val AUDIT_REQUEST_URI = "audit_request_uri" +const val AUDIT_REQUEST_KEY = "http_request" +const val AUDIT_SHARE_USER_ID = "audit_share_user_id" diff --git a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImpl.kt b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImpl.kt index 55f0ced68b..36850d3289 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImpl.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImpl.kt @@ -55,7 +55,6 @@ import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo -import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import org.slf4j.LoggerFactory import org.springframework.data.domain.PageRequest @@ -112,13 +111,20 @@ class ArtifactPreloadPlanServiceImpl( if (!properties.enabled) { return } - val option = NodeListOption(pageSize = properties.maxNodes, includeFolder = false) - val res = nodeService.listNodePageBySha256(sha256, option) - val nodes = res.records + val nodes = nodeService.listNodeBySha256( + sha256 = sha256, + limit = properties.maxNodes, + includeMetadata = false, + includeDeleted = false, + tillLimit = false + ) if (nodes.size >= properties.maxNodes) { // 限制查询出来的最大node数量，避免预加载计划创建时间过久 logger.warn("nodes of sha256[$sha256] exceed max page size[${properties.maxNodes}]") return + } else if (nodes.isEmpty()) { + logger.debug("nodes of sha256[$sha256] found") + return } // node属于同一项目仓库的概率较大，缓存避免频繁查询策略 val strategyCache = HashMap>() @@ -127,11 +133,15 @@ class ArtifactPreloadPlanServiceImpl( for (node in nodes) { val repo = repositoryCache.get(buildRepoId(node.projectId, node.repoName)) if (repo.storageCredentialsKey != credentialsKey) { + logger.debug("credentialsKey of repo[${repo.name}] not match dst credentialsKey[${credentialsKey}]") continue } val strategies = strategyCache.getOrPut(buildRepoId(node.projectId, node.repoName)) { strategyService.list(node.projectId, node.repoName) } + if (strategies.isEmpty()) { + logger.debug("preload strategy of repo[${repo.projectId}/${repo.name}] is empty") + } strategies.forEach { strategy -> matchAndGeneratePlan(strategy, node, repo.storageCredentialsKey)?.let { plans.add(it) } } diff --git a/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImplTest.kt b/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImplTest.kt index a08bdddc42..51d518c844 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImplTest.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/test/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/ArtifactPreloadPlanServiceImplTest.kt @@ -55,6 +55,7 @@ import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test +import org.mockito.ArgumentMatchers.anyBoolean import org.mockito.ArgumentMatchers.anyString import org.mockito.kotlin.any import org.mockito.kotlin.anyOrNull @@ -135,9 +136,9 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor( node.copy(fullPath = "test.txt"), node.copy(fullPath = "test2.txt"), ) - whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn( - Pages.ofResponse(Pages.ofRequest(0, 2000), nodes.size.toLong(), nodes) - ) + whenever( + nodeService.listNodeBySha256(anyString(), any(), anyBoolean(), anyBoolean(), anyBoolean()) + ).thenReturn(nodes) preloadPlanService.generatePlan(null, UT_SHA256) plans = preloadPlanService.plans(UT_PROJECT_ID, UT_REPO_NAME, Pages.ofRequest(0, 10)).records @@ -177,9 +178,9 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor( for (i in 0..1000) { nodes.add(buildNodeInfo()) } - whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn( - Pages.ofResponse(Pages.ofRequest(0, 2000), nodes.size.toLong(), nodes) - ) + whenever( + nodeService.listNodeBySha256(anyString(), any(), anyBoolean(), anyBoolean(), anyBoolean()) + ).thenReturn(nodes) preloadPlanService.generatePlan(null, UT_SHA256) val plans = preloadPlanService.plans(UT_PROJECT_ID, UT_REPO_NAME, Pages.ofRequest(0, 10)).records assertEquals(0, plans.size) @@ -218,9 +219,9 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor( buildRepo(projectId = projectId, repoName = repoName) ) val nodes = listOf(buildNodeInfo(projectId, repoName)) - whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn( - Pages.ofResponse(Pages.ofRequest(0, 20), 1L, nodes) - ) + whenever( + nodeService.listNodeBySha256(anyString(), any(), anyBoolean(), anyBoolean(), anyBoolean()) + ).thenReturn(nodes) } // 构造测试数据 diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ActionAuditContent.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ActionAuditContent.kt index 200de35498..add2656c6a 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ActionAuditContent.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ActionAuditContent.kt @@ -68,9 +68,9 @@ object ActionAuditContent { // 节点 const val NODE_SHARE_CREATE_CONTENT = "create share link for node info $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" - const val NODE_SHARE_DOWNLOAD_CONTENT = "download share node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" - const val NODE_DOWNLOAD_WITH_TOKEN_CONTENT = "download node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" - const val NODE_UPLOAD_WITH_TOKEN_CONTENT = "upload node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_SHARE_DOWNLOAD_CONTENT = "download share node $CONTENT_TEMPLATE with token in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_DOWNLOAD_WITH_TOKEN_CONTENT = "download node $CONTENT_TEMPLATE with token in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_UPLOAD_WITH_TOKEN_CONTENT = "upload node $CONTENT_TEMPLATE with token in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" const val NODE_VIEW_CONTENT = "get node info $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" const val NODE_CREATE_CONTENT = "create node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditPostFilter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditPostFilter.kt index ceced95a67..f4f8aad908 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditPostFilter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditPostFilter.kt @@ -56,10 +56,21 @@ package com.tencent.bkrepo.common.artifact.audit import com.tencent.bk.audit.filter.AuditPostFilter import com.tencent.bk.audit.model.AuditEvent +import com.tencent.bkrepo.common.api.constant.AUDITED_UID +import com.tencent.bkrepo.common.api.constant.AUDIT_REQUEST_KEY +import com.tencent.bkrepo.common.api.constant.AUDIT_REQUEST_URI +import com.tencent.bkrepo.common.api.constant.AUDIT_SHARE_USER_ID class BkAuditPostFilter: AuditPostFilter { override fun map(auditEvent: AuditEvent): AuditEvent { - auditEvent.scopeType = "project" + auditEvent.scopeType = PROJECT_RESOURCE + // 特殊处理, 使用token下载时下载用户是根据token去判断后塞入httpAttribute中, 初始化时无法获取 + if (auditEvent.extendData.isNullOrEmpty()) return auditEvent + auditEvent.extendData[AUDIT_SHARE_USER_ID] ?: return auditEvent + val auditedUid = auditEvent.extendData[AUDITED_UID]?.toString() + val auditRequestUri = auditEvent.extendData[AUDIT_REQUEST_URI] + auditEvent.username = auditedUid ?: auditEvent.username + auditEvent.addExtendData(AUDIT_REQUEST_KEY, auditRequestUri) return auditEvent } } \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt index 78ee7e6b7a..c7095d3fb4 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt @@ -148,6 +148,8 @@ class StorageManager( forwardNode = forward(node, SecurityUtils.getUserId()) forwardNode?.let { logger.info("Load[${node.identity()}] forward to [${it.identity()}].") + ActionAuditContext.current().addExtendData("alphaApkSha256", it.sha256) + ActionAuditContext.current().addExtendData("alphaApkMd5", it.md5) } } val load = forwardNode ?: node diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/memory/ByteArrayArtifactFile.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/memory/ByteArrayArtifactFile.kt new file mode 100644 index 0000000000..035bacdbe3 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/memory/ByteArrayArtifactFile.kt @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.resolve.file.memory + +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.hash.md5 +import com.tencent.bkrepo.common.artifact.hash.sha1 +import com.tencent.bkrepo.common.artifact.hash.sha256 +import java.io.File +import java.io.InputStream + +class ByteArrayArtifactFile( + private val data: ByteArray, +) : ArtifactFile { + + private val sha1: String by lazy { getInputStream().sha1() } + private val sha256: String by lazy { getInputStream().sha256() } + private val md5: String by lazy { getInputStream().md5() } + + override fun getInputStream(): InputStream { + return data.inputStream() + } + + override fun getSize(): Long { + return data.size.toLong() + } + + override fun isInMemory(): Boolean { + return true + } + + override fun getFile(): File? { + throw UnsupportedOperationException("not implemented") + } + + override fun flushToFile(): File { + throw UnsupportedOperationException("not implemented") + } + + override fun delete() { + throw UnsupportedOperationException("not implemented") + } + + override fun hasInitialized(): Boolean { + return true + } + + override fun isFallback(): Boolean { + return false + } + + override fun isInLocalDisk(): Boolean { + return false + } + + override fun getFileMd5(): String { + return md5 + } + + override fun getFileSha1(): String { + return sha1 + } + + override fun getFileSha256(): String { + return sha256 + } + + fun byteArray() = data +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/config/RepositoryProperties.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/config/RepositoryProperties.kt index e6044f4521..b995fd4181 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/config/RepositoryProperties.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/config/RepositoryProperties.kt @@ -57,5 +57,9 @@ data class RepositoryProperties( * 当目录节点上的num字段小于该值时，去db中实时count目录大小 * 注意： 此配置的值要比listCountLimit大 */ - var subNodeLimit: Long = 100000000L + var subNodeLimit: Long = 100000000L, + /** + * 是否返回真实项目启用禁用状态 + */ + var returnEnabled: Boolean = true ) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/node/NodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/node/NodeDao.kt index cf85cf3fea..e80e1c5ee7 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/node/NodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/node/NodeDao.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.common.metadata.dao.node +import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.metadata.condition.SyncCondition @@ -145,6 +146,53 @@ class NodeDao : HashShardingMongoDao() { return pageWithoutShardingKey(pageRequest, query) } + /** + * 根据[sha256]查询node列表，用于不需要分页的场景提高查询速度 + * + * @param sha256 待查询sha256 + * @param limit 查询选项 + * @param includeMetadata 是否包含元数据 + * @param includeDeleted 是否包含被删除的制品 + * @param tillLimit 为true时将遍历所有分表直到查询到的结果数量达到limit + * + * @return 指定sha256的node列表 + */ + fun listBySha256( + sha256: String, + limit: Int = DEFAULT_PAGE_SIZE, + includeMetadata: Boolean =false, + includeDeleted: Boolean = true, + tillLimit: Boolean = true, + ): List { + // 构造查询条件 + val criteria = where(TNode::sha256).isEqualTo(sha256).and(TNode::folder).isEqualTo(false) + if (!includeDeleted) { + criteria.and(TNode::deleted).isEqualTo(null) + } + val query = Query(criteria) + if (!includeMetadata) { + query.fields().exclude(TNode::metadata.name) + } + + if (shardingCount <= 0 || shardingCount > MAX_SHARDING_COUNT_OF_PAGE_QUERY) { + throw UnsupportedOperationException() + } + + // 遍历所有分表进行查询 + val template = determineMongoTemplate() + val result = ArrayList() + for (sequence in 0 until shardingCount) { + query.limit(limit - result.size) + val collectionName = parseSequenceToCollectionName(sequence) + result.addAll(template.find(query, classType, collectionName)) + if (result.isNotEmpty() && !tillLimit || result.size == limit) { + break + } + } + + return result + } + companion object { fun buildRootNode(projectId: String, repoName: String): TNode { return TNode( diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeBaseOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeBaseOperation.kt index 8b33618ed1..1fdabd0287 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeBaseOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeBaseOperation.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.common.metadata.service.node +import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -66,6 +67,17 @@ interface NodeBaseOperation { */ fun listNodePageBySha256(sha256: String, option: NodeListOption): Page + /** + * 根据sha256列出指定数量的节点 + */ + fun listNodeBySha256( + sha256: String, + limit: Int = DEFAULT_PAGE_SIZE, + includeMetadata: Boolean =false, + includeDeleted: Boolean = true, + tillLimit: Boolean = true, + ): List + /** * 判断节点是否存在 */ diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index 7ff238397f..eee31154b0 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -160,6 +160,16 @@ abstract class NodeBaseService( ) } + override fun listNodeBySha256( + sha256: String, + limit: Int, + includeMetadata: Boolean, + includeDeleted: Boolean, + tillLimit: Boolean + ): List { + return nodeDao.listBySha256(sha256, limit, includeMetadata, includeDeleted, tillLimit).map { convert(it)!! } + } + override fun checkExist(artifact: ArtifactInfo): Boolean { return nodeDao.exists(artifact.projectId, artifact.repoName, artifact.getArtifactFullPath()) } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/CenterProjectServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/CenterProjectServiceImpl.kt index 0fba41993b..b69fdb8d1e 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/CenterProjectServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/CenterProjectServiceImpl.kt @@ -30,6 +30,7 @@ package com.tencent.bkrepo.common.metadata.service.project.impl import com.tencent.bkrepo.auth.api.ServiceBkiamV3ResourceClient import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition import com.tencent.bkrepo.common.metadata.dao.project.ProjectDao import com.tencent.bkrepo.common.metadata.dao.project.ProjectMetricsDao @@ -45,10 +46,12 @@ class CenterProjectServiceImpl( projectMetricsDao: ProjectMetricsDao, serviceBkiamV3ResourceClient: ServiceBkiamV3ResourceClient, storageCredentialService: StorageCredentialService, -) : ProjectServiceImpl( + repositoryProperties: RepositoryProperties, + ) : ProjectServiceImpl( projectDao, servicePermissionClient, projectMetricsDao, serviceBkiamV3ResourceClient, - storageCredentialService + storageCredentialService, + repositoryProperties ) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/EdgeProjectServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/EdgeProjectServiceImpl.kt index 105cbfe21d..e467b2e28e 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/EdgeProjectServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/EdgeProjectServiceImpl.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.auth.api.ServiceBkiamV3ResourceClient import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.util.ClusterUtils.reportMetadataToCenter import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties @@ -54,12 +55,14 @@ class EdgeProjectServiceImpl( clusterProperties: ClusterProperties, projectMetricsDao: ProjectMetricsDao, storageCredentialService: StorageCredentialService, -) : ProjectServiceImpl( + repositoryProperties: RepositoryProperties, + ) : ProjectServiceImpl( projectDao, servicePermissionClient, projectMetricsDao, serviceBkiamV3ResourceClient, - storageCredentialService + storageCredentialService, + repositoryProperties ) { private val centerProjectClient: ClusterProjectClient by lazy { diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/ProjectServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/ProjectServiceImpl.kt index 38c1ea1cbb..d7a6994332 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/ProjectServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/ProjectServiceImpl.kt @@ -37,6 +37,7 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.dao.project.ProjectDao import com.tencent.bkrepo.common.metadata.dao.project.ProjectMetricsDao import com.tencent.bkrepo.common.metadata.listener.ResourcePermissionListener @@ -84,6 +85,7 @@ class ProjectServiceImpl( private val projectMetricsDao: ProjectMetricsDao, private val serviceBkiamV3ResourceClient: ServiceBkiamV3ResourceClient, private val storageCredentialService: StorageCredentialService, + private val repositoryProperties: RepositoryProperties, ) : ProjectService { @Autowired @@ -134,6 +136,7 @@ class ProjectServiceImpl( } override fun isProjectEnabled(name: String): Boolean { + if (!repositoryProperties.returnEnabled) return true val projectInfo = projectDao.findByName(name) ?: throw ErrorCodeException(ArtifactMessageCode.PROJECT_NOT_FOUND, name) return ProjectServiceHelper.isProjectEnabled(projectInfo) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/RProjectServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/RProjectServiceImpl.kt index 8ca465746a..6cad7f1c35 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/RProjectServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/project/impl/RProjectServiceImpl.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.metadata.client.RAuthClient import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition +import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.dao.project.RProjectDao import com.tencent.bkrepo.common.metadata.dao.project.RProjectMetricsDao import com.tencent.bkrepo.common.metadata.listener.RResourcePermissionListener @@ -82,6 +83,7 @@ class RProjectServiceImpl( private val projectMetricsDao: RProjectMetricsDao, private val storageCredentialService: RStorageCredentialService, private val rAuthClient: RAuthClient, + private val repositoryProperties: RepositoryProperties, ) : RProjectService { @Autowired @@ -133,6 +135,7 @@ class RProjectServiceImpl( } override suspend fun isProjectEnabled(name: String): Boolean { + if (!repositoryProperties.returnEnabled) return true val projectInfo = projectDao.findByName(name) ?: throw ErrorCodeException(ArtifactMessageCode.PROJECT_NOT_FOUND, name) return ProjectServiceHelper.isProjectEnabled(projectInfo) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/ClusterUtils.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/ClusterUtils.kt index 36232bf192..db024912a2 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/ClusterUtils.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/ClusterUtils.kt @@ -38,6 +38,7 @@ import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.exception.RemoteErrorCodeException import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.SpringContextUtils +import feign.RetryableException import org.slf4j.LoggerFactory import org.springframework.util.AntPathMatcher @@ -154,6 +155,12 @@ object ClusterUtils { if (!messageCodes.map { it.getCode() }.contains(e.errorCode)) { throw e } + } catch (e: RetryableException) { + if (e.message?.contains("Read timed out") == true) { + logger.info("ignore feign exception: ${e.message}") + return + } + throw e } } } diff --git a/src/backend/common/common-metrics-push/src/main/kotlin/com/tencent/bkrepo/common/metrics/CustomMetricsPushAutoConfiguration.kt b/src/backend/common/common-metrics-push/src/main/kotlin/com/tencent/bkrepo/common/metrics/CustomMetricsPushAutoConfiguration.kt index 9b001532fd..4b30537e63 100644 --- a/src/backend/common/common-metrics-push/src/main/kotlin/com/tencent/bkrepo/common/metrics/CustomMetricsPushAutoConfiguration.kt +++ b/src/backend/common/common-metrics-push/src/main/kotlin/com/tencent/bkrepo/common/metrics/CustomMetricsPushAutoConfiguration.kt @@ -101,10 +101,10 @@ class CustomMetricsPushAutoConfiguration { fun customMetricsExporter( drive: PrometheusDrive, prometheusProperties: PrometheusProperties, - scheduler: ThreadPoolTaskScheduler, + taskScheduler: ThreadPoolTaskScheduler, customPushConfig: CustomPushConfig, ): CustomMetricsExporter { - return CustomMetricsExporter(customPushConfig, CollectorRegistry(), drive, prometheusProperties, scheduler) + return CustomMetricsExporter(customPushConfig, CollectorRegistry(), drive, prometheusProperties, taskScheduler) } diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt index ab19360ee1..58b66f815f 100644 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt +++ b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt @@ -312,6 +312,6 @@ abstract class ShardingMongoDao : AbstractMongoDao() { companion object { private val logger = LoggerFactory.getLogger(ShardingMongoDao::class.java) - private const val MAX_SHARDING_COUNT_OF_PAGE_QUERY = 256 + const val MAX_SHARDING_COUNT_OF_PAGE_QUERY = 256 } } diff --git a/src/backend/common/common-redis/src/main/kotlin/com/tencent/bkrepo/common/redis/RedisOperation.kt b/src/backend/common/common-redis/src/main/kotlin/com/tencent/bkrepo/common/redis/RedisOperation.kt index e851e8475b..de21b92aa1 100644 --- a/src/backend/common/common-redis/src/main/kotlin/com/tencent/bkrepo/common/redis/RedisOperation.kt +++ b/src/backend/common/common-redis/src/main/kotlin/com/tencent/bkrepo/common/redis/RedisOperation.kt @@ -63,8 +63,8 @@ class RedisOperation(private val redisTemplate: RedisTemplate) { } } - fun delete(key: String) { - redisTemplate.delete(key) + fun delete(key: String): Boolean { + return redisTemplate.delete(key) } fun delete(keys: Collection) { diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/http/HttpAuthSecurityConfiguration.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/http/HttpAuthSecurityConfiguration.kt index 18bafae662..dab3823234 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/http/HttpAuthSecurityConfiguration.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/http/HttpAuthSecurityConfiguration.kt @@ -110,7 +110,7 @@ class HttpAuthSecurityConfiguration( ) } if (httpAuthSecurity.oauthEnabled) { - httpAuthSecurity.addHttpAuthHandler(OauthAuthHandler(authenticationManager)) + httpAuthSecurity.addHttpAuthHandler(OauthAuthHandler(authenticationManager, cryptoProperties)) } if (httpAuthSecurity.temporaryTokenEnabled) { httpAuthSecurity.addHttpAuthHandler(TemporaryTokenAuthHandler(authenticationManager)) diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/http/oauth/OauthAuthHandler.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/http/oauth/OauthAuthHandler.kt index b8de944aaa..43ea388c76 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/http/oauth/OauthAuthHandler.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/http/oauth/OauthAuthHandler.kt @@ -30,15 +30,21 @@ package com.tencent.bkrepo.common.security.http.oauth import com.tencent.bkrepo.common.api.constant.AUTHORITIES_KEY import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.OAUTH_AUTH_PREFIX +import com.tencent.bkrepo.common.security.crypto.CryptoProperties import com.tencent.bkrepo.common.security.exception.AuthenticationException import com.tencent.bkrepo.common.security.http.core.HttpAuthHandler import com.tencent.bkrepo.common.security.http.credentials.AnonymousCredentials import com.tencent.bkrepo.common.security.http.credentials.HttpAuthCredentials import com.tencent.bkrepo.common.security.manager.AuthenticationManager +import com.tencent.bkrepo.common.security.util.JwtUtils +import com.tencent.bkrepo.common.security.util.RsaUtils +import org.slf4j.LoggerFactory import javax.servlet.http.HttpServletRequest -import javax.servlet.http.HttpServletResponse -open class OauthAuthHandler(val authenticationManager: AuthenticationManager) : HttpAuthHandler { +open class OauthAuthHandler( + val authenticationManager: AuthenticationManager, + val cryptoProperties: CryptoProperties +) : HttpAuthHandler { override fun extractAuthCredentials(request: HttpServletRequest): HttpAuthCredentials { val authorizationHeader = request.getHeader(HttpHeaders.AUTHORIZATION).orEmpty() return if (authorizationHeader.startsWith(OAUTH_AUTH_PREFIX)) { @@ -53,14 +59,24 @@ open class OauthAuthHandler(val authenticationManager: AuthenticationManager) : override fun onAuthenticate(request: HttpServletRequest, authCredentials: HttpAuthCredentials): String { require(authCredentials is OauthAuthCredentials) - return authenticationManager.checkOauthToken(authCredentials.token) + return try { + val claims = JwtUtils.validateToken( + signingKey = RsaUtils.stringToPrivateKey(cryptoProperties.privateKeyStr2048PKCS8), + token = authCredentials.token + ) + val scopeList = claims.body["scope"] as? List<*> + val scope = scopeList?.joinToString(",") + ?: authenticationManager.findOauthToken(authCredentials.token)?.scope + ?: throw AuthenticationException("Invalid access token: $authCredentials.token") + request.setAttribute(AUTHORITIES_KEY, scope) + claims.body.subject + } catch (e: Exception) { + logger.info("invalid oauth token[${authCredentials.token}]: ${e.message}") + throw AuthenticationException("Invalid token") + } } - override fun onAuthenticateSuccess(request: HttpServletRequest, response: HttpServletResponse, userId: String) { - val authorizationHeader = request.getHeader(HttpHeaders.AUTHORIZATION).orEmpty() - val accessToken = authorizationHeader.removePrefix(OAUTH_AUTH_PREFIX).trim() - val oauthToken = authenticationManager.findOauthToken(accessToken) - ?: throw AuthenticationException("Invalid access token") - request.setAttribute(AUTHORITIES_KEY, oauthToken.scope) + companion object { + private val logger = LoggerFactory.getLogger(OauthAuthHandler::class.java) } } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanController.kt index 2ad6d20855..b07f2d4232 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanController.kt @@ -40,11 +40,7 @@ import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission -import com.tencent.bkrepo.conan.constant.CONANFILE -import com.tencent.bkrepo.conan.constant.CONANINFO import com.tencent.bkrepo.conan.constant.CONAN_MANIFEST -import com.tencent.bkrepo.conan.constant.EXPORT_SOURCES_TGZ_NAME -import com.tencent.bkrepo.conan.constant.PACKAGE_TGZ_NAME import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.GET_CONANFILE_DOWNLOAD_URLS_V1 import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.GET_PACKAGE_DOWNLOAD_URLS_V1 @@ -86,7 +82,7 @@ class ConanController( ): ResponseEntity { return ConanCommonController.buildResponse( conanService.getConanFileDownloadUrls( - conanArtifactInfo, mutableListOf(CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME, CONANFILE) + conanArtifactInfo, mutableListOf(CONAN_MANIFEST) ) ) } @@ -101,7 +97,7 @@ class ConanController( ): ResponseEntity { return ConanCommonController.buildResponse( conanService.getPackageDownloadUrls( - conanArtifactInfo, mutableListOf(CONAN_MANIFEST, CONANINFO, PACKAGE_TGZ_NAME) + conanArtifactInfo, mutableListOf(CONAN_MANIFEST) ) ) } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt index 4dc9ab4299..3b49c49421 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt @@ -27,10 +27,7 @@ package com.tencent.bkrepo.conan.service.impl -import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.metadata.service.packages.PackageService -import com.tencent.bkrepo.conan.constant.ConanMessageCode -import com.tencent.bkrepo.conan.exception.ConanSearchNotFoundException import com.tencent.bkrepo.conan.pojo.ConanInfo import com.tencent.bkrepo.conan.pojo.ConanSearchResult import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo @@ -61,12 +58,6 @@ class ConanSearchServiceImpl : ConanSearchService { ): ConanSearchResult { val realPattern = pattern?.replace("*", ".*") val recipes = searchRecipes(projectId, repoName, realPattern, ignoreCase) - if (recipes.isEmpty()) { - throw ConanSearchNotFoundException( - ConanMessageCode.CONAN_SEARCH_NOT_FOUND, - pattern ?: StringPool.EMPTY, "$projectId/$repoName" - ) - } return ConanSearchResult(recipes) } diff --git a/src/backend/config/boot-config/build.gradle.kts b/src/backend/config/boot-config/build.gradle.kts index 01bbf7507b..516184f9fb 100644 --- a/src/backend/config/boot-config/build.gradle.kts +++ b/src/backend/config/boot-config/build.gradle.kts @@ -32,4 +32,5 @@ dependencies { implementation("org.springframework.cloud:spring-cloud-config-server:3.1.6") implementation("org.springframework.cloud:spring-cloud-starter-bootstrap:3.1.6") + implementation("io.undertow:undertow-servlet") } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt index 40f1946cab..b45a6953e9 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt @@ -39,6 +39,7 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository +import com.tencent.bkrepo.common.artifact.resolve.file.memory.ByteArrayArtifactFile import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range @@ -55,6 +56,7 @@ import com.tencent.bkrepo.ddc.exception.ReferenceIsMissingBlobsException import com.tencent.bkrepo.ddc.metrics.DdcMeterBinder import com.tencent.bkrepo.ddc.pojo.Blob import com.tencent.bkrepo.ddc.pojo.ContentHash +import com.tencent.bkrepo.ddc.pojo.CreateRefResponse import com.tencent.bkrepo.ddc.pojo.Reference import com.tencent.bkrepo.ddc.pojo.UploadCompressedBlobResponse import com.tencent.bkrepo.ddc.serialization.CbObject @@ -73,6 +75,7 @@ import com.tencent.bkrepo.ddc.utils.isAttachment import com.tencent.bkrepo.ddc.utils.isBinaryAttachment import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest +import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.io.ByteArrayInputStream @@ -207,21 +210,13 @@ class DdcLocalRepository( } private fun onUploadReference(context: ArtifactUploadContext) { - val contentType = context.request.contentType - val artifactInfo = context.artifactInfo as ReferenceArtifactInfo - when (contentType) { + when (val contentType = context.request.contentType) { MEDIA_TYPE_UNREAL_COMPACT_BINARY -> { - val payload = context.getArtifactFile().getInputStream().use { it.readBytes() } - val ref = referenceService.create(Reference.from(artifactInfo, payload)) - if (ref.inlineBlob == null) { - // inlineBlob为null时表示inlineBlob过大，需要存到文件中 - val nodeCreateRequest = buildRefNodeCreateRequest(context) - storageManager.storeArtifactFile( - nodeCreateRequest, context.getArtifactFile(), context.storageCredentials - ) - } - val res = referenceService.finalize(ref, payload) - HttpContextHolder.getResponse().writer.println(res.toJsonString()) + val artifactInfo = context.artifactInfo as ReferenceArtifactInfo + val artifactFile = context.getArtifactFile() + val repoDetail = context.repositoryDetail + val res = uploadReference(repoDetail, artifactInfo, artifactFile, context.userId).toJsonString() + HttpContextHolder.getResponse().writer.println(res) } else -> throw BadRequestException( @@ -230,8 +225,32 @@ class DdcLocalRepository( } } - private fun buildRefNodeCreateRequest(context: ArtifactUploadContext): NodeCreateRequest { - val artifactInfo = context.artifactInfo as ReferenceArtifactInfo + fun uploadReference( + repositoryDetail: RepositoryDetail, + artifactInfo: ReferenceArtifactInfo, + artifactFile: ArtifactFile, + operator: String, + ): CreateRefResponse { + val payload = if (artifactFile is ByteArrayArtifactFile) { + artifactFile.byteArray() + } else { + artifactFile.getInputStream().use { it.readBytes() } + } + val ref = referenceService.create(Reference.from(artifactInfo, payload), operator) + if (ref.inlineBlob == null) { + // inlineBlob为null时表示inlineBlob过大，需要存到文件中 + val nodeCreateRequest = buildRefNodeCreateRequest(repositoryDetail, artifactInfo, artifactFile, operator) + storageManager.storeArtifactFile(nodeCreateRequest, artifactFile, repositoryDetail.storageCredentials) + } + return referenceService.finalize(ref, payload) + } + + private fun buildRefNodeCreateRequest( + repositoryDetail: RepositoryDetail, + artifactInfo: ReferenceArtifactInfo, + artifactFile: ArtifactFile, + operator: String, + ): NodeCreateRequest { val metadata = ArrayList() metadata.add( MetadataModel( @@ -241,7 +260,15 @@ class DdcLocalRepository( ) ) - return buildNodeCreateRequest(context).copy( + return NodeCreateRequest( + projectId = repositoryDetail.projectId, + repoName = repositoryDetail.name, + folder = false, + fullPath = artifactInfo.getArtifactFullPath(), + size = artifactFile.getSize(), + sha256 = artifactFile.getFileSha256(), + md5 = artifactFile.getFileMd5(), + operator = operator, overwrite = true, nodeMetadata = metadata ) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcConfiguration.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcConfiguration.kt index 0472f74d5a..18ab00597a 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcConfiguration.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcConfiguration.kt @@ -27,9 +27,46 @@ package com.tencent.bkrepo.ddc.config +import org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.task.TaskExecutorBuilder +import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Lazy +import org.springframework.context.annotation.Primary +import org.springframework.scheduling.annotation.AsyncAnnotationBeanPostProcessor +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor +import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(DdcProperties::class) -class DdcConfiguration +class DdcConfiguration { + @Bean(BEAN_NAME_REF_BATCH_EXECUTOR) + fun refBatchExecutor(ddcProperties: DdcProperties): ThreadPoolTaskExecutor { + return ThreadPoolTaskExecutor().apply { + corePoolSize = ddcProperties.refBatchWorker + maxPoolSize = ddcProperties.refBatchWorker + setAllowCoreThreadTimeOut(true) + keepAliveSeconds = 60 + queueCapacity = ddcProperties.refBatchQueueSize + threadNamePrefix = "ddc-ref-batch-%d" + setRejectedExecutionHandler(CallerRunsPolicy()) + } + } + + @Lazy + @Bean( + name = [ + TaskExecutionAutoConfiguration.APPLICATION_TASK_EXECUTOR_BEAN_NAME, + AsyncAnnotationBeanPostProcessor.DEFAULT_TASK_EXECUTOR_BEAN_NAME + ] + ) + @Primary + fun applicationTaskExecutor(builder: TaskExecutorBuilder): ThreadPoolTaskExecutor { + return builder.build() + } + + companion object { + const val BEAN_NAME_REF_BATCH_EXECUTOR = "refBatchExecutor" + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcProperties.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcProperties.kt index c895b808c7..ba31c55986 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcProperties.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcProperties.kt @@ -35,4 +35,12 @@ import org.springframework.util.unit.DataUnit data class DdcProperties( var inlineBlobMaxSize: DataSize = DataSize.of(64L, DataUnit.KILOBYTES), var updateRefLastAccessTime: Boolean = true, + /** + * 批量操作接口工作线程数 + */ + var refBatchWorker: Int = Runtime.getRuntime().availableProcessors() * 2, + /** + * 批量操作接口线程池队列大小 + */ + var refBatchQueueSize: Int = 16, ) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt index ce7e37cb75..e492000e4d 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt @@ -37,13 +37,15 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode.PARAMETER_INVALID import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE -import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo import com.tencent.bkrepo.ddc.artifact.repository.DdcLocalRepository.Companion.HEADER_NAME_HASH import com.tencent.bkrepo.ddc.component.PermissionHelper +import com.tencent.bkrepo.ddc.pojo.BatchOps +import com.tencent.bkrepo.ddc.pojo.Operation import com.tencent.bkrepo.ddc.service.ReferenceArtifactService import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_JUPITER_INLINED_PAYLOAD import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_COMPACT_BINARY @@ -58,6 +60,7 @@ import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.PutMapping import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RestController +import java.nio.ByteBuffer @RequestMapping("/{projectId}/api/v1/refs") @RestController @@ -153,6 +156,36 @@ class ReferencesController( referenceArtifactService.finalize(artifactInfo) } + @ApiOperation("批量读写") + @PostMapping( + "/{repoName}", + consumes = [MEDIA_TYPE_UNREAL_COMPACT_BINARY], + produces = [MEDIA_TYPE_UNREAL_COMPACT_BINARY], + ) + fun batchOp( + @PathVariable projectId: String, + @PathVariable repoName: String, + ): ByteArray { + // 检查权限 + val ops = BatchOps.deserialize(HttpContextHolder.getRequest().inputStream.use { it.readBytes() }) + var requiredPermissionAction = PermissionAction.READ + for (op in ops.ops) { + if (op.op == Operation.PUT.name) { + requiredPermissionAction = PermissionAction.WRITE + break + } + } + permissionHelper.checkPathPermission(requiredPermissionAction) + + // 执行操作 + val opsResponse = referenceArtifactService.batch(projectId, repoName, ops) + // 序列化,由于getView返回的是readOnly ByteBuffer，为了避免数组复制，通过反射获取内部数组返回 + val data = opsResponse.serialize().getView() + val field = ByteBuffer::class.java.getDeclaredField("hb") + field.isAccessible = true + return field.get(data) as ByteArray + } + private fun getResponseType(format: String?, default: String): String { if (!format.isNullOrEmpty()) { return when (format.toLowerCase()) { diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/ReferenceIsMissingBlobsException.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/ReferenceIsMissingBlobsException.kt index 13fcfdba5a..54719deac1 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/ReferenceIsMissingBlobsException.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/ReferenceIsMissingBlobsException.kt @@ -27,8 +27,15 @@ package com.tencent.bkrepo.ddc.exception +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.ddc.pojo.ContentHash class ReferenceIsMissingBlobsException( val missingBlobs: List -) : RuntimeException("References is missing these blobs: ${missingBlobs.joinToString(",")}") +) : ErrorCodeException( + CommonMessageCode.RESOURCE_NOT_FOUND, + missingBlobs.joinToString(","), + status = HttpStatus.NOT_FOUND +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt index 843e0159de..c0d3f05249 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt @@ -27,14 +27,21 @@ package com.tencent.bkrepo.ddc.metrics +import com.tencent.bkrepo.ddc.config.DdcConfiguration.Companion.BEAN_NAME_REF_BATCH_EXECUTOR import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.Gauge import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.Timer import io.micrometer.core.instrument.binder.MeterBinder +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor import org.springframework.stereotype.Component @Component -class DdcMeterBinder : MeterBinder { +class DdcMeterBinder( + @Qualifier(BEAN_NAME_REF_BATCH_EXECUTOR) + val refBatchExecutor: ThreadPoolTaskExecutor +) : MeterBinder { /** * ref inline加载耗时 */ @@ -103,6 +110,14 @@ class DdcMeterBinder : MeterBinder { .tag("type", "compressed") .tag("method", "load") .register(registry) + + Gauge.builder(DDC_REF_BATCH_EXECUTOR_ACTIVE, refBatchExecutor) { it.activeCount.toDouble() } + .description("ddc ref batch executor active thread") + .register(registry) + + Gauge.builder(DDC_REF_BATCH_EXECUTOR_QUEUE_SIZE, refBatchExecutor) { it.queueSize.toDouble() } + .description("ddc ref batch executor queue size") + .register(registry) } /** @@ -137,5 +152,7 @@ class DdcMeterBinder : MeterBinder { private const val DDC_REF_LOAD = "ddc.ref.load" private const val DDC_REF_STORE = "ddc.ref.store" private const val DDC_BLOB = "ddc.blob" + private const val DDC_REF_BATCH_EXECUTOR_ACTIVE = "ddc.ref.batch.executor.active.count" + private const val DDC_REF_BATCH_EXECUTOR_QUEUE_SIZE = "ddc.ref.batch.executor.queue.size" } } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/BatchOps.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/BatchOps.kt new file mode 100644 index 0000000000..9b01254fa5 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/BatchOps.kt @@ -0,0 +1,133 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.ddc.serialization.CbArray +import com.tencent.bkrepo.ddc.serialization.CbField +import com.tencent.bkrepo.ddc.serialization.CbFieldType +import com.tencent.bkrepo.ddc.serialization.CbObject +import com.tencent.bkrepo.ddc.utils.beginUniformArray +import com.tencent.bkrepo.ddc.utils.isBool +import com.tencent.bkrepo.ddc.utils.isInteger +import com.tencent.bkrepo.ddc.utils.isObject +import com.tencent.bkrepo.ddc.utils.isString +import com.tencent.bkrepo.ddc.utils.writeBool +import com.tencent.bkrepo.ddc.utils.writeInteger +import com.tencent.bkrepo.ddc.utils.writeString +import com.tencent.bkrepo.ddc.utils.writerObject +import java.nio.ByteBuffer + +/** + * 批量操作 + */ +data class BatchOps( + val ops: List +) { + + fun serialize(): CbObject { + return CbObject.build { writer -> + writer.beginUniformArray(BatchOps::ops.name, CbFieldType.Object) + ops.forEach { op -> + writer.beginObject() + writer.writeInteger(BatchOp::opId.name, op.opId) + writer.writeString(BatchOp::bucket.name, op.bucket) + writer.writeString(BatchOp::key.name, op.key) + writer.writeString(BatchOp::op.name, op.op) + writer.writeBool(BatchOp::resolveAttachments.name, op.resolveAttachments) + op.payload?.let { writer.writerObject(BatchOp::payload.name, it) } + op.payloadHash?.let { writer.writeString(BatchOp::payloadHash.name, it) } + writer.endObject() + } + writer.endArray() + } + } + + companion object { + fun deserialize(byteArray: ByteArray): BatchOps { + val opsCbArray = CbObject(ByteBuffer.wrap(byteArray))[BatchOps::ops.name].asArray() + if (opsCbArray == CbArray.EMPTY) { + throw ErrorCodeException(CommonMessageCode.PARAMETER_INVALID, "ops is empty") + } + val ops = opsCbArray.map { deserializeBatchOp(BatchOp(), it.asObject()) } + return BatchOps(ops) + } + + /** + * 仅支持反序列化最外层的属性，且只支持了BatchOp的字段类型 + */ + private fun deserializeBatchOp(obj: BatchOp, cbObject: CbObject): BatchOp { + fun valOf(field: CbField) = when { + field.isBool() -> field.asBool() as Any + field.isString() -> field.asString() + field.isInteger() -> field.asUInt32() + field.isObject() -> field.asObject() + else -> throw RuntimeException("unsupported field type ${field.getType()}") + } + cbObject.forEach { cbField -> + val field = obj.javaClass.getDeclaredField(cbField.name) + field.isAccessible = true + field.set(obj, valOf(cbField)) + } + return obj + } + } +} + +/** + * 操作 + */ +data class BatchOp( + val opId: Int = 0, + val bucket: String = "", + val key: String = "", + val op: String = Operation.INVALID.toString(), + /** + * 是否检查ref引用的所有blob是否存在 + */ + val resolveAttachments: Boolean = false, + /** + * ref inline blob，op为PUT时有值 + */ + val payload: CbObject? = null, + /** + * ref inline blob hash, op为PUT时有值 + */ + val payloadHash: String? = null, +) + +/** + * 操作类型 + */ +enum class Operation { + INVALID, + GET, + PUT, + HEAD; +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/BatchOpsResponse.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/BatchOpsResponse.kt new file mode 100644 index 0000000000..c022598c35 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/BatchOpsResponse.kt @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +import com.tencent.bkrepo.ddc.serialization.CbFieldType +import com.tencent.bkrepo.ddc.serialization.CbObject +import com.tencent.bkrepo.ddc.utils.beginUniformArray +import com.tencent.bkrepo.ddc.utils.writeInteger +import com.tencent.bkrepo.ddc.utils.writerObject + +data class BatchOpsResponse( + val results: List +) { + fun serialize(): CbObject { + return CbObject.build { writer -> + writer.beginUniformArray(BatchOpsResponse::results.name, CbFieldType.Object) + results.forEach { + writer.beginObject() + writer.writeInteger(OpResponse::opId.name, it.opId) + writer.writerObject(OpResponse::response.name, it.response) + writer.writeInteger(OpResponse::statusCode.name, it.statusCode) + writer.endObject() + } + writer.endArray() + } + } +} + +data class OpResponse( + val opId: Int, + val response: CbObject, + val statusCode: Int +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/CreateRefResponse.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/CreateRefResponse.kt index 021971fde7..e0421029f5 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/CreateRefResponse.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/CreateRefResponse.kt @@ -27,6 +27,19 @@ package com.tencent.bkrepo.ddc.pojo +import com.tencent.bkrepo.ddc.serialization.CbFieldType +import com.tencent.bkrepo.ddc.serialization.CbObject +import com.tencent.bkrepo.ddc.utils.beginUniformArray +import com.tencent.bkrepo.ddc.utils.writeStringValue + data class CreateRefResponse( val needs: Set -) +) { + fun serialize(): CbObject { + return CbObject.build { writer -> + writer.beginUniformArray(CreateRefResponse::needs.name, CbFieldType.String) + needs.forEach { writer.writeStringValue(it) } + writer.endArray() + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObject.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObject.kt index d947c21fe4..930b0dd9ed 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObject.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObject.kt @@ -154,36 +154,72 @@ class CbObject : Iterable { } writer.writeEndObject() } else if (CbFieldUtils.isArray(field.typeWithFlags)) { - writer.writeArrayFieldStart(field.name) + if (field.nameLen != 0) { + writer.writeArrayFieldStart(field.name) + } else { + writer.writeStartArray() + } val array = field.asArray() for (objectField in array) { writeField(objectField, writer) } writer.writeEndArray() } else if (CbFieldUtils.isInteger(field.typeWithFlags)) { - if (field.getType() == CbFieldType.IntegerNegative) { - writer.writeNumberField(field.name, -field.asInt64()) + writeInt(field, writer) + } else if (CbFieldUtils.isBool(field.typeWithFlags)) { + if (field.nameLen != 0) { + writer.writeBooleanField(field.name, field.asBool()) } else { - writer.writeNumberField(field.name, field.asUInt64()) + writer.writeBoolean(field.asBool()) } - } else if (CbFieldUtils.isBool(field.typeWithFlags)) { - writer.writeBooleanField(field.name, field.asBool()) } else if (CbFieldUtils.isNull(field.typeWithFlags)) { - writer.writeNull() + if (field.nameLen != 0) { + writer.writeNullField(field.name) + } else { + writer.writeNull() + } } else if (CbFieldUtils.isDateTime(field.typeWithFlags)) { throw NotImplementedException() // writer.writeStringField(field.name, field.asDateTime()) } else if (CbFieldUtils.isHash(field.typeWithFlags)) { - writer.writeStringField(field.name, field.asHash().toString()) + if (field.nameLen != 0) { + writer.writeStringField(field.name, field.asHash().toString()) + } else { + writer.writeString(field.asHash().toString()) + } } else if (CbFieldUtils.isString(field.typeWithFlags)) { - writer.writeStringField(field.name, field.asString()) + if (field.nameLen != 0) { + writer.writeStringField(field.name, field.asString()) + } else { + writer.writeString(field.asString()) + } } else if (CbFieldUtils.isObjectId(field.typeWithFlags)) { - writer.writeStringField(field.name, field.asObjectId().hex()) + if (field.nameLen != 0) { + writer.writeStringField(field.name, field.asObjectId().hex()) + } else { + writer.writeString(field.asObjectId().hex()) + } } else { throw NotImplementedException("Unhandled type ${field.getType()} when attempting to convert to json") } } + private fun writeInt(field: CbField, writer: JsonGenerator) { + if (field.nameLen != 0) { + if (field.getType() == CbFieldType.IntegerNegative) { + writer.writeNumberField(field.name, -field.asInt64()) + } else { + writer.writeNumberField(field.name, field.asUInt64()) + } + } else { + if (field.getType() == CbFieldType.IntegerNegative) { + writer.writeNumber(-field.asInt64()) + } else { + writer.writeNumber(field.asUInt64()) + } + } + } + companion object { val EMPTY = fromFieldNoCheck(CbField(ByteBuffer.wrap(byteArrayOf(CbFieldType.Object.value, 0)))) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterBase.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterBase.kt index 1e5d87c0bf..35833a2b44 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterBase.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterBase.kt @@ -313,6 +313,10 @@ abstract class CbWriterBase : ICbWriter { return buffer.array() } + fun toObject(): CbObject { + return CbObject(ByteBuffer.wrap(toByteArray())) + } + fun getSegments(): List { val segments = mutableListOf() getSegments(rootScope, segments) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt index ea3cfaa12e..1a903acf7d 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt @@ -27,23 +27,58 @@ package com.tencent.bkrepo.ddc.service +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.constant.HttpStatus.NOT_FOUND import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.message.CommonMessageCode.PARAMETER_MISSING import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.common.artifact.resolve.file.memory.ByteArrayArtifactFile +import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo +import com.tencent.bkrepo.ddc.artifact.repository.DdcLocalRepository +import com.tencent.bkrepo.ddc.component.RefDownloadListener +import com.tencent.bkrepo.ddc.config.DdcConfiguration.Companion.BEAN_NAME_REF_BATCH_EXECUTOR +import com.tencent.bkrepo.ddc.event.RefDownloadedEvent +import com.tencent.bkrepo.ddc.metrics.DdcMeterBinder +import com.tencent.bkrepo.ddc.pojo.BatchOp +import com.tencent.bkrepo.ddc.pojo.BatchOps +import com.tencent.bkrepo.ddc.pojo.BatchOpsResponse +import com.tencent.bkrepo.ddc.pojo.OpResponse +import com.tencent.bkrepo.ddc.pojo.Operation +import com.tencent.bkrepo.ddc.pojo.RefKey +import com.tencent.bkrepo.ddc.serialization.CbObject +import com.tencent.bkrepo.ddc.utils.BlakeUtils +import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import com.tencent.bkrepo.ddc.utils.DdcUtils +import com.tencent.bkrepo.ddc.utils.writeBool +import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor import org.springframework.stereotype.Service +import java.nio.ByteBuffer +import java.util.concurrent.Future @Service class ReferenceArtifactService( private val referenceService: ReferenceService, + private val refDownloadListener: RefDownloadListener, + private val ddcMeterBinder: DdcMeterBinder, + private val referenceResolver: ReferenceResolver, + private val ddcLocalRepository: DdcLocalRepository, + @Qualifier(BEAN_NAME_REF_BATCH_EXECUTOR) + private val executor: ThreadPoolTaskExecutor ) : ArtifactService() { + fun downloadRef(artifactInfo: ReferenceArtifactInfo) { repository.download(ArtifactDownloadContext()) } @@ -70,4 +105,107 @@ class ReferenceArtifactService( HttpContextHolder.getResponse().writer.println(res.toJsonString()) } } + + /** + * 批量操作 + */ + fun batch(projectId: String, repoName: String, ops: BatchOps): BatchOpsResponse { + val results = HashMap>>(ops.ops.size) + val repo = ArtifactContextHolder.getRepoDetail()!! + val userId = SecurityUtils.getUserId() + for (op in ops.ops) { + results[op.opId] = when (op.op) { + Operation.GET.name -> executor.submit> { getRef(projectId, repoName, op, userId) } + Operation.HEAD.name -> executor.submit> { headRef(projectId, repoName, op, userId) } + Operation.PUT.name -> executor.submit> { putRef(repo, op, userId) } + else -> throw UnsupportedOperationException("unsupported op: ${op.op}") + } + } + val opResponse = results.map { + val result = it.value.get() + OpResponse(it.key, result.first, result.second) + } + return BatchOpsResponse(opResponse) + } + + private fun getRef(projectId: String, repoName: String, op: BatchOp, user: String): Pair { + val refFullKey = "$projectId/$repoName/${op.bucket}/${op.key}" + return try { + ddcMeterBinder.incCacheCount(projectId, repoName) + val ref = referenceService.getReference(projectId, repoName, op.bucket, op.key) + ?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, refFullKey, status = NOT_FOUND) + val refCb = CbObject(ByteBuffer.wrap(ref.inlineBlob!!)) + if (op.resolveAttachments) { + referenceResolver.getReferencedBlobs(projectId, repoName, refCb) + } + refDownloadListener.onRefDownloaded(RefDownloadedEvent(ref, user)) + ddcMeterBinder.incCacheHitCount(projectId, repoName) + logger.info("User[${user}] get ref [$refFullKey] success") + return Pair(refCb, HttpStatus.OK.value) + } catch (e: Exception) { + DdcUtils.toError(e) + } + } + + private fun headRef(projectId: String, repoName: String, op: BatchOp, user: String): Pair { + val refFullKey = "$projectId/$repoName/${op.bucket}/${op.key}" + return try { + val ref = referenceService.getReference(projectId, repoName, op.bucket, op.key) + ?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, refFullKey, status = NOT_FOUND) + if (!ref.finalized!!) { + return Pair(CbObject.build { it.writeBool("exists", false) }, NOT_FOUND.value) + } + + val refCb = CbObject(ByteBuffer.wrap(ref.inlineBlob!!)) + if (op.resolveAttachments) { + referenceResolver.getReferencedBlobs(projectId, repoName, refCb) + } + + logger.info("User[${user}] head ref [$refFullKey] success") + return Pair(CbObject.build { it.writeBool("exists", true) }, HttpStatus.OK.value) + } catch (e: Exception) { + if (e is ErrorCodeException && e.status == NOT_FOUND) { + Pair(CbObject.build { it.writeBool("exists", false) }, NOT_FOUND.value) + } else { + DdcUtils.toError(e) + } + } + } + + private fun putRef(repo: RepositoryDetail, op: BatchOp, operator: String): Pair { + return try { + // 检查payload参数是否正确 + if (op.payload == null || op.payload == CbObject.EMPTY) { + throw ErrorCodeException(PARAMETER_MISSING, "Missing payload for operation: ${op.opId}") + } + + if (op.payloadHash.isNullOrEmpty()) { + throw ErrorCodeException(PARAMETER_MISSING, "Missing payload hash for operation: ${op.opId}") + } + + val inlineBlob = op.payload.getView().array() + val inlineBlobHash = BlakeUtils.hash(inlineBlob).hex() + if (op.payloadHash != inlineBlobHash) { + throw ErrorCodeException(ArtifactMessageCode.DIGEST_CHECK_FAILED, "blake3") + } + + // 创建ref + val artifactInfo = ReferenceArtifactInfo( + projectId = repo.projectId, + repoName = repo.name, + bucket = op.bucket, + refKey = RefKey.create(op.key), + inlineBlobHash = inlineBlobHash + ) + val artifactFile = ByteArrayArtifactFile(inlineBlob) + val res = ddcLocalRepository.uploadReference(repo, artifactInfo, artifactFile, operator) + return Pair(res.serialize(), HttpStatus.OK.value) + } catch (e: Exception) { + DdcUtils.toError(e) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(ReferenceArtifactService::class.java) + } } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt index bc83bc02ac..272eb636a9 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt @@ -64,14 +64,13 @@ class ReferenceService( private val nodeService: NodeService, private val storageManager: StorageManager, ) { - fun create(ref: Reference): Reference { + fun create(ref: Reference, userId: String): Reference { val inlineBlob = if (ref.inlineBlob!!.size > ddcProperties.inlineBlobMaxSize.toBytes()) { null } else { ref.inlineBlob } - val userId = SecurityUtils.getUserId() val now = LocalDateTime.now() val tRef = TDdcRef( createdBy = userId, diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbWriterUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbWriterUtils.kt index 1be70035ff..39659ff4f0 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbWriterUtils.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbWriterUtils.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.ddc.utils import com.tencent.bkrepo.ddc.serialization.CbFieldType +import com.tencent.bkrepo.ddc.serialization.CbObject import com.tencent.bkrepo.ddc.serialization.CbWriterBase import com.tencent.bkrepo.ddc.serialization.VarULong import com.tencent.bkrepo.ddc.utils.BlakeUtils.OUT_LEN @@ -104,7 +105,8 @@ fun CbWriterBase.writeStringValue(value: String) = writeString(null, value) fun CbWriterBase.writeString(name: String? = null, value: String?) { if (value != null) { - writeFieldWithLength(CbFieldType.String, name, value.length).put(value.toByteArray()) + val valueByteArray = value.toByteArray() + writeFieldWithLength(CbFieldType.String, name, valueByteArray.size).put(valueByteArray) } } @@ -124,3 +126,11 @@ fun CbWriterBase.writeBinary(name: String? = null, value: ByteBuffer) { fun CbWriterBase.writeBinaryArrayValue(value: ByteArray) = writeBinaryValue(ByteBuffer.wrap(value)) fun CbWriterBase.writeBinaryArray(name: String, value: ByteArray) = writeBinary(name, ByteBuffer.wrap(value)) + +fun CbWriterBase.writerObject(name: String, value: CbObject) { + val view = value.getView() + // 由于类型信息已经包含在view中，此处需要跳过类型信息 + view.position(1) + // 写入数据 + writeField(CbFieldType.Object, name, view.remaining()).put(view) +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/DdcUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/DdcUtils.kt index f6b7b2f0fc..b2adee5803 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/DdcUtils.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/DdcUtils.kt @@ -27,10 +27,17 @@ package com.tencent.bkrepo.ddc.utils +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.service.util.LocaleMessageUtils import com.tencent.bkrepo.ddc.model.TDdcBlob import com.tencent.bkrepo.ddc.model.TDdcRef +import com.tencent.bkrepo.ddc.serialization.CbObject +import org.slf4j.LoggerFactory object DdcUtils { + private val logger = LoggerFactory.getLogger(DdcUtils::class.java) + const val DIR_BLOBS = "blobs" fun TDdcRef.fullPath() = "/$bucket/$key" @@ -38,4 +45,31 @@ object DdcUtils { fun TDdcBlob.fullPath() = "/blobs/$blobId" fun buildRef(bucket: String, key: String): String = "ref/$bucket/$key" + + fun toError(e: Exception): Pair { + val statusCode = if (e is ErrorCodeException) { + e.status.value + } else { + HttpStatus.INTERNAL_SERVER_ERROR.value + } + val msg = if (e is ErrorCodeException) { + LocaleMessageUtils.getLocalizedMessage(e.messageCode, e.params) + } else { + HttpStatus.INTERNAL_SERVER_ERROR.name + } + return toError(e, statusCode, msg) + } + + fun toError(e: Exception, statusCode: Int, msg: String): Pair { + if (statusCode == HttpStatus.INTERNAL_SERVER_ERROR.value) { + logger.error("batch op failed:\n${e.stackTraceToString()}") + } else if (statusCode != HttpStatus.NOT_FOUND.value && statusCode != HttpStatus.BAD_REQUEST.value) { + logger.info("batch op failed:\n${e.stackTraceToString()}") + } + val obj = CbObject.build { + it.writeString("title", msg) + it.writeInteger("status", statusCode) + } + return Pair(obj, statusCode) + } } diff --git a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/pojo/SerializationTest.kt b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/pojo/SerializationTest.kt new file mode 100644 index 0000000000..9c69b00c16 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/pojo/SerializationTest.kt @@ -0,0 +1,168 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.ddc.serialization.CbFieldType +import com.tencent.bkrepo.ddc.serialization.CbObject +import com.tencent.bkrepo.ddc.serialization.CbWriter +import com.tencent.bkrepo.ddc.utils.BlakeUtils +import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import com.tencent.bkrepo.ddc.utils.DdcUtils +import com.tencent.bkrepo.ddc.utils.beginUniformArray +import com.tencent.bkrepo.ddc.utils.writeBinaryAttachment +import com.tencent.bkrepo.ddc.utils.writeBool +import com.tencent.bkrepo.ddc.utils.writeInteger +import com.tencent.bkrepo.ddc.utils.writeString +import com.tencent.bkrepo.ddc.utils.writeStringValue +import com.tencent.bkrepo.ddc.utils.writerObject +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.nio.ByteBuffer + +class SerializationTest { + @Test + fun testSerializeBatchOpsResponse() { + val response = CbObject.build { responseWriter -> + responseWriter.beginUniformArray("needs", CbFieldType.String) + responseWriter.writeStringValue("o1") + responseWriter.writeStringValue("o2") + responseWriter.endArray() + } + + val opResponse1 = OpResponse(1, response, 0) + val opResponse2 = OpResponse(2, response, 0) + val opsResponse = BatchOpsResponse(listOf(opResponse1, opResponse2)) + val opsResCb = opsResponse.serialize() + val json = opsResCb.toJson(jacksonObjectMapper()) + val expectedJson = "{\"results\":[" + + "{\"opId\":1,\"response\":{\"needs\":[\"o1\",\"o2\"]},\"statusCode\":0}," + + "{\"opId\":2,\"response\":{\"needs\":[\"o1\",\"o2\"]},\"statusCode\":0}]" + + "}" + assertEquals(expectedJson, json) + } + + @Test + fun testDeserializeBatchOps() { + val writer = CbWriter() + writer.beginObject() + writer.beginUniformArray(BatchOps::ops.name, CbFieldType.Object) + writer.beginObject() + writer.writeInteger(BatchOp::opId.name, 0) + writer.writeString(BatchOp::bucket.name, "bucket") + writer.writeString(BatchOp::key.name, "key") + writer.writeString(BatchOp::op.name, Operation.GET.name) + writer.writeBool(BatchOp::resolveAttachments.name, true) + writer.endObject() + + writer.beginObject() + writer.writeInteger(BatchOp::opId.name, 0) + writer.writeString(BatchOp::bucket.name, "bucket") + writer.writeString(BatchOp::key.name, "key") + writer.writeString(BatchOp::op.name, Operation.HEAD.name) + writer.endObject() + + val payload = CbObject.build { innerWriter -> innerWriter.writeString("test", "test value") } + writer.beginObject() + writer.writeInteger(BatchOp::opId.name, 0) + writer.writeString(BatchOp::bucket.name, "bucket") + writer.writeString(BatchOp::key.name, "key") + writer.writeString(BatchOp::op.name, Operation.PUT.name) + writer.writerObject(BatchOp::payload.name, payload) + writer.writeString(BatchOp::payloadHash.name, "test hash") + writer.endObject() + writer.endArray() + writer.endObject() + + val batchOps = BatchOps.deserialize(writer.toByteArray()) + println(batchOps.ops[2].payload!!.toJson(jacksonObjectMapper())) + assertEquals( + "{\"test\":\"test value\"}", + batchOps.ops[2].payload!!.toJson(jacksonObjectMapper()) + ) + } + + @Test + fun testSerializeCreateRefResponse() { + val res = CreateRefResponse(needs = setOf("a", "b", "c")) + assertEquals("{\"needs\":[\"a\",\"b\",\"c\"]}", res.serialize().toJson(jacksonObjectMapper())) + } + + @Test + fun testSerializeError() { + val e = ErrorCodeException(CommonMessageCode.PARAMETER_INVALID, "test") + val (error, code) = DdcUtils.toError( + e, e.status.value, "[250105]system.parameter.invalid" + ) + assertEquals(HttpStatus.BAD_REQUEST.value, code) + assertEquals( + "{\"title\":\"[250105]system.parameter.invalid\",\"status\":400}", + error.toJson(jacksonObjectMapper()) + ) + } + + @Test + fun testSerializeBatchOp() { + val getOp = BatchOp( + opId = 1, + bucket = "testbucket", + key = "testtkeyyytesttkeyyytesttkeyyytesttkeyyy", + op = Operation.GET.name, + resolveAttachments = true + ) + val headOp = BatchOp( + opId = 2, + bucket = "testbucket", + key = "testtkeyyytesttkeyyytesttkeyyytesttkeyyy", + op = Operation.HEAD.name, + resolveAttachments = false + ) + val payload = CbObject.build { + it.writeBinaryAttachment("test", "testttestttestttestt".toByteArray()) + } + + // 计算payload fieldData hash + val cbFieldData = payload.innerField.fieldData + val field = ByteBuffer::class.java.getDeclaredField("hb") + field.isAccessible = true + val payloadHash = BlakeUtils.hash(field.get(cbFieldData) as ByteArray).hex() + val putOp = BatchOp( + opId = 3, + bucket = "testbucket", + key = "testtkeyyytesttkeyyytesttkeyyytesttkeyyy", + op = Operation.PUT.name, + payload = payload, + payloadHash = payloadHash, + ) + val batchOps = BatchOps(listOf(getOp, headOp, putOp)) + assertEquals(407, batchOps.serialize().getSize()) + } +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/StreamRequest.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/StreamRequest.kt index cb64e9c9f9..c89d9c6980 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/StreamRequest.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/StreamRequest.kt @@ -1,12 +1,22 @@ package com.tencent.bkrepo.fs.server.request import com.tencent.bkrepo.common.api.exception.ParameterInvalidException +import org.slf4j.LoggerFactory import org.springframework.web.reactive.function.server.ServerRequest import org.springframework.web.reactive.function.server.queryParamOrNull class StreamRequest(val request: ServerRequest) : NodeRequest(request) { - val size = request.queryParamOrNull("size")?.toLong() - ?: throw ParameterInvalidException("required size parameter.") + val size = try { + request.queryParamOrNull("size")?.toLong() ?: throw ParameterInvalidException("size") + } catch (e: NumberFormatException) { + logger.info("invalid size parameter: ${request.queryParamOrNull("size")}") + throw ParameterInvalidException("size") + } + val overwrite = request.headers().header("X-BKREPO-OVERWRITE").firstOrNull()?.toBoolean() ?: false val expires = request.headers().header("X-BKREPO-EXPIRES").firstOrNull()?.toLong() ?: 0 + + companion object { + private val logger = LoggerFactory.getLogger(StreamRequest::class.java) + } } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/TemporaryAccessController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/TemporaryAccessController.kt index 661a010118..7e6f9fcbd6 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/TemporaryAccessController.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/TemporaryAccessController.kt @@ -31,7 +31,6 @@ import com.tencent.bk.audit.annotations.ActionAuditRecord import com.tencent.bk.audit.annotations.AuditAttribute import com.tencent.bk.audit.annotations.AuditEntry import com.tencent.bk.audit.annotations.AuditInstanceRecord -import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenCreateRequest import com.tencent.bkrepo.auth.pojo.token.TokenType @@ -41,13 +40,13 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable -import com.tencent.bkrepo.common.artifact.metrics.ChunkArtifactTransferMetrics -import com.tencent.bkrepo.common.artifact.router.Router -import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE -import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.metrics.ChunkArtifactTransferMetrics +import com.tencent.bkrepo.common.artifact.router.Router +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -131,10 +130,6 @@ class TemporaryAccessController( AuditAttribute( name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName" - ), - AuditAttribute( - name = ActionAuditContent.TOKEN_TEMPLATE, - value = "#token" ) ], scopeId = "#artifactInfo?.projectId", @@ -151,7 +146,6 @@ class TemporaryAccessController( artifactInfo: GenericArtifactInfo ) { val downloadUser = downloadUserId ?: userId - ActionAuditContext.current().addExtendData("downloadUser", downloadUser) val tokenInfo = temporaryAccessService.validateToken(token, artifactInfo, TokenType.DOWNLOAD) temporaryAccessService.downloadByShare(downloadUser, tokenInfo.createdBy, artifactInfo) temporaryAccessService.decrementPermits(tokenInfo) @@ -170,8 +164,7 @@ class TemporaryAccessController( ), attributes = [ AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), - AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), - AuditAttribute(name = ActionAuditContent.TOKEN_TEMPLATE, value = "#token") + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") ], scopeId = "#artifactInfo?.projectId", content = ActionAuditContent.NODE_DOWNLOAD_WITH_TOKEN_CONTENT @@ -200,9 +193,8 @@ class TemporaryAccessController( ), attributes = [ AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), - AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), - AuditAttribute(name = ActionAuditContent.TOKEN_TEMPLATE, value = "#token") - ], + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], scopeId = "#artifactInfo?.projectId", content = ActionAuditContent.NODE_UPLOAD_WITH_TOKEN_CONTENT ) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt index f9fea00d69..0c0096e8a2 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt @@ -31,12 +31,16 @@ package com.tencent.bkrepo.generic.service +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.api.ServiceTemporaryTokenClient import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenCreateRequest import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenInfo import com.tencent.bkrepo.auth.pojo.token.TokenType import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.api.constant.AUDITED_UID +import com.tencent.bkrepo.common.api.constant.AUDIT_REQUEST_URI +import com.tencent.bkrepo.common.api.constant.AUDIT_SHARE_USER_ID import com.tencent.bkrepo.common.api.constant.AUTH_HEADER_UID import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.StringPool @@ -60,8 +64,8 @@ import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext -import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -137,6 +141,11 @@ class TemporaryAccessService( ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) val context = ArtifactDownloadContext(repo = repo, userId = downloadUser) HttpContextHolder.getRequest().setAttribute(USER_KEY, downloadUser) + ActionAuditContext.current().addExtendData(AUDITED_UID, downloadUser) + ActionAuditContext.current().addExtendData( + AUDIT_REQUEST_URI, "{${HttpContextHolder.getRequestOrNull()?.requestURI}}" + ) + ActionAuditContext.current().addExtendData(AUDIT_SHARE_USER_ID, shareBy) context.shareUserId = shareBy val repository = ArtifactContextHolder.getRepository(context.repositoryDetail.category) repository.download(context) @@ -449,6 +458,11 @@ class TemporaryAccessService( } // 设置审计uid到session中 HttpContextHolder.getRequestOrNull()?.setAttribute(USER_KEY, auditedUid) + ActionAuditContext.current().addExtendData(AUDITED_UID, auditedUid) + ActionAuditContext.current().addExtendData( + AUDIT_REQUEST_URI, "{${HttpContextHolder.getRequestOrNull()?.requestURI}}" + ) + ActionAuditContext.current().addExtendData(AUDIT_SHARE_USER_ID, tokenInfo.createdBy) // 校验ip授权 val clientIp = HttpContextHolder.getClientAddress() if (tokenInfo.authorizedIpList.isNotEmpty() && clientIp !in tokenInfo.authorizedIpList) { diff --git a/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/interceptor/ProxyInterceptor.kt b/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/interceptor/ProxyInterceptor.kt index 4d9ff434e6..7fed0aa58c 100644 --- a/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/interceptor/ProxyInterceptor.kt +++ b/src/backend/git/biz-git/src/main/kotlin/com/tencent/bkrepo/git/interceptor/ProxyInterceptor.kt @@ -22,9 +22,9 @@ class ProxyInterceptor : HandlerInterceptor { override fun preHandle(request: HttpServletRequest, response: HttpServletResponse, handler: Any): Boolean { request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE) ?: return false - val repo = ArtifactContextHolder.getRepoDetail()!! + val repo = ArtifactContextHolder.getRepoDetailOrNull() // 只有PROXY类型的仓库才进行拦截 - if (repo.category != RepositoryCategory.PROXY) { + if (repo == null || repo.category != RepositoryCategory.PROXY) { return true } val projectId = repo.projectId diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt index 724a3ea1ab..8b7e4a7985 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt @@ -135,7 +135,7 @@ class DeletedNodeCleanupJob( override fun run(row: Node, collectionName: String, context: JobContext) { require(context is DeletedNodeCleanupJobContext) - // 仓库正在迁移时删除node会导致迁移任务分页查询数据重复或缺失，需要等迁移完后再执行清理 + // 仓库正在迁移时删除node会导致迁移任务分页查询数据重复或缺失，且无法确认修改哪个存储的引用数，需要等迁移完后再执行清理 if (migrateProperties.enabled && migrateRepoStorageService.migrating(row.projectId, row.repoName)) { logger.info("repo[${row.projectId}/${row.repoName}] storage was migrating, skip clean node[${row.sha256}]") return diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/listener/RefreshJobPropertiesListener.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/listener/RefreshJobPropertiesListener.kt index 50b5685e85..3338a75df1 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/listener/RefreshJobPropertiesListener.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/listener/RefreshJobPropertiesListener.kt @@ -53,7 +53,8 @@ class RefreshJobPropertiesListener(private val jobRegistrar: JobRegistrar) : Sma refreshKeys.forEach { key -> try { val jobName = key.substring(4, key.indexOf(".", 4)) - .replace(pattern) { it.value.last().uppercase() } + JOB_SUFFIX + .replace(pattern) { it.value.last().uppercase() } + .replaceFirstChar { it.lowercase() } + JOB_SUFFIX val jobBean = SpringContextUtils.getBean(BatchJob::class.java, jobName) logger.info("Job [$jobName] config updated") val job = JobUtils.parseBatchJob(jobBean) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt index e1af17781f..1ed97c2bb3 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt @@ -162,12 +162,17 @@ abstract class BaseTaskExecutor( val projectId = node.projectId val repoName = node.repoName // 文件已存在于目标存储则不处理 + val dstFileReferenceExists = fileReferenceService.count(sha256, context.task.dstStorageKey) > 0 if (storageService.exist(sha256, context.dstCredentials)) { + if (!dstFileReferenceExists) { + updateFileReference(context.task.srcStorageKey, context.task.dstStorageKey, sha256) + logger.info("correct reference[$sha256] success, task[$projectId/$repoName], state[${task.state}]") + } logger.info("file[$sha256] already exists in dst credentials[${context.task.dstStorageKey}]") return } - if (fileReferenceService.count(sha256, context.task.dstStorageKey) > 0) { + if (dstFileReferenceExists) { /* 可能由于在上传制品时使用的旧存储，而创建Node时由于会重新查一遍仓库的存储凭据而使用新存储 这种情况会导致目标存储引用大于0但是文件不再目标存储，此时仅迁移存储不修改引用数 @@ -190,6 +195,13 @@ abstract class BaseTaskExecutor( // 跨存储迁移数据 transferData(context, node) + // 更新引用 + updateFileReference(srcStorageKey, dstStorageKey, sha256) + } + + open fun close(timeout: Long, unit: TimeUnit) {} + + private fun updateFileReference(srcStorageKey: String?, dstStorageKey: String?, sha256: String) { // FileReferenceCleanupJob 会定期清理引用为0的文件数据，所以不需要删除文件数据 // 迁移前后使用的存储相同时，如果加引用失败减引用成功，可能导致文件误删，因此先增后减 // new引用计数 +1 @@ -203,8 +215,6 @@ abstract class BaseTaskExecutor( } } - open fun close(timeout: Long, unit: TimeUnit) {} - /** * 执行数据迁移 */ diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt index 1a2cb4cf93..4c2fa13b21 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt @@ -37,7 +37,6 @@ import com.tencent.bkrepo.common.mongo.util.Pages import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.job.RESTORE import com.tencent.bkrepo.job.SEPARATE -import com.tencent.bkrepo.job.SEPARATION_TASK_COLLECTION_NAME import com.tencent.bkrepo.job.separation.config.DataSeparationConfig import com.tencent.bkrepo.job.separation.dao.SeparationFailedRecordDao import com.tencent.bkrepo.job.separation.dao.SeparationTaskDao @@ -100,10 +99,10 @@ class SeparationTaskServiceImpl( projectId?.apply { criteria.and(TSeparationTask::projectId.name).isEqualTo(projectId) } repoName?.apply { criteria.and(TSeparationTask::repoName.name).isEqualTo(repoName) } val query = Query(criteria) - val dateRecords = mongoTemplate.findDistinct( - query, TSeparationTask::separationDate.name, SEPARATION_TASK_COLLECTION_NAME, LocalDateTime::class.java - ) - result.addAll(dateRecords) + val dateRecords = mongoTemplate.find(query, TSeparationTask::class.java) + dateRecords.forEach { + result.add(it.separationDate) + } return result } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/MavenRepoSpecialDataSeparatorHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/MavenRepoSpecialDataSeparatorHandler.kt index 985286b4d1..8aa293c02d 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/MavenRepoSpecialDataSeparatorHandler.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/MavenRepoSpecialDataSeparatorHandler.kt @@ -46,7 +46,7 @@ import com.tencent.bkrepo.job.separation.pojo.query.NodeDetailInfo import com.tencent.bkrepo.job.separation.service.RepoSpecialDataSeparator import com.tencent.bkrepo.job.separation.util.SeparationUtils import com.tencent.bkrepo.job.separation.util.SeparationUtils.getNodeCollectionName -import com.tencent.bkrepo.maven.util.MavenGAVCUtils.toMavenGAVC +import com.tencent.bkrepo.maven.util.MavenGAVCUtils.mavenGAVC import com.tencent.bkrepo.maven.util.MavenStringUtils.formatSeparator import com.tencent.bkrepo.maven.util.MavenUtil.extractGroupIdAndArtifactId import com.tencent.bkrepo.maven.util.MavenUtil.extractPath @@ -210,7 +210,7 @@ class MavenRepoSpecialDataSeparatorHandler( override fun getRecoveryPackageVersionData(recoveryInfo: RecoveryNodeInfo): RecoveryVersionInfo { with(recoveryInfo) { - val mavenGAVC = fullPath.toMavenGAVC() + val mavenGAVC = fullPath.mavenGAVC() val version = mavenGAVC.version val artifactId = mavenGAVC.artifactId val groupId = mavenGAVC.groupId.formatSeparator("/", ".") diff --git a/src/backend/job/boot-job-worker/shell/media-process.sh b/src/backend/job/boot-job-worker/shell/media-process.sh deleted file mode 100644 index 38c0e8ce68..0000000000 --- a/src/backend/job/boot-job-worker/shell/media-process.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -get_field() { - local field_name=$1 - field_value=$(echo $DEVOPS_SCHEDULE_JOB_PARAMETERS| jq -r ".$field_name") - if [ -z "$field_value" ]; then - echo "Error: Field '$field_name' not found in input file" - exit 1 - fi - echo "$field_value" -} - -echo "1. 获取参数" -inputUrl=$(get_field inputUrl) -callbackUrl=$(get_field callbackUrl) -inputFileName=$(get_field inputFileName) -scale=$(get_field scale) -videoCodec=$(get_field videoCodec) -outputFileName=$(get_field outputFileName) - -echo "2. 下载待转码文件 - $inputFileName" -# 使用 -s 参数静默执行，-w "%{http_code}" 输出HTTP状态码 -http_status=$(curl -s -w "%{http_code}" -o $inputFileName $inputUrl) -if [ $http_status -ne 200 ];then - echo "文件下载失败[$http_status]，Url: $inputUrl" - exit 1 -fi -ls -l - -echo "3. 开始转码 - $inputFileName > $outputFileName" -echo "ffmpeg -i $inputFileName -vf scale=$scale -c:a copy -c:v $videoCodec $outputFileName" -ffmpeg -i $inputFileName -vf scale=$scale -c:a copy -c:v $videoCodec $outputFileName -if [ $? -ne 0 ];then - echo "转码失败" - exit 1 -fi - -echo "4. 上传转码后的文件 - $outputFileName" -http_status=$(curl -s -w "%{http_code}" -X PUT -T $outputFileName "$callbackUrl") -if [ $http_status -ne 200 ];then - echo 文件上传失败[$http_status],Url: $callbackUrl - exit 1 -fi - -echo "转码完成 - $inputFileName" \ No newline at end of file diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/config/WebConfig.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/config/WebConfig.kt new file mode 100644 index 0000000000..d3d6717aa0 --- /dev/null +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/config/WebConfig.kt @@ -0,0 +1,16 @@ +package com.tencent.bkrepo.media.config + +import com.tencent.bkrepo.media.web.PluginDelegateFilter +import org.springframework.boot.web.servlet.FilterRegistrationBean +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +class WebConfig { + @Bean + fun pluginDelegateFilter(): FilterRegistrationBean { + val registrationBean = FilterRegistrationBean() + registrationBean.filter = PluginDelegateFilter() + return registrationBean + } +} diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt index 1833726fe9..74425c05f7 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt @@ -59,7 +59,7 @@ class StreamService( type = RepositoryType.MEDIA, category = RepositoryCategory.LOCAL, public = false, - display = display + display = display, ) repositoryService.createRepo(createRepoRequest) val nodeCreateRequest = NodeCreateRequest( @@ -100,17 +100,20 @@ class StreamService( userId: String, remux: Boolean = false, saveType: MediaType = MediaType.RAW, + transcodeExtraParams: String? = null, ): ClientStream { val repoId = RepositoryId(projectId, repoName) val repo = ArtifactContextHolder.getRepoDetail(repoId) val credentials = repo.storageCredentials ?: storageProperties.defaultStorageCredentials() + val transcodeConfig = getTranscodeConfig(projectId) + transcodeConfig?.let { it.extraParams = transcodeExtraParams } val fileConsumer = MediaArtifactFileConsumer( storageManager, transcodeService, repo, userId, STREAM_PATH, - getTranscodeConfig(projectId), + transcodeConfig, ) val recordingListener = if (remux) { RemuxRecordingListener(credentials.upload.location, scheduler, saveType, fileConsumer) diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt index a35226c8d6..304915854d 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt @@ -150,7 +150,7 @@ class TokenService( /** * 检查token并返回token信息 */ - private fun checkToken(token: String): TemporaryTokenInfo { + fun checkToken(token: String): TemporaryTokenInfo { if (token.isBlank()) { throw ErrorCodeException(ArtifactMessageCode.TEMPORARY_TOKEN_INVALID, token) } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TranscodeService.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TranscodeService.kt index 15734405a3..8e2719f016 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TranscodeService.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TranscodeService.kt @@ -76,7 +76,7 @@ class TranscodeService( projectId = projectId, repoName = repoName, fullPath = newArtifactInfo.getArtifactFullPath(), - metadata = originMetadata + metadata = originMetadata, ) metadataService.saveMetadata(copyRequest) val removeContext = ArtifactRemoveContext(repo, originArtifactInfo) @@ -106,6 +106,7 @@ class TranscodeService( audioCodec = audioCodec, inputFileName = artifactInfo.getResponseName(), outputFileName = outputArtifactInfo.getResponseName(), + extraParams = transcodeConfig.extraParams.orEmpty(), ) } } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/MediaArtifactFileConsumer.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/MediaArtifactFileConsumer.kt index 6d7ed3f1e7..b8b43b9505 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/MediaArtifactFileConsumer.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/MediaArtifactFileConsumer.kt @@ -8,7 +8,6 @@ import com.tencent.bkrepo.media.service.TranscodeService import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail -import org.slf4j.LoggerFactory import java.io.File /** @@ -39,7 +38,6 @@ class MediaArtifactFileConsumer( storageManager.storeArtifactFile(nodeCreateRequest, file, repo.storageCredentials) if (transcodeConfig != null) { transcodeService.transcode(artifactInfo, transcodeConfig, userId) - logger.info("Add transcode task for artifact[$artifactInfo]") } } @@ -68,7 +66,6 @@ class MediaArtifactFileConsumer( } companion object { - private val logger = LoggerFactory.getLogger(MediaArtifactFileConsumer::class.java) private const val METADATA_KEY_MEDIA_START_TIME = "media.startTime" private const val METADATA_KEY_MEDIA_STOP_TIME = "media.stopTime" } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeConfig.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeConfig.kt index bc785920ce..774f962809 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeConfig.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeConfig.kt @@ -4,5 +4,6 @@ data class TranscodeConfig( var scale: String = "", // 分辨率,比如1280x720 var videoCodec: String = "", // 视频编码 var audioCodec: String = "", // 音频编码 - var jobId: String = "", // 转码任务id + var jobId: String = "", // 转码任务id， + var extraParams: String? = null, ) diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeParam.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeParam.kt index fdf6c9ce85..ddd85e9554 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeParam.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeParam.kt @@ -7,5 +7,6 @@ data class TranscodeParam( val videoCodec: String? = null, // 视频编码 val audioCodec: String? = null, // 音频编码 var inputFileName: String, // 源文件名 - var outputFileName: String, // 输出文件名 + var outputFileName: String, // 输出文件名, + var extraParams: String, // 额外参数 ) diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/web/PluginDelegateFilter.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/web/PluginDelegateFilter.kt new file mode 100644 index 0000000000..2b67713438 --- /dev/null +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/web/PluginDelegateFilter.kt @@ -0,0 +1,20 @@ +package com.tencent.bkrepo.media.web + +import javax.servlet.Filter +import javax.servlet.FilterChain +import javax.servlet.ServletRequest +import javax.servlet.ServletResponse + +class PluginDelegateFilter : Filter { + override fun doFilter(request: ServletRequest, response: ServletResponse, chain: FilterChain) { + if (delegate != null) { + delegate!!.doFilter(request, response, chain) + } else { + chain.doFilter(request, response) + } + } + + companion object { + var delegate: Filter? = null + } +} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataController.kt index 7fccda0d93..2a792cd9f4 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataController.kt @@ -43,9 +43,10 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo.Companion.DEFAULT_MAPPING_URI import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_EDIT_ACTION import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.artifact.audit.NODE_VIEW_ACTION -import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder @@ -53,7 +54,6 @@ import com.tencent.bkrepo.repository.pojo.metadata.MetadataDeleteRequest import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest import com.tencent.bkrepo.repository.pojo.metadata.UserMetadataDeleteRequest import com.tencent.bkrepo.repository.pojo.metadata.UserMetadataSaveRequest -import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import org.springframework.web.bind.annotation.DeleteMapping @@ -104,10 +104,10 @@ class UserMetadataController( } @AuditEntry( - actionId = REPO_EDIT_ACTION + actionId = NODE_EDIT_ACTION ) @ActionAuditRecord( - actionId = REPO_EDIT_ACTION, + actionId = NODE_EDIT_ACTION, instance = AuditInstanceRecord( resourceType = NODE_RESOURCE, instanceIds = "#artifactInfo?.getArtifactFullPath()", @@ -144,10 +144,10 @@ class UserMetadataController( } @AuditEntry( - actionId = REPO_EDIT_ACTION + actionId = NODE_EDIT_ACTION ) @ActionAuditRecord( - actionId = REPO_EDIT_ACTION, + actionId = NODE_EDIT_ACTION, instance = AuditInstanceRecord( resourceType = NODE_RESOURCE, instanceIds = "#artifactInfo?.getArtifactFullPath()", @@ -182,10 +182,10 @@ class UserMetadataController( } @AuditEntry( - actionId = REPO_EDIT_ACTION + actionId = NODE_EDIT_ACTION ) @ActionAuditRecord( - actionId = REPO_EDIT_ACTION, + actionId = NODE_EDIT_ACTION, instance = AuditInstanceRecord( resourceType = NODE_RESOURCE, instanceIds = "#artifactInfo?.getArtifactFullPath()", diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserShareController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserShareController.kt index 6c7a04552a..c0152d8bca 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserShareController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserShareController.kt @@ -165,10 +165,6 @@ class UserShareController( AuditAttribute( name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName" - ), - AuditAttribute( - name = ActionAuditContent.TOKEN_TEMPLATE, - value = "#token" ) ], scopeId = "#artifactInfo?.projectId", @@ -183,7 +179,6 @@ class UserShareController( @ArtifactPathVariable artifactInfo: ArtifactInfo ) { val downloadUser = downloadUserId ?: userId - ActionAuditContext.current().addExtendData("downloadUser", downloadUser) shareService.download(downloadUser, token, artifactInfo) } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ShareServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ShareServiceImpl.kt index 55342317d6..acf1af7350 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ShareServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/ShareServiceImpl.kt @@ -31,10 +31,14 @@ package com.tencent.bkrepo.repository.service.file.impl +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.api.ServiceTemporaryTokenClient import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenCreateRequest import com.tencent.bkrepo.auth.pojo.token.TokenType import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.api.constant.AUDITED_UID +import com.tencent.bkrepo.common.api.constant.AUDIT_REQUEST_URI +import com.tencent.bkrepo.common.api.constant.AUDIT_SHARE_USER_ID import com.tencent.bkrepo.common.api.constant.USER_KEY import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.artifact.api.ArtifactInfo @@ -42,6 +46,8 @@ import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.exception.PermissionException import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -49,8 +55,6 @@ import com.tencent.bkrepo.repository.model.TShareRecord import com.tencent.bkrepo.repository.pojo.share.ShareRecordCreateRequest import com.tencent.bkrepo.repository.pojo.share.ShareRecordInfo import com.tencent.bkrepo.repository.service.file.ShareService -import com.tencent.bkrepo.common.metadata.service.node.NodeService -import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.slf4j.LoggerFactory import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.MongoTemplate @@ -144,6 +148,11 @@ class ShareServiceImpl( ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) val context = ArtifactDownloadContext(repo = repo, userId = userId) HttpContextHolder.getRequest().setAttribute(USER_KEY, downloadUser) + ActionAuditContext.current().addExtendData(AUDITED_UID, downloadUser) + ActionAuditContext.current().addExtendData( + AUDIT_REQUEST_URI, "{${HttpContextHolder.getRequestOrNull()?.requestURI}}" + ) + ActionAuditContext.current().addExtendData(AUDIT_SHARE_USER_ID, shareRecord.createdBy) context.shareUserId = shareRecord.createdBy val repository = ArtifactContextHolder.getRepository(context.repositoryDetail.category) repository.download(context) diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt index b0a5a47d21..2ec4352d0f 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt @@ -40,7 +40,8 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService -import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test @@ -61,68 +62,90 @@ class NodeQueryWithoutShardingKeyTest @Autowired constructor( private val nodeService: NodeService ) : ServiceBaseTest() { + // 创建测试数据sha256为sha256-0,sha256-1...sha256-8,sha256-9 + // sha256-0和sha256-1有18条数据，其余sha256各有15条数据 + val generateSha256Func = { i: Int -> "sha256-${i % 10}" } + @BeforeAll fun beforeAll() { initMock() + generateTestData(52, generateSha256Func) } @Test @DisplayName("测试按SHA256分页查询") fun testListNodePageBySha256() { - // 创建测试数据sha256为sha256-0,sha256-1...sha256-8,sha256-9 - // sha256-0和sha256-1有18条数据，其余sha256各有15条数据 - val generateSha256Func = { i: Int -> "sha256-${i % 10}" } - generateTestData(52, generateSha256Func) val option = NodeListOption(1, 5, includeMetadata = true, sort = true) // 测试获取不存在的Node列表 nodeService.listNodePageBySha256("notExistsSha256", option).apply { - Assertions.assertEquals(0L, totalRecords) - Assertions.assertEquals(0L, totalPages) - Assertions.assertTrue(records.isEmpty()) + assertEquals(0L, totalRecords) + assertEquals(0L, totalPages) + assertTrue(records.isEmpty()) } // 测试数据量小于pageSize的情况 nodeService.listNodePageBySha256(generateSha256Func(1), option.copy(pageSize = 20)).apply { - Assertions.assertEquals(18, totalRecords) - Assertions.assertEquals(1, totalPages) - Assertions.assertEquals(18, records.size) + assertEquals(18, totalRecords) + assertEquals(1, totalPages) + assertEquals(18, records.size) } // 测试获取数据量等于pageSize的页 nodeService.listNodePageBySha256(generateSha256Func(1), option.copy(pageSize = 4)).apply { - Assertions.assertEquals(18, totalRecords) - Assertions.assertEquals(5, totalPages) - Assertions.assertEquals(4, records.size) + assertEquals(18, totalRecords) + assertEquals(5, totalPages) + assertEquals(4, records.size) } // 测试获数据量小于pageSize的页 nodeService.listNodePageBySha256(generateSha256Func(1), option.copy(pageSize = 4, pageNumber = 5)).apply { - Assertions.assertEquals(18, totalRecords) - Assertions.assertEquals(5, totalPages) - Assertions.assertEquals(2, records.size) + assertEquals(18, totalRecords) + assertEquals(5, totalPages) + assertEquals(2, records.size) } // 测试获取不存在的页 nodeService.listNodePageBySha256(generateSha256Func(1), option.copy(pageSize = 4, pageNumber = 6)).apply { - Assertions.assertEquals(18, totalRecords) - Assertions.assertEquals(5, totalPages) - Assertions.assertEquals(0, records.size) + assertEquals(18, totalRecords) + assertEquals(5, totalPages) + assertEquals(0, records.size) } // 测试分页数据在两个分表的情况,[0,1,2,3,4, 5][6,7,8,9, 10,11][12,13,14,15,16,17] nodeService.listNodePageBySha256(generateSha256Func(1), option.copy(pageSize = 5, pageNumber = 2)).apply { - Assertions.assertEquals(18, totalRecords) - Assertions.assertEquals(4, totalPages) - Assertions.assertEquals(5, records.size) - Assertions.assertEquals(PROJECT_SHARDING_207, records[0].projectId) - Assertions.assertEquals(PROJECT_SHARDING_208, records[1].projectId) - Assertions.assertEquals(PROJECT_SHARDING_208, records[2].projectId) - Assertions.assertEquals(PROJECT_SHARDING_208, records[3].projectId) - Assertions.assertEquals(PROJECT_SHARDING_208, records[4].projectId) + assertEquals(18, totalRecords) + assertEquals(4, totalPages) + assertEquals(5, records.size) + assertEquals(PROJECT_SHARDING_207, records[0].projectId) + assertEquals(PROJECT_SHARDING_208, records[1].projectId) + assertEquals(PROJECT_SHARDING_208, records[2].projectId) + assertEquals(PROJECT_SHARDING_208, records[3].projectId) + assertEquals(PROJECT_SHARDING_208, records[4].projectId) } } + @Test + @DisplayName("测试按SHA256查询") + fun testListNodeBySha256() { + val sha256 = generateSha256Func(1) + + // 测试获取不存在的Node列表 + nodeService.listNodeBySha256("notExistsSha256").apply { assertEquals(0, size) } + + // 测试单表数据量小于limit的情况 + nodeService.listNodeBySha256(sha256, 20, tillLimit = false).apply { assertEquals(6, size) } + nodeService.listNodeBySha256(sha256, 100, tillLimit = true).apply { assertEquals(18, size) } + + // 测试单表数据量等于limit的页 + nodeService.listNodeBySha256(sha256, 6, tillLimit = true).apply { assertEquals(6, size) } + nodeService.listNodeBySha256(sha256, 6, tillLimit = false).apply { assertEquals(6, size) } + + // 测试获数据量小于pageSize的页 + nodeService.listNodeBySha256(sha256, 3, tillLimit = true).apply { assertEquals(3, size) } + nodeService.listNodeBySha256(sha256, 3, tillLimit = false).apply { assertEquals(3, size) } + } + private fun generateTestData(size: Int, generateSha256Func: (Int) -> String) { for (projectId in arrayOf(PROJECT_SHARDING_207, PROJECT_SHARDING_208, PROJECT_SHARDING_209)) { createProject(projectService, projectId) diff --git a/src/backend/settings.gradle.kts b/src/backend/settings.gradle.kts index 74159e543f..1d2e7ef582 100644 --- a/src/backend/settings.gradle.kts +++ b/src/backend/settings.gradle.kts @@ -94,3 +94,4 @@ includeAll(":media") includeAll(":common:common-metadata") includeAll(":common:common-service") includeAll(":preview") +includeAll(":websocket") \ No newline at end of file diff --git a/src/backend/websocket/biz-websocket/build.gradle.kts b/src/backend/websocket/biz-websocket/build.gradle.kts new file mode 100644 index 0000000000..8e10e0a113 --- /dev/null +++ b/src/backend/websocket/biz-websocket/build.gradle.kts @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED " AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +dependencies { + api(project(":common:common-stream")) + api(project(":common:common-service")) + api(project(":common:common-artifact:artifact-service")) + implementation("org.springframework.boot:spring-boot-starter-websocket") + implementation("javax.websocket:javax.websocket-api") + implementation("io.undertow:undertow-servlet") + implementation("io.undertow:undertow-websockets-jsr") +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebSocketConfigurer.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebSocketConfigurer.kt new file mode 100644 index 0000000000..892cb634f9 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebSocketConfigurer.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.config + +import com.tencent.bkrepo.common.artifact.config.ArtifactConfigurerSupport +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository +import com.tencent.bkrepo.common.artifact.repository.remote.RemoteRepository +import com.tencent.bkrepo.common.artifact.repository.virtual.VirtualRepository +import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurityCustomizer +import org.springframework.context.annotation.Configuration + +@Configuration +class WebSocketConfigurer : ArtifactConfigurerSupport() { + + override fun getRepositoryType() = RepositoryType.NONE + override fun getLocalRepository(): LocalRepository = object : LocalRepository() {} + override fun getRemoteRepository(): RemoteRepository = object : RemoteRepository() {} + override fun getVirtualRepository(): VirtualRepository = object : VirtualRepository() {} + + override fun getAuthSecurityCustomizer() = + HttpAuthSecurityCustomizer { httpAuthSecurity -> httpAuthSecurity.withPrefix("/websocket") } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebSocketProperties.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebSocketProperties.kt new file mode 100644 index 0000000000..79fe623d87 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebSocketProperties.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.config + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties("websocket") +data class WebSocketProperties( + var cacheLimit: Int = 3600, + var minThread: Int = 8, + var transfer: Boolean = false +) diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebsocketConfiguration.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebsocketConfiguration.kt new file mode 100644 index 0000000000..16717a366c --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WebsocketConfiguration.kt @@ -0,0 +1,113 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.config + +import com.tencent.bkrepo.common.security.http.jwt.JwtAuthProperties +import com.tencent.bkrepo.common.security.manager.AuthenticationManager +import com.tencent.bkrepo.websocket.constant.APP_ENDPOINT +import com.tencent.bkrepo.websocket.constant.DESKTOP_ENDPOINT +import com.tencent.bkrepo.websocket.constant.USER_ENDPOINT +import com.tencent.bkrepo.websocket.dispatch.push.TransferPush +import com.tencent.bkrepo.websocket.handler.SessionWebSocketHandlerDecoratorFactory +import com.tencent.bkrepo.websocket.listener.TransferPushListener +import com.tencent.bkrepo.websocket.service.WebsocketService +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.messaging.Message +import org.springframework.messaging.simp.config.ChannelRegistration +import org.springframework.messaging.simp.config.MessageBrokerRegistry +import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker +import org.springframework.web.socket.config.annotation.StompEndpointRegistry +import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer +import org.springframework.web.socket.config.annotation.WebSocketTransportRegistration +import java.util.function.Consumer + +@Configuration +@EnableWebSocketMessageBroker +@EnableConfigurationProperties(WebSocketProperties::class) +class WebsocketConfiguration( + private val webSocketProperties: WebSocketProperties, + private val websocketService: WebsocketService, + private val jwtAuthProperties: JwtAuthProperties, + private val authenticationManager: AuthenticationManager, +) : WebSocketMessageBrokerConfigurer { + + override fun configureMessageBroker(config: MessageBrokerRegistry) { + config.setCacheLimit(webSocketProperties.cacheLimit) + config.enableSimpleBroker("/topic") + config.setApplicationDestinationPrefixes("/app") + } + + override fun registerStompEndpoints(registry: StompEndpointRegistry) { + registry.addEndpoint(USER_ENDPOINT, APP_ENDPOINT, DESKTOP_ENDPOINT) + .setAllowedOriginPatterns("*") + registry.addEndpoint(USER_ENDPOINT, APP_ENDPOINT, DESKTOP_ENDPOINT) + .setAllowedOriginPatterns("*") + .withSockJS() + } + + @Override + override fun configureClientInboundChannel(registration: ChannelRegistration) { + var defaultCorePoolSize = webSocketProperties.minThread + if (defaultCorePoolSize < Runtime.getRuntime().availableProcessors() * 2) { + defaultCorePoolSize = Runtime.getRuntime().availableProcessors() * 2 + } + registration.taskExecutor().corePoolSize(defaultCorePoolSize) + .maxPoolSize(defaultCorePoolSize * 2) + .keepAliveSeconds(60) + } + + @Override + override fun configureClientOutboundChannel(registration: ChannelRegistration) { + var defaultCorePoolSize = webSocketProperties.minThread + if (defaultCorePoolSize < Runtime.getRuntime().availableProcessors() * 2) { + defaultCorePoolSize = Runtime.getRuntime().availableProcessors() * 2 + } + registration.taskExecutor().corePoolSize(defaultCorePoolSize).maxPoolSize(defaultCorePoolSize * 2) + } + + override fun configureWebSocketTransport(registration: WebSocketTransportRegistration) { + registration.addDecoratorFactory(wsHandlerDecoratorFactory()) + super.configureWebSocketTransport(registration) + } + + @Bean + fun wsHandlerDecoratorFactory(): SessionWebSocketHandlerDecoratorFactory { + return SessionWebSocketHandlerDecoratorFactory( + websocketService = websocketService, + authenticationManager = authenticationManager, + jwtAuthProperties = jwtAuthProperties + ) + } + + @Bean + fun websocketTransferConsumer(transferPushListener: TransferPushListener): Consumer> { + return Consumer { transferPushListener.accept(it) } + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WsThreadPoolTaskExeccutorConfiguration.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WsThreadPoolTaskExeccutorConfiguration.kt new file mode 100644 index 0000000000..9ac531f7d6 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/config/WsThreadPoolTaskExeccutorConfiguration.kt @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.config + +import org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration +import org.springframework.boot.task.TaskExecutorBuilder +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Lazy +import org.springframework.context.annotation.Primary +import org.springframework.scheduling.annotation.AsyncAnnotationBeanPostProcessor +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor + +/** + * Websocket会注册自定义的ThreadPoolTaskExecutor + * + * [org.springframework.messaging.simp.config.AbstractMessageBrokerConfiguration.clientInboundChannelExecutor] + * + * [org.springframework.messaging.simp.config.AbstractMessageBrokerConfiguration.clientOutboundChannelExecutor] + * + * [org.springframework.messaging.simp.config.AbstractMessageBrokerConfiguration.brokerChannelExecutor] + * + * 导致默认的ThreadPoolTaskExecutor不会实例化 + * + * [org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration.applicationTaskExecutor] + * + */ +@Configuration +class WsThreadPoolTaskExeccutorConfiguration { + + @Lazy + @Bean(name = [ + TaskExecutionAutoConfiguration.APPLICATION_TASK_EXECUTOR_BEAN_NAME, + AsyncAnnotationBeanPostProcessor.DEFAULT_TASK_EXECUTOR_BEAN_NAME + ]) + @Primary + fun applicationTaskExecutor(builder: TaskExecutorBuilder): ThreadPoolTaskExecutor { + return builder.build() + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/constant/WebsocketKeys.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/constant/WebsocketKeys.kt new file mode 100644 index 0000000000..bf0769d39c --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/constant/WebsocketKeys.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.constant + +const val USER_ENDPOINT = "/ws/user" +const val APP_ENDPOINT = "/ws/app" +const val DESKTOP_ENDPOINT = "/ws/desktop" + +const val SESSION_ID = "sessionId" diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/controller/ClipboardController.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/controller/ClipboardController.kt new file mode 100644 index 0000000000..1f929a99e5 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/controller/ClipboardController.kt @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.controller + +import com.tencent.bkrepo.websocket.pojo.fs.CopyPDU +import com.tencent.bkrepo.websocket.pojo.fs.PastePDU +import com.tencent.bkrepo.websocket.service.ClipboardService +import org.springframework.messaging.handler.annotation.MessageMapping +import org.springframework.stereotype.Controller + +@Controller +@MessageMapping("/clipboard") +class ClipboardController( + private val clipboardService: ClipboardService +) { + + @MessageMapping("/copy") + fun copy(copyPDU: CopyPDU) { + clipboardService.copy(copyPDU) + } + + @MessageMapping("/paste") + fun paste(pastePDU: PastePDU) { + clipboardService.paste(pastePDU) + } +} + diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/Dispatcher.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/Dispatcher.kt new file mode 100644 index 0000000000..81c5cb5bd9 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/Dispatcher.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.dispatch + +/** + * 下发接口 + */ +interface Dispatcher { + + fun dispatch(data: T) +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/TransferDispatch.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/TransferDispatch.kt new file mode 100644 index 0000000000..4986de5d81 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/TransferDispatch.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.dispatch + +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier +import com.tencent.bkrepo.websocket.config.WebSocketProperties +import com.tencent.bkrepo.websocket.dispatch.push.TransferPush +import org.springframework.messaging.simp.SimpMessagingTemplate +import org.springframework.stereotype.Component + +@Component +class TransferDispatch( + private val messageSupplier: MessageSupplier, + private val simpMessagingTemplate: SimpMessagingTemplate, + private val webSocketProperties: WebSocketProperties +) : Dispatcher { + override fun dispatch(data: TransferPush) { + if (webSocketProperties.transfer) { + messageSupplier.delegateToSupplier(data, topic = TOPIC) + } else { + simpMessagingTemplate.convertAndSend(data.topic, data.data.toJsonString()) + } + } + + companion object { + private const val TOPIC = "websocket-transfer-out-0" + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/CopyPDUTransferPush.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/CopyPDUTransferPush.kt new file mode 100644 index 0000000000..7ccd8c870a --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/CopyPDUTransferPush.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.dispatch.push + +import com.tencent.bkrepo.websocket.pojo.fs.CopyPDU + +class CopyPDUTransferPush( + copyPDU: CopyPDU, +) : TransferPush( + topic = "/topic/clipboard/copy/${copyPDU.workspaceName}", + data = copyPDU +) diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/PastePDUTransferPush.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/PastePDUTransferPush.kt new file mode 100644 index 0000000000..f6da31f94b --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/PastePDUTransferPush.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.dispatch.push + +import com.tencent.bkrepo.websocket.pojo.fs.PastePDU + +class PastePDUTransferPush( + pastePDU: PastePDU, +) : TransferPush( + topic = "/topic/clipboard/paste/${pastePDU.workspaceName}", + data = pastePDU +) diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/TransferPush.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/TransferPush.kt new file mode 100644 index 0000000000..450654566f --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/dispatch/push/TransferPush.kt @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.dispatch.push + +open class TransferPush( + open val topic: String, + open val data: Any +) diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/exception/WebsocketExceptionHandler.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/exception/WebsocketExceptionHandler.kt new file mode 100644 index 0000000000..60ae6dbb17 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/exception/WebsocketExceptionHandler.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.exception + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.service.exception.AbstractExceptionHandler +import org.springframework.messaging.handler.annotation.MessageExceptionHandler +import org.springframework.web.bind.annotation.RestControllerAdvice + +@RestControllerAdvice +class WebsocketExceptionHandler : AbstractExceptionHandler() { + + @MessageExceptionHandler(ErrorCodeException::class) + fun handleException(exception: ErrorCodeException): Response { + return response(exception) + } + + @MessageExceptionHandler(Exception::class) + fun handleException(exception: Exception): Response { + return response(exception) + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/handler/SessionHandler.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/handler/SessionHandler.kt new file mode 100644 index 0000000000..e89fc9652c --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/handler/SessionHandler.kt @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.handler + +import com.tencent.bkrepo.common.api.constant.AUTH_HEADER_UID +import com.tencent.bkrepo.common.api.constant.CharPool.COLON +import com.tencent.bkrepo.common.api.constant.HttpHeaders +import com.tencent.bkrepo.common.api.constant.PLATFORM_AUTH_PREFIX +import com.tencent.bkrepo.common.api.constant.PLATFORM_KEY +import com.tencent.bkrepo.common.api.constant.USER_KEY +import com.tencent.bkrepo.common.artifact.stream.closeQuietly +import com.tencent.bkrepo.common.security.exception.AuthenticationException +import com.tencent.bkrepo.common.security.http.jwt.JwtAuthProperties +import com.tencent.bkrepo.common.security.manager.AuthenticationManager +import com.tencent.bkrepo.common.security.util.JwtUtils +import com.tencent.bkrepo.websocket.constant.APP_ENDPOINT +import com.tencent.bkrepo.websocket.constant.DESKTOP_ENDPOINT +import com.tencent.bkrepo.websocket.constant.SESSION_ID +import com.tencent.bkrepo.websocket.constant.USER_ENDPOINT +import com.tencent.bkrepo.websocket.service.WebsocketService +import com.tencent.bkrepo.websocket.util.HostUtils +import io.jsonwebtoken.ExpiredJwtException +import io.jsonwebtoken.MalformedJwtException +import io.jsonwebtoken.UnsupportedJwtException +import io.jsonwebtoken.security.SignatureException +import org.slf4j.LoggerFactory +import org.springframework.web.socket.CloseStatus +import org.springframework.web.socket.WebSocketHandler +import org.springframework.web.socket.WebSocketSession +import org.springframework.web.socket.handler.WebSocketHandlerDecorator +import java.util.Base64 + +class SessionHandler( + delegate: WebSocketHandler, + private val websocketService: WebsocketService, + private val authenticationManager: AuthenticationManager, + jwtProperties: JwtAuthProperties +) : WebSocketHandlerDecorator(delegate) { + + private val signingKey = JwtUtils.createSigningKey(jwtProperties.secretKey) + + // 链接关闭记录去除session + override fun afterConnectionClosed(session: WebSocketSession, closeStatus: CloseStatus) { + val uri = session.uri + if (closeStatus.code != CloseStatus.NORMAL.code && closeStatus.code != CloseStatus.PROTOCOL_ERROR.code) { + logger.warn("websocket close abnormal, [$closeStatus] [${session.uri}] [${session.remoteAddress}]") + } + val sessionId = HostUtils.getRealSession(session.uri?.query) + if (sessionId.isNullOrEmpty()) { + logger.warn("connection closed can not find sessionId, $uri| ${session.remoteAddress}") + super.afterConnectionClosed(session, closeStatus) + } + websocketService.removeCacheSession(sessionId!!) + + super.afterConnectionClosed(session, closeStatus) + } + + override fun afterConnectionEstablished(session: WebSocketSession) { + val uri = session.uri + val remoteAddr = session.remoteAddress + val sessionId = HostUtils.getRealSession(uri?.query) + try { + authorization(session) + } catch (e: Exception) { + val authException = e is AuthenticationException || e is ExpiredJwtException || + e is UnsupportedJwtException || e is MalformedJwtException || + e is SignatureException || e is IllegalArgumentException + if (authException) { + logger.info("auth failed: |$sessionId| $uri | $remoteAddr | ${e.message}") + session.closeQuietly() + } else { + throw e + } + } + } + + private fun authorization(session: WebSocketSession) { + val uri = session.uri + val remoteId = session.remoteAddress + val sessionId = HostUtils.getRealSession(uri?.query) + when { + uri == null -> throw AuthenticationException("uri is null") + uri.path.startsWith(USER_ENDPOINT) || uri.path.startsWith(DESKTOP_ENDPOINT) -> { + val platformToken = session.handshakeHeaders[HttpHeaders.AUTHORIZATION]?.firstOrNull()?.toString() + ?.removePrefix(PLATFORM_AUTH_PREFIX) ?: throw AuthenticationException("platform credential is null") + val (accessKey, secretKey) = String(Base64.getDecoder().decode(platformToken)).split(COLON) + val appId = authenticationManager.checkPlatformAccount(accessKey, secretKey) + session.attributes[PLATFORM_KEY] = appId + session.attributes[USER_KEY] = session.handshakeHeaders[AUTH_HEADER_UID] + } + uri.path.startsWith(APP_ENDPOINT) -> { + val token = session.handshakeHeaders[HttpHeaders.AUTHORIZATION]?.firstOrNull().orEmpty() + val claims = JwtUtils.validateToken(signingKey, token).body + session.attributes[USER_KEY] = claims.subject + } + else -> throw AuthenticationException("invalid uri") + } + websocketService.addCacheSession(sessionId!!) + session.attributes[SESSION_ID] = sessionId + logger.info("connection success: |$sessionId| $uri | $remoteId | ${session.attributes[USER_KEY]} ") + super.afterConnectionEstablished(session) + } + + companion object { + private val logger = LoggerFactory.getLogger(SessionHandler::class.java) + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/handler/SessionWebSocketHandlerDecoratorFactory.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/handler/SessionWebSocketHandlerDecoratorFactory.kt new file mode 100644 index 0000000000..6b1a45a695 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/handler/SessionWebSocketHandlerDecoratorFactory.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.handler + +import com.tencent.bkrepo.common.security.http.jwt.JwtAuthProperties +import com.tencent.bkrepo.common.security.manager.AuthenticationManager +import com.tencent.bkrepo.websocket.service.WebsocketService +import org.springframework.web.socket.WebSocketHandler +import org.springframework.web.socket.handler.WebSocketHandlerDecoratorFactory + +class SessionWebSocketHandlerDecoratorFactory ( + private val websocketService: WebsocketService, + private val authenticationManager: AuthenticationManager, + private val jwtAuthProperties: JwtAuthProperties, +) : WebSocketHandlerDecoratorFactory { + + override fun decorate(handler: WebSocketHandler): WebSocketHandler { + return SessionHandler(handler, websocketService, authenticationManager, jwtAuthProperties) + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/listener/TransferPushListener.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/listener/TransferPushListener.kt new file mode 100644 index 0000000000..1a21c7f3d1 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/listener/TransferPushListener.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.listener + +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.websocket.dispatch.push.TransferPush +import org.slf4j.LoggerFactory +import org.springframework.messaging.Message +import org.springframework.messaging.simp.SimpMessagingTemplate +import org.springframework.stereotype.Component + +@Component +class TransferPushListener( + private val simpMessagingTemplate: SimpMessagingTemplate +) { + + fun accept(message: Message) { + logger.debug(message.payload.toString()) + val transferPush = message.payload + simpMessagingTemplate.convertAndSend(transferPush.topic, transferPush.data.toJsonString()) + } + + companion object { + private val logger = LoggerFactory.getLogger(TransferPushListener::class.java) + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/pojo/fs/CopyPDU.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/pojo/fs/CopyPDU.kt new file mode 100644 index 0000000000..01da102c98 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/pojo/fs/CopyPDU.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.pojo.fs + +/** + * 复制协议数据单元 + */ +data class CopyPDU( + val projectId: String, + val userId: String, + val workspaceName: String, + val files: Map, + val timestamp: Long, + val dstPath: String? = null +) diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/pojo/fs/PastePDU.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/pojo/fs/PastePDU.kt new file mode 100644 index 0000000000..f295772852 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/pojo/fs/PastePDU.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.pojo.fs + +/** + * 粘贴协议数据单元 + */ +data class PastePDU( + val workspaceName: String, + val timestamp: Long, +) diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/service/ClipboardService.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/service/ClipboardService.kt new file mode 100644 index 0000000000..641c2b68fb --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/service/ClipboardService.kt @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.service + +import com.tencent.bkrepo.websocket.dispatch.TransferDispatch +import com.tencent.bkrepo.websocket.dispatch.push.CopyPDUTransferPush +import com.tencent.bkrepo.websocket.dispatch.push.PastePDUTransferPush +import com.tencent.bkrepo.websocket.pojo.fs.CopyPDU +import com.tencent.bkrepo.websocket.pojo.fs.PastePDU +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service + +@Service +class ClipboardService( + private val transferDispatch: TransferDispatch +) { + + fun copy(copyPDU: CopyPDU) { + logger.info("CopyPDU: $copyPDU") + val copyPDUTransferPush = CopyPDUTransferPush(copyPDU) + transferDispatch.dispatch(copyPDUTransferPush) + } + + fun paste(pastePDU: PastePDU) { + logger.info("PastePDU: $pastePDU") + val pastePDUTransferPush = PastePDUTransferPush(pastePDU) + transferDispatch.dispatch(pastePDUTransferPush) + } + + companion object { + private val logger = LoggerFactory.getLogger(ClipboardService::class.java) + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/service/WebsocketService.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/service/WebsocketService.kt new file mode 100644 index 0000000000..559f495445 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/service/WebsocketService.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.service + +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import java.util.Collections + +@Service +class WebsocketService { + private val cacheSessionList = Collections.synchronizedList(mutableListOf()) + + fun addCacheSession(sessionId: String) { + if (cacheSessionList.contains(sessionId)) { + logger.warn("this session[$sessionId] already in cacheSession") + return + } + cacheSessionList.add(sessionId) + } + + // 清楚实例内部缓存的session + fun removeCacheSession(sessionId: String) { + cacheSessionList.remove(sessionId) + } + + // 判断获取到的session是否由该实例持有 + fun isCacheSession(sessionId: String): Boolean { + if (cacheSessionList.contains(sessionId)) { + logger.debug("sessionId[$sessionId] is in this host") + return true + } + return false + } + + companion object { + private val logger = LoggerFactory.getLogger(WebsocketService::class.java) + } +} diff --git a/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/util/HostUtils.kt b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/util/HostUtils.kt new file mode 100644 index 0000000000..19a64fdc90 --- /dev/null +++ b/src/backend/websocket/biz-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/util/HostUtils.kt @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket.util + +import org.slf4j.LoggerFactory +import java.net.DatagramSocket +import java.net.InetAddress +import java.net.URL + +object HostUtils { + + fun getHostIp(gateway: String?): String { + try { + val localHost = InetAddress.getLocalHost() + return if (localHost.isLoopbackAddress) { + getFromUDP(gateway) ?: DEFAULT_IP + } else { + localHost.hostAddress + } + } catch (e: Throwable) { + logger.warn("Fail to get local host ip", e) + try { + return getFromUDP(gateway) ?: DEFAULT_IP + } catch (t: Throwable) { + logger.warn("Fail to use socket to get the localhost host") + } + } + return DEFAULT_IP + } + + private fun getFromUDP(gateway: String?): String? { + if (gateway.isNullOrBlank()) { + return null + } + + val gatewayHost = try { + val url = URL(gateway) + url.host + } catch (t: Throwable) { + logger.warn("Fail to get the gateway host", t) + return null + } + + DatagramSocket().use { socket -> + socket.connect(InetAddress.getByName(gatewayHost), 10002) + return socket.localAddress.hostAddress + } + } + + fun getRealSession(query: String?): String? { + if (query.isNullOrEmpty()) { + return null + } + return query.substringAfter("sessionId=").substringBefore("&t=") + } + + private const val DEFAULT_IP = "127.0.0.1" + private val logger = LoggerFactory.getLogger(javaClass) +} diff --git a/src/backend/websocket/boot-websocket/build.gradle.kts b/src/backend/websocket/boot-websocket/build.gradle.kts new file mode 100644 index 0000000000..49bd2145ce --- /dev/null +++ b/src/backend/websocket/boot-websocket/build.gradle.kts @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED " AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +dependencies { + implementation(project(":websocket:biz-websocket")) +} diff --git a/src/backend/websocket/boot-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/WebsocketApplication.kt b/src/backend/websocket/boot-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/WebsocketApplication.kt new file mode 100644 index 0000000000..5a71325d24 --- /dev/null +++ b/src/backend/websocket/boot-websocket/src/main/kotlin/com/tencent/bkrepo/websocket/WebsocketApplication.kt @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.websocket + +import com.tencent.bkrepo.common.service.condition.MicroService +import org.springframework.boot.runApplication + +/** + * Websocket微服务启动类 + */ +@MicroService +class WebsocketApplication + +fun main(args: Array) { + runApplication(*args) +} diff --git a/src/backend/websocket/boot-websocket/src/main/resources/bootstrap.yml b/src/backend/websocket/boot-websocket/src/main/resources/bootstrap.yml new file mode 100644 index 0000000000..1fcf7233d0 --- /dev/null +++ b/src/backend/websocket/boot-websocket/src/main/resources/bootstrap.yml @@ -0,0 +1,4 @@ +spring.application.name: websocket +server.port: 25914 + +group: ${spring.cloud.consul.discovery.instance-id} diff --git a/src/backend/websocket/build.gradle.kts b/src/backend/websocket/build.gradle.kts new file mode 100644 index 0000000000..aa1c8e01b8 --- /dev/null +++ b/src/backend/websocket/build.gradle.kts @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED " AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ diff --git a/src/frontend/devops-op/src/api/executionClusters.js b/src/frontend/devops-op/src/api/executionClusters.js new file mode 100644 index 0000000000..a5edd0b4b8 --- /dev/null +++ b/src/frontend/devops-op/src/api/executionClusters.js @@ -0,0 +1,33 @@ +import request from '@/utils/request' + +const PREFIX_SERVICES = 'analyst/api/execution/clusters' + +export function clusters() { + return request({ + url: `${PREFIX_SERVICES}`, + method: 'get' + }) +} + +export function update(body) { + return request({ + url: `${PREFIX_SERVICES}/${body.name}`, + method: 'put', + data: body + }) +} + +export function create(body) { + return request({ + url: `${PREFIX_SERVICES}`, + method: 'post', + data: body + }) +} + +export function remove(name) { + return request({ + url: `${PREFIX_SERVICES}/${name}`, + method: 'delete' + }) +} diff --git a/src/frontend/devops-op/src/router/index.js b/src/frontend/devops-op/src/router/index.js index e5e859f09c..fab87ed73c 100644 --- a/src/frontend/devops-op/src/router/index.js +++ b/src/frontend/devops-op/src/router/index.js @@ -26,6 +26,7 @@ export const ROUTER_NAME_FILE_SYSTEM_RECORD = 'FileSystemRecord' export const ROUTER_NAME_REPO_CONFIG = 'RepoConfig' export const ROUTER_NAME_RATE_LIMITER_CONFIG = 'RateLimiterConfig' export const ROUTER_NAME_PRELOAD_CONFIG = 'PreloadConfig' +export const ROUTER_NAME_EXECUTION_CLUSTERS_CONFIG = 'ExecutionClustersConfig' Vue.use(Router) @@ -174,6 +175,12 @@ export const asyncRoutes = [ name: ROUTER_NAME_PROJECT_METRICS, meta: { title: '仓库大小统计', icon: 'file' }, component: () => import('@/views/node/ProjectMetrics') + }, + { + path: 'preloadConfig', + name: ROUTER_NAME_PRELOAD_CONFIG, + meta: { title: '制品预加载配置', icon: 'service-config' }, + component: () => import('@/views/preload/index') } ] }, @@ -261,6 +268,12 @@ export const asyncRoutes = [ name: ROUTER_NAME_PROJECT_SCAN_CONFIGURATIONS, component: () => import('@/views/scan/ProjectScanConfiguration'), meta: { title: '项目配置', icon: 'setting' } + }, + { + path: 'executionClustersConfig', + name: ROUTER_NAME_EXECUTION_CLUSTERS_CONFIG, + meta: { title: '扫描执行集群配置', icon: 'service-config' }, + component: () => import('@/views/execution-clusters/index') } ] }, @@ -327,18 +340,6 @@ export const asyncRoutes = [ } ] }, - { - path: '/preload-config', - component: Layout, - children: [ - { - path: '/', - name: ROUTER_NAME_PRELOAD_CONFIG, - meta: { title: '制品预加载配置', icon: 'service-config' }, - component: () => import('@/views/preload/index') - } - ] - }, // 404 page must be placed at the end !!! { path: '*', redirect: '/404', hidden: true } ] diff --git a/src/frontend/devops-op/src/views/execution-clusters/components/EditClusterConfigDialog.vue b/src/frontend/devops-op/src/views/execution-clusters/components/EditClusterConfigDialog.vue new file mode 100644 index 0000000000..1c5e1b2983 --- /dev/null +++ b/src/frontend/devops-op/src/views/execution-clusters/components/EditClusterConfigDialog.vue @@ -0,0 +1,276 @@ + + + + diff --git a/src/frontend/devops-op/src/views/execution-clusters/index.vue b/src/frontend/devops-op/src/views/execution-clusters/index.vue new file mode 100644 index 0000000000..142f2d8457 --- /dev/null +++ b/src/frontend/devops-op/src/views/execution-clusters/index.vue @@ -0,0 +1,119 @@ + + + + + diff --git a/src/frontend/devops-op/src/views/service/Instance.vue b/src/frontend/devops-op/src/views/service/Instance.vue index 4fffbb1d17..7c839e91ae 100644 --- a/src/frontend/devops-op/src/views/service/Instance.vue +++ b/src/frontend/devops-op/src/views/service/Instance.vue @@ -1,13 +1,12 @@