Skip to content

Commit

Permalink
Merge branch 'master' into master_issue2765
Browse files Browse the repository at this point in the history
  • Loading branch information
liuliaozhong authored Dec 6, 2024
2 parents f615cdf + 794126e commit 9f15b35
Show file tree
Hide file tree
Showing 106 changed files with 3,065 additions and 255 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,9 @@ open class SubtaskPoller(
open fun dispatch() {
executionClusterService.list().forEach {
executor.execute {
logger.info("cluster [${it.name}] start to dispatch subtask")
logger.debug("cluster [${it.name}] start to dispatch subtask")
dispatcherCache.get(it.name).dispatch()
logger.info("cluster [${it.name}] dispatch finished")
logger.debug("cluster [${it.name}] dispatch finished")
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ abstract class SubtaskPushDispatcher<T : ExecutionCluster>(

override fun dispatch() {
if (scanService.peek(executionCluster.name) == null) {
logger.info("cluster [${executionCluster.name}] has no subtask to dispatch")
logger.debug("cluster [${executionCluster.name}] has no subtask to dispatch")
return
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ class TemporaryScanTokenServiceImpl(
.build()
)
if (nodes.records.isEmpty()) {
throw SystemErrorException(RESOURCE_NOT_FOUND, sha256)
throw ErrorCodeException(RESOURCE_NOT_FOUND, "file[$sha256] of [$projectId:$repoName] not found")
}
val fullPath = nodes.records[0][NodeDetail::fullPath.name].toString()
return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))?.let { node ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import com.tencent.bkrepo.auth.pojo.oauth.JsonWebKey
import com.tencent.bkrepo.auth.pojo.oauth.JsonWebKeySet
import com.tencent.bkrepo.auth.pojo.oauth.OauthToken
import com.tencent.bkrepo.auth.dao.repository.OauthTokenRepository
import com.tencent.bkrepo.auth.pojo.enums.ResourceType
import com.tencent.bkrepo.auth.pojo.oauth.OidcConfiguration
import com.tencent.bkrepo.auth.pojo.oauth.UserInfo
import com.tencent.bkrepo.auth.service.OauthAuthorizationService
Expand Down Expand Up @@ -146,17 +147,16 @@ class OauthAuthorizationServiceImpl(
with(generateTokenRequest) {
Preconditions.checkNotNull(clientId, this::clientId.name)
Preconditions.checkNotNull(refreshToken, this::refreshToken.name)
val token = oauthTokenRepository.findFirstByAccountIdAndRefreshToken(clientId!!, refreshToken!!)
var token = oauthTokenRepository.findFirstByAccountIdAndRefreshToken(clientId!!, refreshToken!!)
?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, refreshToken!!)
val idToken = generateOpenIdToken(
clientId = clientId!!,
val client = accountDao.findById(clientId!!)
?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, clientId!!)
token = buildOauthToken(
userId = token.userId,
nonce = OauthUtils.generateRandomString(10)
nonce = OauthUtils.generateRandomString(10),
client = client,
openId = token.idToken != null
)
token.accessToken = idToken.toJwtToken()
token.issuedAt = Instant.now(Clock.systemDefaultZone())
token.idToken?.let { token.idToken = idToken }
oauthTokenRepository.save(token)
responseToken(transfer(token))
}
}
Expand Down Expand Up @@ -201,7 +201,7 @@ class OauthAuthorizationServiceImpl(
): TOauthToken {
val idToken = generateOpenIdToken(client.id!!, userId, nonce)
val tOauthToken = TOauthToken(
accessToken = idToken.toJwtToken(),
accessToken = idToken.toJwtToken(client.scope),
refreshToken = OauthUtils.generateRefreshToken(),
expireSeconds = oauthProperties.expiredDuration.seconds,
type = "Bearer",
Expand Down Expand Up @@ -300,7 +300,7 @@ class OauthAuthorizationServiceImpl(
accessToken = tOauthToken.accessToken,
tokenType = tOauthToken.type,
scope = if (tOauthToken.scope == null) "" else tOauthToken.scope!!.joinToString(StringPool.COMMA),
idToken = tOauthToken.idToken?.toJwtToken(),
idToken = tOauthToken.idToken?.toJwtToken(tOauthToken.scope),
refreshToken = tOauthToken.refreshToken,
expiresIn = if (tOauthToken.expireSeconds == null) {
null
Expand All @@ -310,12 +310,15 @@ class OauthAuthorizationServiceImpl(
}
)

private fun IdToken.toJwtToken(): String {
private fun IdToken.toJwtToken(scope: Set<ResourceType>?): String {
val claims = mutableMapOf<String, Any>()
claims["scope"] = scope.orEmpty()
claims.putAll(JsonUtils.objectMapper.convertValue(this, jacksonTypeRef()))
return JwtUtils.generateToken(
signingKey = RsaUtils.stringToPrivateKey(cryptoProperties.privateKeyStr2048PKCS8),
expireDuration = oauthProperties.expiredDuration,
subject = sub,
claims = JsonUtils.objectMapper.convertValue(this, jacksonTypeRef()),
claims = claims,
header = mapOf(KEY_ID_NAME to KEY_ID_VALUE),
algorithm = SignatureAlgorithm.RS256
)
Expand Down
1 change: 1 addition & 0 deletions src/backend/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ allprojects {
exclude(group = "log4j", module = "log4j")
exclude(group = "org.slf4j", module = "slf4j-log4j12")
exclude(group = "commons-logging", module = "commons-logging")
exclude(group = "org.springframework.boot", module = "spring-boot-starter-tomcat")
}

tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ object DockerUtils {
password: String?,
) {
val images = listImagesCmd().exec()
val exists = images.any { image ->
image.repoTags.any { it == tag }
}
val exists = images?.any { image ->
image.repoTags?.any { it == tag } ?: false
} ?: false
if (exists) {
return
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,3 +167,9 @@ const val BKREPO_TRACE = "X-BKREPO-RID"
*/
const val CODE_PROJECT_PREFIX = "CODE_"
const val CLOSED_SOURCE_PREFIX = "CLOSED_SOURCE_"


const val AUDITED_UID = "audited_uid"
const val AUDIT_REQUEST_URI = "audit_request_uri"
const val AUDIT_REQUEST_KEY = "http_request"
const val AUDIT_SHARE_USER_ID = "audit_share_user_id"
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService
import com.tencent.bkrepo.common.mongo.dao.util.Pages
import com.tencent.bkrepo.repository.pojo.node.NodeDetail
import com.tencent.bkrepo.repository.pojo.node.NodeInfo
import com.tencent.bkrepo.repository.pojo.node.NodeListOption
import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo
import org.slf4j.LoggerFactory
import org.springframework.data.domain.PageRequest
Expand Down Expand Up @@ -112,13 +111,20 @@ class ArtifactPreloadPlanServiceImpl(
if (!properties.enabled) {
return
}
val option = NodeListOption(pageSize = properties.maxNodes, includeFolder = false)
val res = nodeService.listNodePageBySha256(sha256, option)
val nodes = res.records
val nodes = nodeService.listNodeBySha256(
sha256 = sha256,
limit = properties.maxNodes,
includeMetadata = false,
includeDeleted = false,
tillLimit = false
)
if (nodes.size >= properties.maxNodes) {
// 限制查询出来的最大node数量,避免预加载计划创建时间过久
logger.warn("nodes of sha256[$sha256] exceed max page size[${properties.maxNodes}]")
return
} else if (nodes.isEmpty()) {
logger.debug("nodes of sha256[$sha256] found")
return
}
// node属于同一项目仓库的概率较大,缓存避免频繁查询策略
val strategyCache = HashMap<String, List<ArtifactPreloadStrategy>>()
Expand All @@ -127,11 +133,15 @@ class ArtifactPreloadPlanServiceImpl(
for (node in nodes) {
val repo = repositoryCache.get(buildRepoId(node.projectId, node.repoName))
if (repo.storageCredentialsKey != credentialsKey) {
logger.debug("credentialsKey of repo[${repo.name}] not match dst credentialsKey[${credentialsKey}]")
continue
}
val strategies = strategyCache.getOrPut(buildRepoId(node.projectId, node.repoName)) {
strategyService.list(node.projectId, node.repoName)
}
if (strategies.isEmpty()) {
logger.debug("preload strategy of repo[${repo.projectId}/${repo.name}] is empty")
}
strategies.forEach { strategy ->
matchAndGeneratePlan(strategy, node, repo.storageCredentialsKey)?.let { plans.add(it) }
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.mockito.ArgumentMatchers.anyBoolean
import org.mockito.ArgumentMatchers.anyString
import org.mockito.kotlin.any
import org.mockito.kotlin.anyOrNull
Expand Down Expand Up @@ -135,9 +136,9 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor(
node.copy(fullPath = "test.txt"),
node.copy(fullPath = "test2.txt"),
)
whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn(
Pages.ofResponse(Pages.ofRequest(0, 2000), nodes.size.toLong(), nodes)
)
whenever(
nodeService.listNodeBySha256(anyString(), any(), anyBoolean(), anyBoolean(), anyBoolean())
).thenReturn(nodes)

preloadPlanService.generatePlan(null, UT_SHA256)
plans = preloadPlanService.plans(UT_PROJECT_ID, UT_REPO_NAME, Pages.ofRequest(0, 10)).records
Expand Down Expand Up @@ -177,9 +178,9 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor(
for (i in 0..1000) {
nodes.add(buildNodeInfo())
}
whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn(
Pages.ofResponse(Pages.ofRequest(0, 2000), nodes.size.toLong(), nodes)
)
whenever(
nodeService.listNodeBySha256(anyString(), any(), anyBoolean(), anyBoolean(), anyBoolean())
).thenReturn(nodes)
preloadPlanService.generatePlan(null, UT_SHA256)
val plans = preloadPlanService.plans(UT_PROJECT_ID, UT_REPO_NAME, Pages.ofRequest(0, 10)).records
assertEquals(0, plans.size)
Expand Down Expand Up @@ -218,9 +219,9 @@ class ArtifactPreloadPlanServiceImplTest @Autowired constructor(
buildRepo(projectId = projectId, repoName = repoName)
)
val nodes = listOf(buildNodeInfo(projectId, repoName))
whenever(nodeService.listNodePageBySha256(anyString(), any())).thenReturn(
Pages.ofResponse(Pages.ofRequest(0, 20), 1L, nodes)
)
whenever(
nodeService.listNodeBySha256(anyString(), any(), anyBoolean(), anyBoolean(), anyBoolean())
).thenReturn(nodes)
}

// 构造测试数据
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ object ActionAuditContent {

// 节点
const val NODE_SHARE_CREATE_CONTENT = "create share link for node info $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"
const val NODE_SHARE_DOWNLOAD_CONTENT = "download share node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"
const val NODE_DOWNLOAD_WITH_TOKEN_CONTENT = "download node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"
const val NODE_UPLOAD_WITH_TOKEN_CONTENT = "upload node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"
const val NODE_SHARE_DOWNLOAD_CONTENT = "download share node $CONTENT_TEMPLATE with token in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"
const val NODE_DOWNLOAD_WITH_TOKEN_CONTENT = "download node $CONTENT_TEMPLATE with token in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"
const val NODE_UPLOAD_WITH_TOKEN_CONTENT = "upload node $CONTENT_TEMPLATE with token in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"

const val NODE_VIEW_CONTENT = "get node info $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"
const val NODE_CREATE_CONTENT = "create node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,21 @@ package com.tencent.bkrepo.common.artifact.audit

import com.tencent.bk.audit.filter.AuditPostFilter
import com.tencent.bk.audit.model.AuditEvent
import com.tencent.bkrepo.common.api.constant.AUDITED_UID
import com.tencent.bkrepo.common.api.constant.AUDIT_REQUEST_KEY
import com.tencent.bkrepo.common.api.constant.AUDIT_REQUEST_URI
import com.tencent.bkrepo.common.api.constant.AUDIT_SHARE_USER_ID

class BkAuditPostFilter: AuditPostFilter {
override fun map(auditEvent: AuditEvent): AuditEvent {
auditEvent.scopeType = "project"
auditEvent.scopeType = PROJECT_RESOURCE
// 特殊处理, 使用token下载时下载用户是根据token去判断后塞入httpAttribute中, 初始化时无法获取
if (auditEvent.extendData.isNullOrEmpty()) return auditEvent
auditEvent.extendData[AUDIT_SHARE_USER_ID] ?: return auditEvent
val auditedUid = auditEvent.extendData[AUDITED_UID]?.toString()
val auditRequestUri = auditEvent.extendData[AUDIT_REQUEST_URI]
auditEvent.username = auditedUid ?: auditEvent.username
auditEvent.addExtendData(AUDIT_REQUEST_KEY, auditRequestUri)
return auditEvent
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ class StorageManager(
forwardNode = forward(node, SecurityUtils.getUserId())
forwardNode?.let {
logger.info("Load[${node.identity()}] forward to [${it.identity()}].")
ActionAuditContext.current().addExtendData("alphaApkSha256", it.sha256)
ActionAuditContext.current().addExtendData("alphaApkMd5", it.md5)
}
}
val load = forwardNode ?: node
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.common.artifact.resolve.file.memory

import com.tencent.bkrepo.common.artifact.api.ArtifactFile
import com.tencent.bkrepo.common.artifact.hash.md5
import com.tencent.bkrepo.common.artifact.hash.sha1
import com.tencent.bkrepo.common.artifact.hash.sha256
import java.io.File
import java.io.InputStream

class ByteArrayArtifactFile(
private val data: ByteArray,
) : ArtifactFile {

private val sha1: String by lazy { getInputStream().sha1() }
private val sha256: String by lazy { getInputStream().sha256() }
private val md5: String by lazy { getInputStream().md5() }

override fun getInputStream(): InputStream {
return data.inputStream()
}

override fun getSize(): Long {
return data.size.toLong()
}

override fun isInMemory(): Boolean {
return true
}

override fun getFile(): File? {
throw UnsupportedOperationException("not implemented")
}

override fun flushToFile(): File {
throw UnsupportedOperationException("not implemented")
}

override fun delete() {
throw UnsupportedOperationException("not implemented")
}

override fun hasInitialized(): Boolean {
return true
}

override fun isFallback(): Boolean {
return false
}

override fun isInLocalDisk(): Boolean {
return false
}

override fun getFileMd5(): String {
return md5
}

override fun getFileSha1(): String {
return sha1
}

override fun getFileSha256(): String {
return sha256
}

fun byteArray() = data
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,5 +57,9 @@ data class RepositoryProperties(
* 当目录节点上的num字段小于该值时,去db中实时count目录大小
* 注意: 此配置的值要比listCountLimit大
*/
var subNodeLimit: Long = 100000000L
var subNodeLimit: Long = 100000000L,
/**
* 是否返回真实项目启用禁用状态
*/
var returnEnabled: Boolean = true
)
Loading

0 comments on commit 9f15b35

Please sign in to comment.