From 86425507986a115cb719b4d36b457c2e8b3f79b3 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Tue, 3 Dec 2024 15:59:56 +0800 Subject: [PATCH 01/28] =?UTF-8?q?feat:=20=E5=88=86=E5=9D=97=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E4=BC=98=E5=8C=96=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/AbstractArtifactRepository.kt | 31 ++++ .../repository/core/ArtifactRepository.kt | 8 + .../service/blocknode/BlockNodeService.kt | 7 + .../blocknode/impl/BlockNodeServiceImpl.kt | 23 ++- .../common/mongo/dao/AbstractMongoDao.kt | 9 ++ .../bkrepo/generic/constant/Constants.kt | 2 + .../generic/constant/GenericMessageCode.kt | 3 + .../artifact/GenericLocalRepository.kt | 89 +++++++++-- .../generic/controller/GenericController.kt | 52 ++++++- .../bkrepo/generic/service/UploadService.kt | 138 ++++++++++++++++++ .../resources/i18n/messages_en.properties | 3 + .../resources/i18n/messages_zh_CN.properties | 5 +- .../resources/i18n/messages_zh_TW.properties | 5 +- 13 files changed, 349 insertions(+), 26 deletions(-) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt index 7de1bbe1d9..41fe1247ad 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt @@ -55,6 +55,7 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.packages.PackageDownloadsService @@ -85,6 +86,9 @@ abstract class AbstractArtifactRepository : ArtifactRepository { @Autowired lateinit var nodeService: NodeService + @Autowired + lateinit var blockNodeService: BlockNodeService + @Autowired lateinit var nodeSearchService: NodeSearchService @@ -133,6 +137,19 @@ abstract class AbstractArtifactRepository : ArtifactRepository { } } + + override fun newUpload(context: ArtifactUploadContext) { + try { + this.onNewUploadBefore(context) + this.onNewUpload(context) + this.onUploadSuccess(context) + }catch (exception: RuntimeException){ + this.onUploadFailed(context, exception) + }finally { + this.onUploadFinished(context) + } + } + override fun download(context: ArtifactDownloadContext) { try { this.onDownloadBefore(context) @@ -193,6 +210,13 @@ abstract class AbstractArtifactRepository : ArtifactRepository { artifactMetrics.uploadingCount.incrementAndGet() } + /** + * 分块上传前回调 + */ + open fun onNewUploadBefore(context: ArtifactUploadContext) { + artifactMetrics.uploadingCount.incrementAndGet() + } + /** * 上传构件 */ @@ -200,6 +224,13 @@ abstract class AbstractArtifactRepository : ArtifactRepository { throw MethodNotAllowedException() } + /** + * 分块上传构件 + */ + open fun onNewUpload(context: ArtifactUploadContext) { + throw MethodNotAllowedException() + } + /** * 上传成功回调 */ diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt index 578badc2f2..fd13b86490 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt @@ -50,6 +50,13 @@ interface ArtifactRepository { */ fun upload(context: ArtifactUploadContext) + /** + * 分块上传构件 + * + * @param context 构件上传上下文 + */ + fun newUpload(context: ArtifactUploadContext) + /** * 下载构件 * @@ -84,4 +91,5 @@ interface ArtifactRepository { * @param context 构件迁移上下文 */ fun migrate(context: ArtifactMigrateContext): MigrateDetail + } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index 3557450932..b7bdbcdb34 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -55,6 +55,13 @@ interface BlockNodeService { storageCredentials: StorageCredentials? ): TBlockNode + /** + * 更新分块 + * */ + fun updateBlock( + blockNode: TBlockNode, + ) + /** * 删除旧分块，即删除非指定的nodeCurrentSha256的分块。 * 如果未指定nodeCurrentSha256，则删除节点所有分块 diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index eea4787b5f..87624957d8 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -43,11 +43,7 @@ import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.slf4j.LoggerFactory import org.springframework.context.annotation.Conditional -import org.springframework.data.mongodb.core.query.Criteria -import org.springframework.data.mongodb.core.query.Query -import org.springframework.data.mongodb.core.query.Update -import org.springframework.data.mongodb.core.query.and -import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.* import org.springframework.stereotype.Service import java.time.LocalDateTime @@ -68,6 +64,23 @@ class BlockNodeServiceImpl( } } + override fun updateBlock(blockNode: TBlockNode) { + with(blockNode) { + val criteria = Criteria.where(TBlockNode::id.name).`is`(id) + .and(TBlockNode::createdBy.name).`is`(createdBy) + .and(TBlockNode::nodeFullPath.name).`is`(nodeFullPath) + .and(TBlockNode::sha256.name).`is`(sha256) + .and(TBlockNode::projectId.name).`is`(projectId) + .and(TBlockNode::repoName.name).`is`(repoName) + .and(TBlockNode::size.name).`is`(size) + val update = Update() + .set(TBlockNode::startPos.name, startPos) + .set(TBlockNode::endPos.name, endPos) + blockNodeDao.updateBlock(Query(criteria), update, TBlockNode::class.java) + logger.info("Update block node[$projectId/$repoName/$nodeFullPath-$startPos], sha256[$sha256] success.") + } + } + override fun listBlocks( range: Range, projectId: String, diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt index 2cd5e7ccac..9bb84ce298 100644 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt +++ b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt @@ -135,6 +135,15 @@ abstract class AbstractMongoDao : MongoDao { return determineMongoTemplate().updateFirst(query, update, determineCollectionName(query)) } + fun updateBlock(query: Query, update: Update, clazz: Class): UpdateResult + { + if (logger.isDebugEnabled) { + logger.debug("Mongo Dao update block: [$query], [$update]") + } + // 同样的方法，classtype识别不出来，且更新不传classtype会更新不到对应的记录 + return determineMongoTemplate().updateFirst(query, update, clazz, determineCollectionName(query)) + } + override fun updateMulti(query: Query, update: Update): UpdateResult { if (logger.isDebugEnabled) { logger.debug("Mongo Dao updateMulti: [$query], [$update]") diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt index a57b395d8b..2f80458955 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt @@ -40,6 +40,8 @@ const val HEADER_EXPIRES = BKREPO_PREFIX + "EXPIRES" const val HEADER_SIZE = BKREPO_PREFIX + "SIZE" const val HEADER_UPLOAD_ID = BKREPO_PREFIX + "UPLOAD-ID" const val HEADER_SEQUENCE = BKREPO_PREFIX + "SEQUENCE" +const val HEADER_OFFSET = BKREPO_PREFIX + "OFFSET" +const val HEADER_FILE_SIZE = BKREPO_PREFIX + "SIZE" const val HEADER_OLD_FILE_PATH = BKREPO_PREFIX + "OLD-FILE-PATH" const val BKREPO_META_PREFIX = "X-BKREPO-META-" diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt index 931071fed2..8a9f2b3e49 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt @@ -49,6 +49,9 @@ enum class GenericMessageCode(private val businessCode: Int, private val key: St PIPELINE_REPO_MANUAL_UPLOAD_NOT_ALLOWED(10, "generic.pipeline-repo.manual-upload.not-allowed"), PIPELINE_ARTIFACT_PATH_ILLEGAL(11, "generic.pipeline.artifact.path.illegal"), CHUNKED_ARTIFACT_BROKEN(12, "generic.chunked.artifact.broken"), + BLOCK_FILE_NODE_NOT_CREATE(13, "generic.block.file.node.not-create"), + BLOCK_OFFSET_NOT_FOUND(14, "generic.block.node.offset.not-found"), + BLOCK_LIST_PATH_IS_FOLDER(15, "generic.block.list.path.is.folder") ; override fun getBusinessCode() = businessCode diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index 5c49f8be96..1170080033 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -66,6 +66,7 @@ import com.tencent.bkrepo.common.artifact.stream.EmptyInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.chunked.ChunkedUploadUtils import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils +import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.node.PipelineNodeService import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager @@ -87,19 +88,7 @@ import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.common.storage.message.StorageErrorException import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext -import com.tencent.bkrepo.generic.constant.BKREPO_META -import com.tencent.bkrepo.generic.constant.BKREPO_META_PREFIX -import com.tencent.bkrepo.generic.constant.CHUNKED_UPLOAD -import com.tencent.bkrepo.generic.constant.GenericMessageCode -import com.tencent.bkrepo.generic.constant.HEADER_BLOCK_APPEND -import com.tencent.bkrepo.generic.constant.HEADER_EXPIRES -import com.tencent.bkrepo.generic.constant.HEADER_MD5 -import com.tencent.bkrepo.generic.constant.HEADER_OVERWRITE -import com.tencent.bkrepo.generic.constant.HEADER_SEQUENCE -import com.tencent.bkrepo.generic.constant.HEADER_SHA256 -import com.tencent.bkrepo.generic.constant.HEADER_SIZE -import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_ID -import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_TYPE +import com.tencent.bkrepo.generic.constant.* import com.tencent.bkrepo.generic.pojo.ChunkedResponseProperty import com.tencent.bkrepo.generic.util.ChunkedRequestUtil.uploadResponse import com.tencent.bkrepo.replication.api.ClusterNodeClient @@ -125,6 +114,7 @@ import org.springframework.http.HttpMethod import org.springframework.stereotype.Component import org.springframework.util.unit.DataSize import java.net.URLDecoder +import java.time.LocalDateTime import java.util.Base64 import java.util.Locale import java.util.UUID @@ -151,6 +141,13 @@ class GenericLocalRepository( super.onUploadBefore(context) // 若不允许覆盖, 提前检查节点是否存在 checkNodeExist(context) + // 通用上传前检查 + baseUploadBefore(context) + // 二次检查，防止接收文件过程中，有并发上传成功的情况 + checkNodeExist(context) + } + + private fun baseUploadBefore(context: ArtifactUploadContext){ // 检查是否是覆盖流水线构件 checkIfOverwritePipelineArtifact(context) // 校验sha256 @@ -165,8 +162,17 @@ class GenericLocalRepository( if (uploadMd5 != null && !calculatedMd5.equals(uploadMd5, true)) { throw ErrorCodeException(ArtifactMessageCode.DIGEST_CHECK_FAILED, "md5") } - // 二次检查，防止接收文件过程中，有并发上传成功的情况 - checkNodeExist(context) + } + + override fun onNewUploadBefore(context: ArtifactUploadContext){ + // 上传前校验 + super.onNewUploadBefore(context) + // 若不允许覆盖, 提前检查Block节点是否存在 + //checkBlockNodeExist(context) + // 通用上传前检查 + baseUploadBefore(context) + // 二次检查，防止接收block文件过程中，有并发上传成功的情况 + //checkBlockNodeExist(context) } override fun onUpload(context: ArtifactUploadContext) { @@ -192,6 +198,46 @@ class GenericLocalRepository( } } + override fun onNewUpload(context: ArtifactUploadContext) { + with(context) { + checkBlockBaseNode(artifactInfo) + + val bArtifactFile = getArtifactFile() + val sha256 = getArtifactSha256() + + val sequence = context.request.getHeader(HEADER_SEQUENCE).toLongOrNull() + val offset = context.request.getHeader(HEADER_OFFSET)?.toLongOrNull() + + val blockNode = TBlockNode( + createdBy = userId, + createdDate = LocalDateTime.now(), + nodeFullPath = artifactInfo.getArtifactFullPath(), + startPos = offset ?: sequence ?: throw ErrorCodeException(GenericMessageCode.BLOCK_OFFSET_NOT_FOUND), + sha256 = sha256, + projectId = projectId, + repoName = repoName, + size = bArtifactFile.getSize() + ) + + val stored = storageService.store(sha256, bArtifactFile, storageCredentials) + + try { + blockNodeService.createBlock(blockNode, storageCredentials) + } catch (e: Exception) { + if (stored > 1) { + storageService.delete(sha256, storageCredentials) + } + // Log the exception for debugging purposes + logger.error("Failed to create block node", e) + throw e + } + + // Set response content type and write success response + context.response.contentType = MediaTypes.APPLICATION_JSON + context.response.writer.println(ResponseBuilder.success().toJsonString()) + } + } + override fun onUploadSuccess(context: ArtifactUploadContext) { super.onUploadSuccess(context) if (HttpContextHolder.getRequestOrNull()?.getParameter(PARAM_REPLICATE).toBoolean()) { @@ -262,6 +308,19 @@ class GenericLocalRepository( } } + private fun checkBlockNodeExist(context: ArtifactUploadContext) { + // todo 判断block node是否存在 + } + + + private fun checkBlockBaseNode(artifactInfo: ArtifactInfo) { + nodeService.getNodeDetail(artifactInfo) + ?: run { + logger.error("Node detail not found for artifact: $artifactInfo") + throw ErrorCodeException(GenericMessageCode.BLOCK_FILE_NODE_NOT_CREATE, artifactInfo) + } + } + private fun checkIfOverwritePipelineArtifact(context: ArtifactUploadContext) { val pipelineSource = context.repoName == PIPELINE || context.repoName == CUSTOM if (!pipelineSource) { diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt index fa9a336e7e..e6c7c1f451 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt @@ -65,13 +65,11 @@ import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.BATCH_M import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.BLOCK_MAPPING_URI import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.GENERIC_MAPPING_URI import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_ID -import com.tencent.bkrepo.generic.pojo.BatchDownloadPaths -import com.tencent.bkrepo.generic.pojo.BlockInfo -import com.tencent.bkrepo.generic.pojo.CompressedFileInfo -import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo +import com.tencent.bkrepo.generic.pojo.* import com.tencent.bkrepo.generic.service.CompressedFileService import com.tencent.bkrepo.generic.service.DownloadService import com.tencent.bkrepo.generic.service.UploadService +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import io.swagger.annotations.ApiOperation import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping @@ -117,6 +115,12 @@ class GenericController( uploadService.upload(artifactInfo, file) } + @PutMapping("/new$GENERIC_MAPPING_URI") + @Permission(ResourceType.NODE, PermissionAction.WRITE) + fun newBlockUpload(@ArtifactPathVariable artifactInfo: GenericArtifactInfo, file: ArtifactFile) { + uploadService.newBlockUpload(artifactInfo, file) + } + @AuditEntry( actionId = NODE_DELETE_ACTION ) @@ -177,6 +181,16 @@ class GenericController( return ResponseBuilder.success(uploadService.startBlockUpload(userId, artifactInfo)) } + @Permission(ResourceType.NODE, PermissionAction.WRITE) + @PostMapping("/new$BLOCK_MAPPING_URI") + fun newStartBlockUpload( + @RequestAttribute userId: String, + @ArtifactPathVariable artifactInfo: GenericArtifactInfo, + ): Response { + uploadService.newStartBlockUpload(userId, artifactInfo) + return ResponseBuilder.success() + } + @Permission(ResourceType.NODE, PermissionAction.WRITE) @DeleteMapping(BLOCK_MAPPING_URI) fun abortBlockUpload( @@ -188,6 +202,16 @@ class GenericController( return ResponseBuilder.success() } + @Permission(ResourceType.NODE, PermissionAction.WRITE) + @DeleteMapping("/new$BLOCK_MAPPING_URI") + fun abortNewBlockUpload( + @RequestAttribute userId: String, + @ArtifactPathVariable artifactInfo: GenericArtifactInfo, + ): Response { + uploadService.abortNewBlockUpload(userId, artifactInfo) + return ResponseBuilder.success() + } + @AuditEntry( actionId = NODE_CREATE_ACTION ) @@ -205,6 +229,7 @@ class GenericController( scopeId = "#artifactInfo?.projectId", content = ActionAuditContent.NODE_UPLOAD_CONTENT ) + @Permission(ResourceType.NODE, PermissionAction.WRITE) @PutMapping(BLOCK_MAPPING_URI) fun completeBlockUpload( @@ -216,6 +241,16 @@ class GenericController( return ResponseBuilder.success() } + @Permission(ResourceType.NODE, PermissionAction.WRITE) + @PutMapping("/new$BLOCK_MAPPING_URI") + fun completeNewBlockUpload( + @RequestAttribute userId: String, + @ArtifactPathVariable artifactInfo: GenericArtifactInfo, + ): Response { + uploadService.completeNewBlockUpload(userId, artifactInfo) + return ResponseBuilder.success() + } + @Permission(ResourceType.REPO, PermissionAction.READ) @GetMapping(BLOCK_MAPPING_URI) fun listBlock( @@ -226,6 +261,15 @@ class GenericController( return ResponseBuilder.success(uploadService.listBlock(userId, uploadId, artifactInfo)) } + @Permission(ResourceType.REPO, PermissionAction.READ) + @GetMapping("/new$BLOCK_MAPPING_URI") + fun listNewBlock( + @RequestAttribute userId: String, + @ArtifactPathVariable artifactInfo: GenericArtifactInfo, + ): Response> { + return ResponseBuilder.success(uploadService.newListBlock(userId, artifactInfo)) + } + @AuditEntry( actionId = NODE_DOWNLOAD_ACTION ) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 198562119e..4d990c1de2 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -37,12 +37,19 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.util.SecurityUtils @@ -53,13 +60,18 @@ import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.message.StorageErrorException import com.tencent.bkrepo.common.storage.pojo.FileInfo +import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo import com.tencent.bkrepo.generic.artifact.GenericLocalRepository import com.tencent.bkrepo.generic.constant.GenericMessageCode import com.tencent.bkrepo.generic.constant.HEADER_EXPIRES +import com.tencent.bkrepo.generic.constant.HEADER_FILE_SIZE import com.tencent.bkrepo.generic.constant.HEADER_OVERWRITE +import com.tencent.bkrepo.generic.model.NodeAttribute import com.tencent.bkrepo.generic.pojo.BlockInfo +import com.tencent.bkrepo.generic.pojo.NewBlockInfo import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -72,6 +84,7 @@ class UploadService( private val nodeService: NodeService, private val storageService: StorageService, private val repositoryService: RepositoryService, + private val blockNodeService: BlockNodeService, ) : ArtifactService() { fun upload(artifactInfo: GenericArtifactInfo, file: ArtifactFile) { @@ -79,6 +92,11 @@ class UploadService( repository.upload(context) } + fun newBlockUpload(artifactInfo: GenericArtifactInfo, file: ArtifactFile) { + val context = ArtifactUploadContext(file) + repository.newUpload(context) + } + fun delete(userId: String, artifactInfo: GenericArtifactInfo) { val context = ArtifactRemoveContext() repository.remove(context) @@ -114,6 +132,33 @@ class UploadService( } } + fun newStartBlockUpload(userId: String, artifactInfo: GenericArtifactInfo){ + val attributes = NodeAttribute( + uid = NodeAttribute.NOBODY, + gid = NodeAttribute.NOBODY, + mode = NodeAttribute.DEFAULT_MODE + ) + val fsAttr = MetadataModel( + key = FS_ATTR_KEY, + value = attributes + ) + val request = NodeCreateRequest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + folder = false, + fullPath = artifactInfo.getArtifactFullPath(), + sha256 = FAKE_SHA256, + md5 = FAKE_MD5, + operator = userId, + size = getLongHeader(HEADER_FILE_SIZE), + overwrite = getBooleanHeader(HEADER_OVERWRITE), + expires = getLongHeader(HEADER_EXPIRES), + nodeMetadata = listOf(fsAttr) + ) + ActionAuditContext.current().setInstance(request) + nodeService.createNode(request) + } + fun abortBlockUpload(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo) { val storageCredentials = getStorageCredentials(artifactInfo) checkUploadId(uploadId, storageCredentials) @@ -122,6 +167,13 @@ class UploadService( logger.info("User[${SecurityUtils.getPrincipal()}] abort upload block [$artifactInfo] success.") } + fun abortNewBlockUpload(userId: String, artifactInfo: GenericArtifactInfo) { + delete(userId, artifactInfo) + with(artifactInfo){ + blockNodeService.deleteBlocks(projectId, repoName, getArtifactFullPath()) + } + } + fun completeBlockUpload( userId: String, uploadId: String, @@ -167,6 +219,51 @@ class UploadService( logger.info("User[${SecurityUtils.getPrincipal()}] complete upload [$artifactInfo] success.") } + fun completeNewBlockUpload( + userId: String, + artifactInfo: GenericArtifactInfo + ) { + // 获取并按起始位置排序块信息列表 + val blockInfoList = listBlocks(artifactInfo).sortedBy { it.startPos } + + // 获取节点详情，如果节点不存在则抛出异常 + val node = ArtifactContextHolder.getNodeDetail(artifactInfo) + ?: throw ErrorCodeException(GenericMessageCode.BLOCK_FILE_NODE_NOT_CREATE, artifactInfo) + + // 如果节点是文件夹，抛出异常 + if (node.folder) { + throw ErrorCodeException(GenericMessageCode.BLOCK_LIST_PATH_IS_FOLDER, artifactInfo) + } + + var offset = 0L // 用于记录当前偏移量 + + // 遍历块信息列表，创建对应的块节点 + blockInfoList.forEach { blockInfo -> + val blockSize = blockInfo.size + val blockNode = TBlockNode( + id = blockInfo.id, + createdBy = blockInfo.createdBy, + createdDate = blockInfo.createdDate, + nodeFullPath = blockInfo.nodeFullPath, + startPos = offset, + endPos = offset + blockSize - 1, + sha256 = blockInfo.sha256, + projectId = blockInfo.projectId, + repoName = blockInfo.repoName, + size = blockSize + ) + + blockNodeService.updateBlock(blockNode) + offset += blockInfo.size // 更新偏移量 + } + + // 验证节点大小是否与块总大小一致 + if (node.size != offset) { + abortNewBlockUpload(userId, artifactInfo) + throw ErrorCodeException(GenericMessageCode.NODE_DATA_HAS_CHANGED, artifactInfo) + } + } + fun listBlock(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo): List { val storageCredentials = getStorageCredentials(artifactInfo) checkUploadId(uploadId, storageCredentials) @@ -177,6 +274,47 @@ class UploadService( } } + fun newListBlock(userId: String, artifactInfo: GenericArtifactInfo): List { + + val blockInfoList = listBlocks(artifactInfo) + + return blockInfoList.map { blockInfo -> + NewBlockInfo(blockInfo.size, blockInfo.sha256, blockInfo.startPos) + } + } + + private fun listBlocks(artifactInfo: GenericArtifactInfo): List + { + val node = ArtifactContextHolder.getNodeDetail(artifactInfo) + val context = ArtifactDownloadContext() + + if (node == null) { + // 安全调用 repositoryDetail，防止空指针异常 + if (context.repositoryDetail.category == RepositoryCategory.LOCAL) { + throw NodeNotFoundException(artifactInfo.getArtifactFullPath()) + } else { + // 当 node 为 null 且不是 LOCAL 仓库时，返回空列表 + return emptyList() + } + } + + if (node.folder) { + throw ErrorCodeException( + GenericMessageCode.BLOCK_LIST_PATH_IS_FOLDER, + artifactInfo + ) + } + + val blockInfoList = blockNodeService.listBlocks( + Range.full(node.size), + node.projectId, + node.repoName, + node.fullPath, + node.createdDate + ) + return blockInfoList + } + private fun checkUploadId(uploadId: String, storageCredentials: StorageCredentials?) { if (!storageService.checkBlockId(uploadId, storageCredentials)) { throw ErrorCodeException(GenericMessageCode.UPLOAD_ID_NOT_FOUND, uploadId) diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties index 475697be26..59f2f67024 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties @@ -41,3 +41,6 @@ generic.pipeline-repo.manual-upload.not-allowed=Manual upload to pipeline reposi generic.pipeline.metadata.incomplete=Pipeline metadata is incomplete. {0} is missing generic.pipeline.artifact.path.illegal=Note: The path parameter is incorrect. The specified file[{0}] belongs to the artifact of the pipeline repository[{1}] generic.chunked.artifact.broken=Chunked artifact broken +generic.block.file.node.not-create=Block file information node [{0}] is not created +generic.block.node.offset.not-found=Both HEADER_OFFSET and HEADER_SEQUENCE are null +generic.block.list.path.is.folder=The list block path is the directory diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties index 6584fac835..e618d20e20 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties @@ -40,4 +40,7 @@ generic.custom-artifact.overwrite.not-allowed=注意：存在窜改风险！指 generic.pipeline-repo.manual-upload.not-allowed=禁止手动上传到流水线仓库 generic.pipeline.metadata.incomplete=流水线元数据不完整. 缺少{0} generic.pipeline.artifact.path.illegal=注意：path参数出错，指定文件[{0}]属于流水线仓库[{1}]的制品 -generic.chunked.artifact.broken=分块上传制品文件已损坏 \ No newline at end of file +generic.chunked.artifact.broken=分块上传制品文件已损坏 +generic.block.file.node.not-create=分块文件信息节点[{0}]未创建 +generic.block.node.offset.not-found=请求头HEADER_OFFSET和HEADER_SEQUENCE不可同时为空 +generic.block.list.path.is.folder=获取分块路径是目录 diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties index fae3bbd2bb..3fec40d2eb 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties @@ -40,4 +40,7 @@ generic.custom-artifact.overwrite.not-allowed=已存在流水線歸檔產品[{0} generic.pipeline-repo.manual-upload.not-allowed=禁止手動上傳到流水線倉庫 generic.pipeline.metadata.incomplete=流水線元數據不完整，缺少{0} generic.pipeline.artifact.path.illegal=注意：path參數出錯，指定文件[{0}]屬於流水線倉庫[{1}]的製品 -generic.chunked.artifact.broken=製品已損壞 \ No newline at end of file +generic.chunked.artifact.broken=製品已損壞 +generic.block.file.node.not-create=分塊文件信息節點[{0}]未創建 +generic.block.node.offset.not-found=請求頭 HEADER_OFFSET 和 HEADER_SEQUENCE 不可同時為空 +generic.block.list.path.is.folder=獲取分塊路徑是目錄 \ No newline at end of file From 5cbe04da6042ab2870446a3719083b9a3a3e9a40 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Tue, 3 Dec 2024 16:37:13 +0800 Subject: [PATCH 02/28] =?UTF-8?q?fix:=20=E5=88=86=E5=9D=97=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E4=BC=98=E5=8C=96-=E9=97=AE=E9=A2=98=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../blocknode/impl/BlockNodeServiceImpl.kt | 7 +++++- .../bkrepo/generic/pojo/NewBlockInfo.kt | 15 +++++++++++++ .../artifact/GenericLocalRepository.kt | 15 ++++++++++++- .../generic/controller/GenericController.kt | 7 +++++- .../bkrepo/generic/model/NodeAttribute.kt | 22 +++++++++++++++++++ 5 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt create mode 100644 src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/model/NodeAttribute.kt diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index 87624957d8..b1170bb633 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -43,7 +43,12 @@ import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.slf4j.LoggerFactory import org.springframework.context.annotation.Conditional -import org.springframework.data.mongodb.core.query.* +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo + import org.springframework.stereotype.Service import java.time.LocalDateTime diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt new file mode 100644 index 0000000000..b31b5371bc --- /dev/null +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt @@ -0,0 +1,15 @@ +package com.tencent.bkrepo.generic.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + + +@ApiModel("新分块信息") +data class NewBlockInfo( + @ApiModelProperty("分块大小") + val size: Long, + @ApiModelProperty("分块sha256") + val sha256: String, + @ApiModelProperty("分块起始位置") + val startPos: Long +) \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index 1170080033..94eb51bfae 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -88,7 +88,20 @@ import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.common.storage.message.StorageErrorException import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext -import com.tencent.bkrepo.generic.constant.* +import com.tencent.bkrepo.generic.constant.BKREPO_META +import com.tencent.bkrepo.generic.constant.BKREPO_META_PREFIX +import com.tencent.bkrepo.generic.constant.CHUNKED_UPLOAD +import com.tencent.bkrepo.generic.constant.GenericMessageCode +import com.tencent.bkrepo.generic.constant.HEADER_MD5 +import com.tencent.bkrepo.generic.constant.HEADER_SEQUENCE +import com.tencent.bkrepo.generic.constant.HEADER_SHA256 +import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_ID +import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_TYPE +import com.tencent.bkrepo.generic.constant.HEADER_OFFSET +import com.tencent.bkrepo.generic.constant.HEADER_SIZE +import com.tencent.bkrepo.generic.constant.HEADER_OVERWRITE +import com.tencent.bkrepo.generic.constant.HEADER_BLOCK_APPEND +import com.tencent.bkrepo.generic.constant.HEADER_EXPIRES import com.tencent.bkrepo.generic.pojo.ChunkedResponseProperty import com.tencent.bkrepo.generic.util.ChunkedRequestUtil.uploadResponse import com.tencent.bkrepo.replication.api.ClusterNodeClient diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt index e6c7c1f451..ca18958273 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt @@ -65,7 +65,12 @@ import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.BATCH_M import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.BLOCK_MAPPING_URI import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.GENERIC_MAPPING_URI import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_ID -import com.tencent.bkrepo.generic.pojo.* +import com.tencent.bkrepo.generic.pojo.BatchDownloadPaths +import com.tencent.bkrepo.generic.pojo.BlockInfo +import com.tencent.bkrepo.generic.pojo.CompressedFileInfo +import com.tencent.bkrepo.generic.pojo.NewBlockInfo +import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo + import com.tencent.bkrepo.generic.service.CompressedFileService import com.tencent.bkrepo.generic.service.DownloadService import com.tencent.bkrepo.generic.service.UploadService diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/model/NodeAttribute.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/model/NodeAttribute.kt new file mode 100644 index 0000000000..803784e3c1 --- /dev/null +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/model/NodeAttribute.kt @@ -0,0 +1,22 @@ +package com.tencent.bkrepo.generic.model + +data class NodeAttribute( + // 用户id + val uid: String, + // 组id + val gid: String, + // 文件权限，八进制 + val mode: Int? = DEFAULT_MODE, + // windows文件flag，十六进制 + val flags: Int? = null, + // 设备文件设备号 + val rdev: Int? = null, + // 文件类型 + val type: Int? = null +) { + companion object { + const val DEFAULT_MODE = 644 + const val NOBODY = "nobody" + } +} + From a80c07af2eee2144881ba9a115f1fc54193fc909 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Fri, 6 Dec 2024 17:00:57 +0800 Subject: [PATCH 03/28] =?UTF-8?q?fix:=20=E5=88=86=E5=9D=97=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E4=BC=98=E5=8C=96-=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4+=E5=8D=95=E6=B5=8B=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/apidoc-user/generic/SeparateBlock.md | 205 +++++++++++++++ docs/apidoc/generic/SeparateBlock.md | 205 +++++++++++++++ .../core/AbstractArtifactRepository.kt | 27 -- .../repository/core/ArtifactRepository.kt | 2 +- .../repository/local/LocalRepository.kt | 31 +++ .../metadata/dao/blocknode/BlockNodeDao.kt | 13 +- .../blocknode/impl/BlockNodeServiceImpl.kt | 5 +- .../service/node/impl/NodeDeleteSupport.kt | 14 +- .../common/metadata/util/NodeDeleteHelper.kt | 4 +- .../common/mongo/dao/AbstractMongoDao.kt | 9 - .../generic/constant/GenericMessageCode.kt | 2 +- .../generic/biz-generic/build.gradle.kts | 3 + .../artifact/GenericLocalRepository.kt | 16 +- .../generic/controller/GenericController.kt | 48 ---- .../controller/SeparateBlockController.kt | 146 +++++++++++ .../bkrepo/generic/service/UploadService.kt | 11 +- .../resources/i18n/messages_en.properties | 2 +- .../resources/i18n/messages_zh_CN.properties | 2 +- .../resources/i18n/messages_zh_TW.properties | 2 +- .../generic/SeparateTestConfiguration.kt | 8 + .../bkrepo/generic/SeparateTestConstants.kt | 7 + .../generic/service/BlockNodeServiceTest.kt | 246 ++++++++++++++++++ 22 files changed, 890 insertions(+), 118 deletions(-) create mode 100644 docs/apidoc-user/generic/SeparateBlock.md create mode 100644 docs/apidoc/generic/SeparateBlock.md create mode 100644 src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt create mode 100644 src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConfiguration.kt create mode 100644 src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConstants.kt create mode 100644 src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt diff --git a/docs/apidoc-user/generic/SeparateBlock.md b/docs/apidoc-user/generic/SeparateBlock.md new file mode 100644 index 0000000000..431de2b9c1 --- /dev/null +++ b/docs/apidoc-user/generic/SeparateBlock.md @@ -0,0 +1,205 @@ +# Generic通用制品仓库分块文件操作 + +[toc] + +## 初始化分块上传 + +- API: POST /generic/separate/block/{project}/{repo}/{path} +- API 名称: start_block_upload +- 功能说明： + - 中文：初始化分块上传 + - English：start block upload + +- 请求体 +此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| +- |X-BKREPO-SIZE|string|是|0L|文件大小|上传文件的总大小| + |X-BKREPO-MD5|string|否|无|文件md5|file md5| + |X-BKREPO-OVERWRITE|boolean|否|false|是否覆盖已存在文件|overwrite exist file| + +- 响应体 + +``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : null + } +``` + +## 上传分块文件 + +- API: PUT /generic/separate/{project}/{repo}/{path} +- API 名称: block_upload +- 功能说明： + - 中文：分块上传通用制品文件 + - English：upload generic artifact file block + +- 请求体 +[文件流] + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + + |字段|类型| 是否必须 |默认值| 说明 | Description | + |---|---|-|---|------|---------| + |X-BKREPO-SEQUENCE|int|是|无| 分块序号(从1开始), SEQUENCE 和 OFFSET 二者不可同时为空 |block sequence(start from 1) | +- |X-BKREPO-OFFSET|int|是|无| 分块偏移量,SEQUENCE 和 OFFSET 二者不可同时为空| block offset(start from 0)| + |X-BKREPO-SHA256|string|否|无| 文件sha256|file sha256| + |X-BKREPO-MD5|string|否|无| 文件md5| file md5| + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : null + } + ``` + +## 完成分块上传 + +- API: PUT /generic/separate/block/{project}/{repo}/{path} +- API 名称: complete_block_upload +- 功能说明： + - 中文：完成化分块上传 + - English：complete block upload + +- 请求体 +此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : "" + } + ``` + +## 终止(取消)分块上传 + +- API: DELETE /generic/separate/block/{project}/{repo}/{path} +- API 名称: abort_block_upload +- 功能说明： + - 中文：终止(取消)分块上传 + - English：abort block upload + +- 请求体 +此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : null + } + ``` + +## 查询已上传的分块列表 + +- API: GET /generic/separate/block/{project}/{repo}/{path} +- API 名称: list_upload_block +- 功能说明： + - 中文：查询已上传的分块列表 + - English：list upload block + +- 请求体 + 此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |Authorization|string|否|无|Basic Auth认证头，Basic base64(username:password)|Basic Auth header| + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : [ { + "size" : 10240, + "sha256" : "d17f25ecfbcc7857f7bebea469308be0b2580943e96d13a3ad98a13675c4bfc2", + "startPos" : xxx + }, { + "size" : 10240, + "sha256" : "cc399d73903f06ee694032ab0538f05634ff7e1ce5e8e50ac330a871484f34cf", + "startPos" : xxx + } ], + "traceId" : null + } + ``` + +- 响应字段说明 + + |字段|类型|说明|Description| + |---|---|---|---| + |code|bool|错误编码。 0表示success，>0表示失败错误|0:success, other: failure| + |message|string|错误消息|the failure message| + |data|list|分块列表|block list| + |traceId|string|请求跟踪id|trace id| + +- 分块信息字段说明 + + |字段| 类型 | 说明 |Description| + |---|-----|----|---| + |size| long |分块大小|block size| + |sha256| string| 分块sha256|block sha256 checksum| + |startPos| long| 分块起始位置|block sequence| \ No newline at end of file diff --git a/docs/apidoc/generic/SeparateBlock.md b/docs/apidoc/generic/SeparateBlock.md new file mode 100644 index 0000000000..431de2b9c1 --- /dev/null +++ b/docs/apidoc/generic/SeparateBlock.md @@ -0,0 +1,205 @@ +# Generic通用制品仓库分块文件操作 + +[toc] + +## 初始化分块上传 + +- API: POST /generic/separate/block/{project}/{repo}/{path} +- API 名称: start_block_upload +- 功能说明： + - 中文：初始化分块上传 + - English：start block upload + +- 请求体 +此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| +- |X-BKREPO-SIZE|string|是|0L|文件大小|上传文件的总大小| + |X-BKREPO-MD5|string|否|无|文件md5|file md5| + |X-BKREPO-OVERWRITE|boolean|否|false|是否覆盖已存在文件|overwrite exist file| + +- 响应体 + +``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : null + } +``` + +## 上传分块文件 + +- API: PUT /generic/separate/{project}/{repo}/{path} +- API 名称: block_upload +- 功能说明： + - 中文：分块上传通用制品文件 + - English：upload generic artifact file block + +- 请求体 +[文件流] + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + + |字段|类型| 是否必须 |默认值| 说明 | Description | + |---|---|-|---|------|---------| + |X-BKREPO-SEQUENCE|int|是|无| 分块序号(从1开始), SEQUENCE 和 OFFSET 二者不可同时为空 |block sequence(start from 1) | +- |X-BKREPO-OFFSET|int|是|无| 分块偏移量,SEQUENCE 和 OFFSET 二者不可同时为空| block offset(start from 0)| + |X-BKREPO-SHA256|string|否|无| 文件sha256|file sha256| + |X-BKREPO-MD5|string|否|无| 文件md5| file md5| + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : null + } + ``` + +## 完成分块上传 + +- API: PUT /generic/separate/block/{project}/{repo}/{path} +- API 名称: complete_block_upload +- 功能说明： + - 中文：完成化分块上传 + - English：complete block upload + +- 请求体 +此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : "" + } + ``` + +## 终止(取消)分块上传 + +- API: DELETE /generic/separate/block/{project}/{repo}/{path} +- API 名称: abort_block_upload +- 功能说明： + - 中文：终止(取消)分块上传 + - English：abort block upload + +- 请求体 +此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : null + } + ``` + +## 查询已上传的分块列表 + +- API: GET /generic/separate/block/{project}/{repo}/{path} +- API 名称: list_upload_block +- 功能说明： + - 中文：查询已上传的分块列表 + - English：list upload block + +- 请求体 + 此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |project|string|是|无|项目名称|project name| + |repo|string|是|无|仓库名称|repo name| + |path|string|是|无|完整路径|full path| + +- 请求头 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |Authorization|string|否|无|Basic Auth认证头，Basic base64(username:password)|Basic Auth header| + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : [ { + "size" : 10240, + "sha256" : "d17f25ecfbcc7857f7bebea469308be0b2580943e96d13a3ad98a13675c4bfc2", + "startPos" : xxx + }, { + "size" : 10240, + "sha256" : "cc399d73903f06ee694032ab0538f05634ff7e1ce5e8e50ac330a871484f34cf", + "startPos" : xxx + } ], + "traceId" : null + } + ``` + +- 响应字段说明 + + |字段|类型|说明|Description| + |---|---|---|---| + |code|bool|错误编码。 0表示success，>0表示失败错误|0:success, other: failure| + |message|string|错误消息|the failure message| + |data|list|分块列表|block list| + |traceId|string|请求跟踪id|trace id| + +- 分块信息字段说明 + + |字段| 类型 | 说明 |Description| + |---|-----|----|---| + |size| long |分块大小|block size| + |sha256| string| 分块sha256|block sha256 checksum| + |startPos| long| 分块起始位置|block sequence| \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt index 41fe1247ad..a6f1157bd8 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt @@ -137,19 +137,6 @@ abstract class AbstractArtifactRepository : ArtifactRepository { } } - - override fun newUpload(context: ArtifactUploadContext) { - try { - this.onNewUploadBefore(context) - this.onNewUpload(context) - this.onUploadSuccess(context) - }catch (exception: RuntimeException){ - this.onUploadFailed(context, exception) - }finally { - this.onUploadFinished(context) - } - } - override fun download(context: ArtifactDownloadContext) { try { this.onDownloadBefore(context) @@ -210,13 +197,6 @@ abstract class AbstractArtifactRepository : ArtifactRepository { artifactMetrics.uploadingCount.incrementAndGet() } - /** - * 分块上传前回调 - */ - open fun onNewUploadBefore(context: ArtifactUploadContext) { - artifactMetrics.uploadingCount.incrementAndGet() - } - /** * 上传构件 */ @@ -224,13 +204,6 @@ abstract class AbstractArtifactRepository : ArtifactRepository { throw MethodNotAllowedException() } - /** - * 分块上传构件 - */ - open fun onNewUpload(context: ArtifactUploadContext) { - throw MethodNotAllowedException() - } - /** * 上传成功回调 */ diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt index fd13b86490..aef9425d14 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt @@ -55,7 +55,7 @@ interface ArtifactRepository { * * @param context 构件上传上下文 */ - fun newUpload(context: ArtifactUploadContext) + fun newUpload(context: ArtifactUploadContext) {} /** * 下载构件 diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/local/LocalRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/local/LocalRepository.kt index b6a610bd7a..db93d1536b 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/local/LocalRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/local/LocalRepository.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.common.artifact.repository.local +import com.tencent.bkrepo.common.api.exception.MethodNotAllowedException import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext @@ -105,4 +106,34 @@ abstract class LocalRepository : AbstractArtifactRepository() { operator = context.userId ) } + + /** + * 分块上传 + */ + override fun newUpload(context: ArtifactUploadContext) { + try { + this.onNewUploadBefore(context) + this.onNewUpload(context) + this.onUploadSuccess(context) + }catch (exception: RuntimeException){ + this.onUploadFailed(context, exception) + }finally { + this.onUploadFinished(context) + } + } + + /** + * 分块上传前回调 + */ + open fun onNewUploadBefore(context: ArtifactUploadContext) { + artifactMetrics.uploadingCount.incrementAndGet() + } + + + /** + * 分块上传构件 + */ + open fun onNewUpload(context: ArtifactUploadContext) { + throw MethodNotAllowedException() + } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt index 3dc8412b1b..915746395e 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt @@ -27,12 +27,23 @@ package com.tencent.bkrepo.common.metadata.dao.blocknode +import com.mongodb.client.result.UpdateResult import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.mongo.dao.sharding.HashShardingMongoDao import org.springframework.context.annotation.Conditional +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update import org.springframework.stereotype.Repository @Repository @Conditional(SyncCondition::class) -class BlockNodeDao : HashShardingMongoDao() +class BlockNodeDao : HashShardingMongoDao(){ + + fun updateBlock(query: Query, update: Update): UpdateResult { + if (logger.isDebugEnabled) { + logger.debug("Mongo Dao updateFirst: [$query], [$update]") + } + return determineMongoTemplate().updateFirst(query, update, TBlockNode::class.java, determineCollectionName(query)) + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index b1170bb633..8b62839c14 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -72,16 +72,13 @@ class BlockNodeServiceImpl( override fun updateBlock(blockNode: TBlockNode) { with(blockNode) { val criteria = Criteria.where(TBlockNode::id.name).`is`(id) - .and(TBlockNode::createdBy.name).`is`(createdBy) .and(TBlockNode::nodeFullPath.name).`is`(nodeFullPath) - .and(TBlockNode::sha256.name).`is`(sha256) .and(TBlockNode::projectId.name).`is`(projectId) .and(TBlockNode::repoName.name).`is`(repoName) - .and(TBlockNode::size.name).`is`(size) val update = Update() .set(TBlockNode::startPos.name, startPos) .set(TBlockNode::endPos.name, endPos) - blockNodeDao.updateBlock(Query(criteria), update, TBlockNode::class.java) + blockNodeDao.updateBlock(Query(criteria), update) logger.info("Update block node[$projectId/$repoName/$nodeFullPath-$startPos], sha256[$sha256] success.") } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 74bae3b2b8..95da4378cc 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.HumanReadable import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode @@ -210,6 +211,7 @@ open class NodeDeleteSupport( if (deletedNum == 0L) { return NodeDeleteResult(deletedNum, deletedSize, deleteTime) } + if (decreaseVolume) { var deletedCriteria = criteria.and(TNode::deleted).isEqualTo(deleteTime) fullPaths?.let { @@ -220,11 +222,17 @@ open class NodeDeleteSupport( deletedSize = nodeBaseService.aggregateComputeSize(deletedCriteria) quotaService.decreaseUsedVolume(projectId, repoName, deletedSize) } - fullPaths?.forEach { + fullPaths?.forEach { fullPath -> if (routerControllerProperties.enabled) { - routerControllerClient.removeNodes(projectId, repoName, it) + routerControllerClient.removeNodes(projectId, repoName, fullPath) + } + publishEvent(buildDeletedEvent(projectId, repoName, fullPath, operator)) + + nodeDao.find(Query(buildCriteria(projectId, repoName, fullPath, deleteTime))).first().let { + if (it.sha256 == FAKE_SHA256) { + nodeBaseService.blockNodeService.deleteBlocks(projectId, repoName, fullPath) + } } - publishEvent(buildDeletedEvent(projectId, repoName, it, operator)) } } catch (exception: DuplicateKeyException) { logger.warn("Delete node[$resourceKey] by [$operator] error: [${exception.message}]") diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt index 7230b47b59..73fa78fca9 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt @@ -6,19 +6,21 @@ import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.and import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where +import java.time.LocalDateTime object NodeDeleteHelper { fun buildCriteria( projectId: String, repoName: String, fullPath: String, + deletedDate: LocalDateTime?= null ): Criteria { val normalizedFullPath = PathUtils.normalizeFullPath(fullPath) val normalizedPath = PathUtils.toPath(normalizedFullPath) val escapedPath = PathUtils.escapeRegex(normalizedPath) val criteria = where(TNode::projectId).isEqualTo(projectId) .and(TNode::repoName).isEqualTo(repoName) - .and(TNode::deleted).isEqualTo(null) + .and(TNode::deleted).isEqualTo(deletedDate) .orOperator( where(TNode::fullPath).regex("^$escapedPath"), where(TNode::fullPath).isEqualTo(normalizedFullPath) diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt index 9bb84ce298..2cd5e7ccac 100644 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt +++ b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt @@ -135,15 +135,6 @@ abstract class AbstractMongoDao : MongoDao { return determineMongoTemplate().updateFirst(query, update, determineCollectionName(query)) } - fun updateBlock(query: Query, update: Update, clazz: Class): UpdateResult - { - if (logger.isDebugEnabled) { - logger.debug("Mongo Dao update block: [$query], [$update]") - } - // 同样的方法，classtype识别不出来，且更新不传classtype会更新不到对应的记录 - return determineMongoTemplate().updateFirst(query, update, clazz, determineCollectionName(query)) - } - override fun updateMulti(query: Query, update: Update): UpdateResult { if (logger.isDebugEnabled) { logger.debug("Mongo Dao updateMulti: [$query], [$update]") diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt index 8a9f2b3e49..d6fb717047 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt @@ -50,7 +50,7 @@ enum class GenericMessageCode(private val businessCode: Int, private val key: St PIPELINE_ARTIFACT_PATH_ILLEGAL(11, "generic.pipeline.artifact.path.illegal"), CHUNKED_ARTIFACT_BROKEN(12, "generic.chunked.artifact.broken"), BLOCK_FILE_NODE_NOT_CREATE(13, "generic.block.file.node.not-create"), - BLOCK_OFFSET_NOT_FOUND(14, "generic.block.node.offset.not-found"), + BLOCK_HEAD_NOT_FOUND(14, "generic.block.node.head.not-found"), BLOCK_LIST_PATH_IS_FOLDER(15, "generic.block.list.path.is.folder") ; diff --git a/src/backend/generic/biz-generic/build.gradle.kts b/src/backend/generic/biz-generic/build.gradle.kts index 88361d8c4c..2be14408e3 100644 --- a/src/backend/generic/biz-generic/build.gradle.kts +++ b/src/backend/generic/biz-generic/build.gradle.kts @@ -36,4 +36,7 @@ dependencies { api(project(":common:common-generic")) api(project(":common:common-artifact:artifact-service")) implementation(project(":common:common-artifact:artifact-cache")) + testImplementation("org.mockito.kotlin:mockito-kotlin") + testImplementation("de.flapdoodle.embed:de.flapdoodle.embed.mongo") + testImplementation("io.mockk:mockk") } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index 94eb51bfae..cb1eb1fc1b 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -179,13 +179,11 @@ class GenericLocalRepository( override fun onNewUploadBefore(context: ArtifactUploadContext){ // 上传前校验 - super.onNewUploadBefore(context) - // 若不允许覆盖, 提前检查Block节点是否存在 - //checkBlockNodeExist(context) + super.onUploadBefore(context) + // 检查BlockNodeBase是否存在 + checkBlockBaseNode(context.artifactInfo) // 通用上传前检查 baseUploadBefore(context) - // 二次检查，防止接收block文件过程中，有并发上传成功的情况 - //checkBlockNodeExist(context) } override fun onUpload(context: ArtifactUploadContext) { @@ -213,7 +211,6 @@ class GenericLocalRepository( override fun onNewUpload(context: ArtifactUploadContext) { with(context) { - checkBlockBaseNode(artifactInfo) val bArtifactFile = getArtifactFile() val sha256 = getArtifactSha256() @@ -225,7 +222,7 @@ class GenericLocalRepository( createdBy = userId, createdDate = LocalDateTime.now(), nodeFullPath = artifactInfo.getArtifactFullPath(), - startPos = offset ?: sequence ?: throw ErrorCodeException(GenericMessageCode.BLOCK_OFFSET_NOT_FOUND), + startPos = offset ?: sequence ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND), sha256 = sha256, projectId = projectId, repoName = repoName, @@ -321,11 +318,6 @@ class GenericLocalRepository( } } - private fun checkBlockNodeExist(context: ArtifactUploadContext) { - // todo 判断block node是否存在 - } - - private fun checkBlockBaseNode(artifactInfo: ArtifactInfo) { nodeService.getNodeDetail(artifactInfo) ?: run { diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt index ca18958273..6fce444949 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt @@ -68,13 +68,10 @@ import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_ID import com.tencent.bkrepo.generic.pojo.BatchDownloadPaths import com.tencent.bkrepo.generic.pojo.BlockInfo import com.tencent.bkrepo.generic.pojo.CompressedFileInfo -import com.tencent.bkrepo.generic.pojo.NewBlockInfo import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo - import com.tencent.bkrepo.generic.service.CompressedFileService import com.tencent.bkrepo.generic.service.DownloadService import com.tencent.bkrepo.generic.service.UploadService -import com.tencent.bkrepo.repository.pojo.node.NodeDetail import io.swagger.annotations.ApiOperation import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping @@ -120,12 +117,6 @@ class GenericController( uploadService.upload(artifactInfo, file) } - @PutMapping("/new$GENERIC_MAPPING_URI") - @Permission(ResourceType.NODE, PermissionAction.WRITE) - fun newBlockUpload(@ArtifactPathVariable artifactInfo: GenericArtifactInfo, file: ArtifactFile) { - uploadService.newBlockUpload(artifactInfo, file) - } - @AuditEntry( actionId = NODE_DELETE_ACTION ) @@ -186,16 +177,6 @@ class GenericController( return ResponseBuilder.success(uploadService.startBlockUpload(userId, artifactInfo)) } - @Permission(ResourceType.NODE, PermissionAction.WRITE) - @PostMapping("/new$BLOCK_MAPPING_URI") - fun newStartBlockUpload( - @RequestAttribute userId: String, - @ArtifactPathVariable artifactInfo: GenericArtifactInfo, - ): Response { - uploadService.newStartBlockUpload(userId, artifactInfo) - return ResponseBuilder.success() - } - @Permission(ResourceType.NODE, PermissionAction.WRITE) @DeleteMapping(BLOCK_MAPPING_URI) fun abortBlockUpload( @@ -207,16 +188,6 @@ class GenericController( return ResponseBuilder.success() } - @Permission(ResourceType.NODE, PermissionAction.WRITE) - @DeleteMapping("/new$BLOCK_MAPPING_URI") - fun abortNewBlockUpload( - @RequestAttribute userId: String, - @ArtifactPathVariable artifactInfo: GenericArtifactInfo, - ): Response { - uploadService.abortNewBlockUpload(userId, artifactInfo) - return ResponseBuilder.success() - } - @AuditEntry( actionId = NODE_CREATE_ACTION ) @@ -246,16 +217,6 @@ class GenericController( return ResponseBuilder.success() } - @Permission(ResourceType.NODE, PermissionAction.WRITE) - @PutMapping("/new$BLOCK_MAPPING_URI") - fun completeNewBlockUpload( - @RequestAttribute userId: String, - @ArtifactPathVariable artifactInfo: GenericArtifactInfo, - ): Response { - uploadService.completeNewBlockUpload(userId, artifactInfo) - return ResponseBuilder.success() - } - @Permission(ResourceType.REPO, PermissionAction.READ) @GetMapping(BLOCK_MAPPING_URI) fun listBlock( @@ -266,15 +227,6 @@ class GenericController( return ResponseBuilder.success(uploadService.listBlock(userId, uploadId, artifactInfo)) } - @Permission(ResourceType.REPO, PermissionAction.READ) - @GetMapping("/new$BLOCK_MAPPING_URI") - fun listNewBlock( - @RequestAttribute userId: String, - @ArtifactPathVariable artifactInfo: GenericArtifactInfo, - ): Response> { - return ResponseBuilder.success(uploadService.newListBlock(userId, artifactInfo)) - } - @AuditEntry( actionId = NODE_DOWNLOAD_ACTION ) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt new file mode 100644 index 0000000000..ac172fae9b --- /dev/null +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt @@ -0,0 +1,146 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.generic.controller + +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bkrepo.auth.pojo.enums.PermissionAction +import com.tencent.bkrepo.auth.pojo.enums.ResourceType +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.security.permission.Permission +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo +import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.BLOCK_MAPPING_URI +import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.GENERIC_MAPPING_URI +import com.tencent.bkrepo.generic.pojo.NewBlockInfo +import com.tencent.bkrepo.generic.service.UploadService +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestAttribute +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/separate") +class SeparateBlockController( + private val uploadService: UploadService, +) { + + @Permission(ResourceType.NODE, PermissionAction.WRITE) + @PostMapping(BLOCK_MAPPING_URI) + fun newStartBlockUpload( + @RequestAttribute userId: String, + @ArtifactPathVariable artifactInfo: GenericArtifactInfo, + ): Response { + uploadService.newStartBlockUpload(userId, artifactInfo) + return ResponseBuilder.success() + } + + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) + @PutMapping(GENERIC_MAPPING_URI) + @Permission(ResourceType.NODE, PermissionAction.WRITE) + fun newBlockUpload(@ArtifactPathVariable artifactInfo: GenericArtifactInfo, file: ArtifactFile) { + uploadService.newBlockUpload(artifactInfo, file) + } + + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) + @Permission(ResourceType.NODE, PermissionAction.WRITE) + @PutMapping(BLOCK_MAPPING_URI) + fun completeNewBlockUpload( + @RequestAttribute userId: String, + @ArtifactPathVariable artifactInfo: GenericArtifactInfo, + ): Response { + uploadService.completeNewBlockUpload(userId, artifactInfo) + return ResponseBuilder.success() + } + + @Permission(ResourceType.NODE, PermissionAction.WRITE) + @DeleteMapping(BLOCK_MAPPING_URI) + fun abortNewBlockUpload( + @RequestAttribute userId: String, + @ArtifactPathVariable artifactInfo: GenericArtifactInfo, + ): Response { + uploadService.abortNewBlockUpload(userId, artifactInfo) + return ResponseBuilder.success() + } + + @Permission(ResourceType.REPO, PermissionAction.READ) + @GetMapping(BLOCK_MAPPING_URI) + fun listNewBlock( + @RequestAttribute userId: String, + @ArtifactPathVariable artifactInfo: GenericArtifactInfo, + ): Response> { + return ResponseBuilder.success(uploadService.newListBlock(userId, artifactInfo)) + } +} diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 4d990c1de2..1e4367be18 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -150,7 +150,8 @@ class UploadService( sha256 = FAKE_SHA256, md5 = FAKE_MD5, operator = userId, - size = getLongHeader(HEADER_FILE_SIZE), + size = getLongHeader(HEADER_FILE_SIZE).takeIf { it > 0L } + ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND) , overwrite = getBooleanHeader(HEADER_OVERWRITE), expires = getLongHeader(HEADER_EXPIRES), nodeMetadata = listOf(fsAttr) @@ -169,9 +170,6 @@ class UploadService( fun abortNewBlockUpload(userId: String, artifactInfo: GenericArtifactInfo) { delete(userId, artifactInfo) - with(artifactInfo){ - blockNodeService.deleteBlocks(projectId, repoName, getArtifactFullPath()) - } } fun completeBlockUpload( @@ -219,10 +217,7 @@ class UploadService( logger.info("User[${SecurityUtils.getPrincipal()}] complete upload [$artifactInfo] success.") } - fun completeNewBlockUpload( - userId: String, - artifactInfo: GenericArtifactInfo - ) { + fun completeNewBlockUpload(userId: String, artifactInfo: GenericArtifactInfo) { // 获取并按起始位置排序块信息列表 val blockInfoList = listBlocks(artifactInfo).sortedBy { it.startPos } diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties index 59f2f67024..cab827727b 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties @@ -42,5 +42,5 @@ generic.pipeline.metadata.incomplete=Pipeline metadata is incomplete. {0} is mis generic.pipeline.artifact.path.illegal=Note: The path parameter is incorrect. The specified file[{0}] belongs to the artifact of the pipeline repository[{1}] generic.chunked.artifact.broken=Chunked artifact broken generic.block.file.node.not-create=Block file information node [{0}] is not created -generic.block.node.offset.not-found=Both HEADER_OFFSET and HEADER_SEQUENCE are null +generic.block.node.head.not-found=Both HEADER_OFFSET and HEADER_SEQUENCE are null, HEADER_FILE_SIZE are null generic.block.list.path.is.folder=The list block path is the directory diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties index e618d20e20..25bdca905a 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties @@ -42,5 +42,5 @@ generic.pipeline.metadata.incomplete=流水线元数据不完整. 缺少{0} generic.pipeline.artifact.path.illegal=注意：path参数出错，指定文件[{0}]属于流水线仓库[{1}]的制品 generic.chunked.artifact.broken=分块上传制品文件已损坏 generic.block.file.node.not-create=分块文件信息节点[{0}]未创建 -generic.block.node.offset.not-found=请求头HEADER_OFFSET和HEADER_SEQUENCE不可同时为空 +generic.block.node.head.not-found=请求头HEADER_OFFSET和HEADER_SEQUENCE不可同时为空, HEADER_FILE_SIZE不可为空 generic.block.list.path.is.folder=获取分块路径是目录 diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties index 3fec40d2eb..afffee61c5 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties @@ -42,5 +42,5 @@ generic.pipeline.metadata.incomplete=流水線元數據不完整，缺少{0} generic.pipeline.artifact.path.illegal=注意：path參數出錯，指定文件[{0}]屬於流水線倉庫[{1}]的製品 generic.chunked.artifact.broken=製品已損壞 generic.block.file.node.not-create=分塊文件信息節點[{0}]未創建 -generic.block.node.offset.not-found=請求頭 HEADER_OFFSET 和 HEADER_SEQUENCE 不可同時為空 +generic.block.node.head.not-found=請求頭 HEADER_OFFSET 和 HEADER_SEQUENCE 不可同時為空, HEADER_FILE_SIZE 不可為空 generic.block.list.path.is.folder=獲取分塊路徑是目錄 \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConfiguration.kt b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConfiguration.kt new file mode 100644 index 0000000000..a706646a1e --- /dev/null +++ b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConfiguration.kt @@ -0,0 +1,8 @@ +package com.tencent.com.bkrepo.generic + +import org.springframework.boot.SpringBootConfiguration +import org.springframework.boot.autoconfigure.EnableAutoConfiguration + +@SpringBootConfiguration +@EnableAutoConfiguration +class SeparateTestConfiguration \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConstants.kt b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConstants.kt new file mode 100644 index 0000000000..7751712861 --- /dev/null +++ b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConstants.kt @@ -0,0 +1,7 @@ +package com.tencent.com.bkrepo.generic + +const val UT_PROJECT_ID = "ut-project" +const val UT_REPO_NAME = "ut-repo" +const val UT_USER = "ut-user" +const val UT_SHA256 = "ut1000000000000000000000000" +const val BLOCK_SIZE = 10 * 1024 * 1024L // 10MB diff --git a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt new file mode 100644 index 0000000000..c56ef5c81e --- /dev/null +++ b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt @@ -0,0 +1,246 @@ +package com.tencent.com.bkrepo.generic.service + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.node.NodeDao +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.storage.StorageAutoConfiguration +import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo +import com.tencent.com.bkrepo.generic.BLOCK_SIZE +import com.tencent.com.bkrepo.generic.UT_PROJECT_ID +import com.tencent.com.bkrepo.generic.UT_REPO_NAME +import com.tencent.com.bkrepo.generic.UT_SHA256 +import com.tencent.com.bkrepo.generic.UT_USER +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.mockito.Mockito +import org.mockito.kotlin.any +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.EnableAutoConfiguration +import org.springframework.boot.autoconfigure.ImportAutoConfiguration +import org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration +import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.context.annotation.ComponentScan +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where +import org.springframework.test.context.TestPropertySource +import java.time.LocalDateTime +import kotlin.random.Random + + +@DataMongoTest +@EnableAutoConfiguration +@ComponentScan(basePackages = + ["com.tencent.bkrepo.generic.service", + "com.tencent.bkrepo.common.storage", + "com.tencent.bkrepo.common.metadata"]) +@ImportAutoConfiguration(StorageAutoConfiguration::class, TaskExecutionAutoConfiguration::class) +@TestPropertySource(locations = ["classpath:bootstrap-ut.properties"]) +class BlockNodeServiceTest { + + @MockBean + lateinit var nodeDao: NodeDao + + @Autowired + lateinit var blockNodeService: BlockNodeService + + @Autowired + lateinit var blockNodeDao: BlockNodeDao + + @Autowired + lateinit var storageService: StorageService + + private val storageCredentials = null + private val range = Range.full(Long.MAX_VALUE) + private lateinit var artifact: GenericArtifactInfo + private lateinit var createdDate: LocalDateTime + private lateinit var node: TNode + + @BeforeEach + fun beforeEach() { + val criteria = where(TBlockNode::repoName).isEqualTo(UT_REPO_NAME) + blockNodeDao.remove(Query(criteria)) + + artifact = GenericArtifactInfo(UT_PROJECT_ID, UT_REPO_NAME, "/newFile") + createdDate = LocalDateTime.now().minusSeconds(1) + node = TNode( + createdBy = UT_USER, + createdDate = createdDate, + lastModifiedBy = UT_USER, + lastModifiedDate = createdDate, + folder = false, + path = StringPool.ROOT, + name = "file", + fullPath = "/file", + size = BLOCK_SIZE, + projectId = UT_PROJECT_ID, + repoName = UT_REPO_NAME + ) + Mockito.`when`(nodeDao.findNode(any(), any(), any())) + .thenReturn(node) + } + + @DisplayName("测试BlockUpload") + @Test + fun testBlockUpload() { + setupBlocks() + val blocks = listBlocks(createdDate) + assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE) + } + + @DisplayName("测试BlockCompletion") + @Test + fun testBlockCompletion() { + setupBlocks() + val blocks = listBlocks(createdDate) + assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE) + + // 完成上传 + completeUpload(blocks) + + val completeBlocks = listBlocks(createdDate) + assertBlocks(completeBlocks, expectedSize = 2, blockSize = BLOCK_SIZE) + Assertions.assertEquals(0, completeBlocks[0].startPos) + Assertions.assertEquals(BLOCK_SIZE, completeBlocks[1].startPos) + } + + @DisplayName("测试BlockAbort") + @Test + fun testBlockAbort() { + setupBlocks() + val blocks = listBlocks(createdDate) + assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE) + + // 中止上传 + blockNodeService.deleteBlocks( + projectId = UT_PROJECT_ID, + repoName = UT_REPO_NAME, + fullPath = "/file" + ) + + val deleteBlocksQuery = deleteBlocksQuery("/file", UT_PROJECT_ID, UT_REPO_NAME, createdDate) + val afterBlocks = blockNodeDao.find(deleteBlocksQuery) + Assertions.assertEquals(2, afterBlocks.size) + afterBlocks.forEach { afterBlock -> + Assertions.assertNotNull(afterBlock.deleted) + } + } + + @DisplayName("测试获取范围内的分块") + @Test + fun testListRangeBlockNodes() { + val createdDate = LocalDateTime.now().minusSeconds(1).toString() + createBlockNode(10) + createBlockNode(20) + createBlockNode(30) + val range = Range(startPosition = 20, endPosition = 40, total = 100) + val blocks = blockNodeService.listBlocks( + range = range, + projectId = UT_PROJECT_ID, + repoName = UT_REPO_NAME, + fullPath = "/file", + createdDate = createdDate + ) + Assertions.assertEquals(2, blocks.size) + Assertions.assertEquals(20, blocks.first().startPos) + Assertions.assertEquals(30, blocks[1].startPos) + } + + private fun createBlockNode( + startPos: Long = 0, + fullPath: String = "/file", + sha256: String = "" + ): TBlockNode { + val blockNode = TBlockNode( + createdBy = UT_USER, + createdDate = LocalDateTime.now(), + nodeFullPath = fullPath, + startPos = startPos, + sha256 = sha256, + projectId = UT_PROJECT_ID, + repoName = UT_REPO_NAME, + size = 1 + ) + return blockNodeService.createBlock(blockNode, storageCredentials) + } + + private fun createTempArtifactFile(): ArtifactFile { + val data = Random.nextBytes(BLOCK_SIZE.toInt()) // 10MB + val tempFile = createTempFile() + tempFile.writeBytes(data) + return FileSystemArtifactFile(tempFile) + } + + private fun createAndStoreBlock(i: Int,fullPath: String = "/file") { + val blockNode = TBlockNode( + createdBy = UT_USER, + createdDate = LocalDateTime.now(), + nodeFullPath = fullPath, + startPos = i.toLong(), + sha256 = "$UT_SHA256$i", + projectId = UT_PROJECT_ID, + repoName = UT_REPO_NAME, + size = BLOCK_SIZE + ) + val artifactFile = createTempArtifactFile() + storageService.store(blockNode.sha256, artifactFile, storageCredentials) + blockNodeService.createBlock(blockNode, storageCredentials) + } + private fun listBlocks(createdDate: LocalDateTime, fullPath: String = "/file"): List { + return blockNodeService.listBlocks( + range = range, + projectId = UT_PROJECT_ID, + repoName = UT_REPO_NAME, + fullPath = fullPath, + createdDate = createdDate.toString() + ) + } + + private fun assertBlocks(blocks: List, expectedSize: Int, blockSize: Long) { + Assertions.assertEquals(expectedSize, blocks.size) + blocks.forEach { block -> + Assertions.assertEquals(blockSize, block.size) + Assertions.assertTrue(storageService.exist(block.sha256, storageCredentials)) + } + } + + private fun completeUpload(blocks: List) { + var offset = 0L + blocks.sortedBy { it.startPos }.forEach { blockInfo -> + val blockSize = blockInfo.size + val blockNode = blockInfo.copy( + startPos = offset, + endPos = offset + blockSize - 1 + ) + blockNodeService.updateBlock(blockNode) + offset += blockSize // 更新偏移量 + } + } + + private fun deleteBlocksQuery(fullPath: String, projectId: String, repoName: String, createdDate: LocalDateTime): Query { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::createdDate).gt(createdDate).lt(LocalDateTime.now()) + val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) + return query + } + + private fun setupBlocks() { + for (i in 0..1) { + createAndStoreBlock(i) + } + } +} From d65bc84b552f82d6f750cdcbaaec12a1f1e407cc Mon Sep 17 00:00:00 2001 From: zzdjx Date: Fri, 6 Dec 2024 17:37:44 +0800 Subject: [PATCH 04/28] =?UTF-8?q?fix:=20=E5=88=86=E5=9D=97=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E4=BC=98=E5=8C=96-=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/metadata/dao/blocknode/BlockNodeDao.kt | 7 ++++++- .../metadata/service/node/impl/NodeDeleteSupport.kt | 10 ++++------ .../com/bkrepo/generic/service/BlockNodeServiceTest.kt | 9 +++++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt index 915746395e..d926367645 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt @@ -44,6 +44,11 @@ class BlockNodeDao : HashShardingMongoDao(){ if (logger.isDebugEnabled) { logger.debug("Mongo Dao updateFirst: [$query], [$update]") } - return determineMongoTemplate().updateFirst(query, update, TBlockNode::class.java, determineCollectionName(query)) + return determineMongoTemplate() + .updateFirst( + query, + update, + TBlockNode::class.java, + determineCollectionName(query)) } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 95da4378cc..6407fdb38f 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -211,7 +211,6 @@ open class NodeDeleteSupport( if (deletedNum == 0L) { return NodeDeleteResult(deletedNum, deletedSize, deleteTime) } - if (decreaseVolume) { var deletedCriteria = criteria.and(TNode::deleted).isEqualTo(deleteTime) fullPaths?.let { @@ -227,11 +226,10 @@ open class NodeDeleteSupport( routerControllerClient.removeNodes(projectId, repoName, fullPath) } publishEvent(buildDeletedEvent(projectId, repoName, fullPath, operator)) - - nodeDao.find(Query(buildCriteria(projectId, repoName, fullPath, deleteTime))).first().let { - if (it.sha256 == FAKE_SHA256) { - nodeBaseService.blockNodeService.deleteBlocks(projectId, repoName, fullPath) - } + val blockCriteria = buildCriteria(projectId, repoName, fullPath, deleteTime) + val node = nodeDao.findOne(Query(blockCriteria)) + if (node?.sha256 == FAKE_SHA256) { + nodeBaseService.blockNodeService.deleteBlocks(projectId, repoName, fullPath) } } } catch (exception: DuplicateKeyException) { diff --git a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt index c56ef5c81e..35a65c5abd 100644 --- a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt +++ b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt @@ -208,7 +208,9 @@ class BlockNodeServiceTest { ) } - private fun assertBlocks(blocks: List, expectedSize: Int, blockSize: Long) { + private fun assertBlocks(blocks: List, + expectedSize: Int, + blockSize: Long) { Assertions.assertEquals(expectedSize, blocks.size) blocks.forEach { block -> Assertions.assertEquals(blockSize, block.size) @@ -229,7 +231,10 @@ class BlockNodeServiceTest { } } - private fun deleteBlocksQuery(fullPath: String, projectId: String, repoName: String, createdDate: LocalDateTime): Query { + private fun deleteBlocksQuery(fullPath: String, + projectId: String, + repoName: String, + createdDate: LocalDateTime): Query { val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) .and(TBlockNode::projectId.name).isEqualTo(projectId) .and(TBlockNode::repoName.name).isEqualTo(repoName) From c21d48836f26b05d531acba955f75d21e2a632f8 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Tue, 10 Dec 2024 15:04:16 +0800 Subject: [PATCH 05/28] =?UTF-8?q?fix:=20NodeAttribute=E6=8A=BD=E7=A6=BB?= =?UTF-8?q?=E5=85=AC=E5=85=B1=E6=94=BE=E5=88=B0metadata=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/metadata}/model/NodeAttribute.kt | 2 +- .../server/handler/NodeOperationsHandler.kt | 6 ++--- .../fs/server/service/FileOperationService.kt | 2 +- .../bkrepo/generic/model/NodeAttribute.kt | 22 ------------------- .../bkrepo/generic/service/UploadService.kt | 2 +- 5 files changed, 6 insertions(+), 28 deletions(-) rename src/backend/{fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server => common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata}/model/NodeAttribute.kt (90%) delete mode 100644 src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/model/NodeAttribute.kt diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/NodeAttribute.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/NodeAttribute.kt similarity index 90% rename from src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/NodeAttribute.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/NodeAttribute.kt index ea6d2001d3..2277288def 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/NodeAttribute.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/NodeAttribute.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.fs.server.model +package com.tencent.bkrepo.common.metadata.model data class NodeAttribute( // 用户id diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt index c278ad681c..04412756a4 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt @@ -32,15 +32,15 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.model.NodeAttribute +import com.tencent.bkrepo.common.metadata.model.NodeAttribute.Companion.DEFAULT_MODE +import com.tencent.bkrepo.common.metadata.model.NodeAttribute.Companion.NOBODY import com.tencent.bkrepo.common.metadata.service.fs.FsService import com.tencent.bkrepo.common.metadata.service.metadata.RMetadataService import com.tencent.bkrepo.common.metadata.service.repo.RRepositoryService import com.tencent.bkrepo.common.storage.core.overlay.OverlayRangeUtils import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder -import com.tencent.bkrepo.fs.server.model.NodeAttribute -import com.tencent.bkrepo.fs.server.model.NodeAttribute.Companion.DEFAULT_MODE -import com.tencent.bkrepo.fs.server.model.NodeAttribute.Companion.NOBODY import com.tencent.bkrepo.fs.server.request.ChangeAttributeRequest import com.tencent.bkrepo.fs.server.request.LinkRequest import com.tencent.bkrepo.fs.server.request.MoveRequest diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt index b590d7fd3f..9e715692bf 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt @@ -31,13 +31,13 @@ import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.model.NodeAttribute import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.fs.FsService import com.tencent.bkrepo.common.metadata.service.metadata.RMetadataService import com.tencent.bkrepo.fs.server.config.properties.StreamProperties import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder -import com.tencent.bkrepo.fs.server.model.NodeAttribute import com.tencent.bkrepo.fs.server.request.BlockRequest import com.tencent.bkrepo.fs.server.request.FlushRequest import com.tencent.bkrepo.fs.server.request.StreamRequest diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/model/NodeAttribute.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/model/NodeAttribute.kt deleted file mode 100644 index 803784e3c1..0000000000 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/model/NodeAttribute.kt +++ /dev/null @@ -1,22 +0,0 @@ -package com.tencent.bkrepo.generic.model - -data class NodeAttribute( - // 用户id - val uid: String, - // 组id - val gid: String, - // 文件权限，八进制 - val mode: Int? = DEFAULT_MODE, - // windows文件flag，十六进制 - val flags: Int? = null, - // 设备文件设备号 - val rdev: Int? = null, - // 文件类型 - val type: Int? = null -) { - companion object { - const val DEFAULT_MODE = 644 - const val NOBODY = "nobody" - } -} - diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 1e4367be18..81b45d90b2 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -48,6 +48,7 @@ import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.model.NodeAttribute import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.node.NodeService @@ -67,7 +68,6 @@ import com.tencent.bkrepo.generic.constant.GenericMessageCode import com.tencent.bkrepo.generic.constant.HEADER_EXPIRES import com.tencent.bkrepo.generic.constant.HEADER_FILE_SIZE import com.tencent.bkrepo.generic.constant.HEADER_OVERWRITE -import com.tencent.bkrepo.generic.model.NodeAttribute import com.tencent.bkrepo.generic.pojo.BlockInfo import com.tencent.bkrepo.generic.pojo.NewBlockInfo import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo From 318f65a3e9832e0abd70003df883cfe3b3479b6d Mon Sep 17 00:00:00 2001 From: zzdjx Date: Wed, 11 Dec 2024 15:16:08 +0800 Subject: [PATCH 06/28] =?UTF-8?q?fix:=20=E6=96=B0=E5=A2=9EGeneric=E4=B8=8B?= =?UTF-8?q?Block=E4=BA=8B=E4=BB=B6=E7=9B=91=E5=90=AC=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/node/impl/NodeDeleteSupport.kt | 5 --- .../listener/BuildDeletedEventListener.kt | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/listener/BuildDeletedEventListener.kt diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 6407fdb38f..689bff0006 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -226,11 +226,6 @@ open class NodeDeleteSupport( routerControllerClient.removeNodes(projectId, repoName, fullPath) } publishEvent(buildDeletedEvent(projectId, repoName, fullPath, operator)) - val blockCriteria = buildCriteria(projectId, repoName, fullPath, deleteTime) - val node = nodeDao.findOne(Query(blockCriteria)) - if (node?.sha256 == FAKE_SHA256) { - nodeBaseService.blockNodeService.deleteBlocks(projectId, repoName, fullPath) - } } } catch (exception: DuplicateKeyException) { logger.warn("Delete node[$resourceKey] by [$operator] error: [${exception.message}]") diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/listener/BuildDeletedEventListener.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/listener/BuildDeletedEventListener.kt new file mode 100644 index 0000000000..55954f303c --- /dev/null +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/listener/BuildDeletedEventListener.kt @@ -0,0 +1,44 @@ +package com.tencent.bkrepo.generic.listener + +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType +import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional +import org.springframework.context.event.EventListener +import org.springframework.scheduling.annotation.Async +import org.springframework.stereotype.Component + +@Component +@Conditional(SyncCondition::class) +class BuildDeletedEventListener( + private val nodeService: NodeService, + private val blockNodeService: BlockNodeService +) { + + @Async + @EventListener(ArtifactEvent::class) + fun handle(event: ArtifactEvent) { + if (event.type == EventType.NODE_DELETED) { + logger.info("accept artifact delete event: $event") + consumer(event) + } + } + + private fun consumer(event: ArtifactEvent) { + with(event) { + val node = nodeService.getDeletedNodeDetail(ArtifactInfo(projectId, repoName, resourceKey)).firstOrNull() + if (node?.sha256 == FAKE_SHA256 && !node.folder) { + blockNodeService.deleteBlocks(projectId, repoName, resourceKey) + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(BuildDeletedEventListener::class.java) + } +} \ No newline at end of file From ba35f7fade1b9ca727ede2d034cae0e5e39614a0 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Wed, 11 Dec 2024 15:20:55 +0800 Subject: [PATCH 07/28] =?UTF-8?q?fix:=20=E4=BB=A3=E7=A0=81=E8=A7=84?= =?UTF-8?q?=E8=8C=83=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/metadata/service/node/impl/NodeDeleteSupport.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 689bff0006..22f7f23f5c 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -32,7 +32,6 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.HumanReadable import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties -import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.model.TNode @@ -60,7 +59,7 @@ import java.time.LocalDateTime /** * 节点删除接口实现 */ -open class NodeDeleteSupport( +open class NodeDeleteSuspport( private val nodeBaseService: NodeBaseService ) : NodeDeleteOperation { From 89a79055d021b8ee0f206e31b712c11afa363d6f Mon Sep 17 00:00:00 2001 From: zzdjx Date: Wed, 11 Dec 2024 15:30:17 +0800 Subject: [PATCH 08/28] =?UTF-8?q?fix:=20=E9=94=99=E8=AF=AF=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/metadata/service/node/impl/NodeDeleteSupport.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 22f7f23f5c..43d72b803a 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -59,7 +59,7 @@ import java.time.LocalDateTime /** * 节点删除接口实现 */ -open class NodeDeleteSuspport( +open class NodeDeleteSupport( private val nodeBaseService: NodeBaseService ) : NodeDeleteOperation { From b71c81a2d4f8adc5f0d703bbe6eecb77881d4ca6 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Thu, 19 Dec 2024 16:52:48 +0800 Subject: [PATCH 09/28] =?UTF-8?q?feat:=20=E9=80=BB=E8=BE=91=E9=87=8D?= =?UTF-8?q?=E6=9E=84=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/apidoc-user/generic/SeparateBlock.md | 312 ++++++++++-------- docs/apidoc/generic/SeparateBlock.md | 312 ++++++++++-------- .../repository/core/ArtifactRepository.kt | 7 - .../repository/local/LocalRepository.kt | 31 -- .../metadata/dao/blocknode/BlockNodeDao.kt | 20 +- .../common/metadata/model/TBlockNode.kt | 3 +- .../service/blocknode/BlockNodeService.kt | 17 +- .../blocknode/impl/BlockNodeServiceImpl.kt | 41 ++- .../metadata/util/BlockNodeQueryHelper.kt | 26 ++ .../common/metadata/util/NodeDeleteHelper.kt | 4 +- .../bkrepo/fs/server/constant/Constants.kt | 2 + .../generic/constant/GenericMessageCode.kt | 4 +- .../bkrepo/generic/pojo/NewBlockInfo.kt | 6 +- .../artifact/GenericLocalRepository.kt | 72 ++-- .../controller/SeparateBlockController.kt | 55 +-- .../bkrepo/generic/service/UploadService.kt | 172 +++++----- .../resources/i18n/messages_en.properties | 4 +- .../resources/i18n/messages_zh_CN.properties | 4 +- .../resources/i18n/messages_zh_TW.properties | 4 +- .../bkrepo/generic/SeparateTestConstants.kt | 1 + .../generic/service/BlockNodeServiceTest.kt | 39 ++- 21 files changed, 618 insertions(+), 518 deletions(-) diff --git a/docs/apidoc-user/generic/SeparateBlock.md b/docs/apidoc-user/generic/SeparateBlock.md index 431de2b9c1..264f5cf39d 100644 --- a/docs/apidoc-user/generic/SeparateBlock.md +++ b/docs/apidoc-user/generic/SeparateBlock.md @@ -1,205 +1,239 @@ -# Generic通用制品仓库分块文件操作 +# Generic 通用制品仓库分块文件操作 -[toc] +[TOC] ## 初始化分块上传 -- API: POST /generic/separate/block/{project}/{repo}/{path} -- API 名称: start_block_upload -- 功能说明： - - 中文：初始化分块上传 - - English：start block upload +- **API**: `POST /generic/separate/block/{project}/{repo}/{path}` +- **API 名称**: `start_block_upload` +- **功能说明**： + - 中文：初始化分块上传 + - English: start block upload -- 请求体 -此接口请求体为空 +- **请求体** -- 请求字段说明 + 此接口请求体为空。 - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| +- **请求字段说明** -- 请求头 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| -- |X-BKREPO-SIZE|string|是|0L|文件大小|上传文件的总大小| - |X-BKREPO-MD5|string|否|无|文件md5|file md5| - |X-BKREPO-OVERWRITE|boolean|否|false|是否覆盖已存在文件|overwrite exist file| +- **请求头** -- 响应体 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------- | ------- | -------- | ------ | ------------------ | --------------------- | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | overwrite exist file | -``` json +- **响应体** + + ```json { - "code" : 0, - "message" : null, - "data" : null, - "traceId" : null + "code": 0, + "message": null, + "data": { + "uploadId": "8be31384f82a45b0aafb6c6add29e94f/VERSION", + "expireSeconds": 43200 + }, + "traceId": null } -``` + ``` + +--- ## 上传分块文件 -- API: PUT /generic/separate/{project}/{repo}/{path} -- API 名称: block_upload -- 功能说明： - - 中文：分块上传通用制品文件 - - English：upload generic artifact file block +- **API**: `PUT /generic/{project}/{repo}/{path}` +- **API 名称**: `block_upload` +- **功能说明**： + - 中文：分块上传通用制品文件 + - English: upload generic artifact file block + +- **请求体** -- 请求体 -[文件流] + [文件流] -- 请求字段说明 +- **请求字段说明** - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | -- 请求头 +- **请求头** - |字段|类型| 是否必须 |默认值| 说明 | Description | - |---|---|-|---|------|---------| - |X-BKREPO-SEQUENCE|int|是|无| 分块序号(从1开始), SEQUENCE 和 OFFSET 二者不可同时为空 |block sequence(start from 1) | -- |X-BKREPO-OFFSET|int|是|无| 分块偏移量,SEQUENCE 和 OFFSET 二者不可同时为空| block offset(start from 0)| - |X-BKREPO-SHA256|string|否|无| 文件sha256|file sha256| - |X-BKREPO-MD5|string|否|无| 文件md5| file md5| + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------- | ------- | -------- | ------ | -------------------------------------------------------------------- | ------------------------------ | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + | X-BKREPO-SEQUENCE | int | 是 | 无 | 分块序号（从 1 开始），`SEQUENCE` 和 `OFFSET` 二者不可同时为空 | block sequence (start from 1) | + | X-BKREPO-OFFSET | int | 是 | 无 | 分块偏移量，`SEQUENCE` 和 `OFFSET` 二者不可同时为空 | block offset (start from 0) | + | X-BKREPO-SHA256 | string | 否 | 无 | 文件 SHA256 | file sha256 | + | X-BKREPO-MD5 | string | 否 | 无 | 文件 MD5 | file md5 | -- 响应体 +- **响应体** - ``` json + ```json { - "code" : 0, - "message" : null, - "data" : null, - "traceId" : null + "code": 0, + "message": null, + "data": null, + "traceId": null } ``` +--- + ## 完成分块上传 -- API: PUT /generic/separate/block/{project}/{repo}/{path} -- API 名称: complete_block_upload -- 功能说明： - - 中文：完成化分块上传 - - English：complete block upload +- **API**: `PUT /generic/separate/block/{project}/{repo}/{path}` +- **API 名称**: `complete_block_upload` +- **功能说明**： + - 中文：完成分块上传 + - English: complete block upload + +- **请求体** + + 此接口请求体为空。 -- 请求体 -此接口请求体为空 +- **请求字段说明** -- 请求字段说明 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| +- **请求头** -- 请求头 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------ | ------- | -------- | ------ | ------------------ | ------------------------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + | X-BKREPO-SIZE | string | 是 | 0L | 文件大小 | total size of the uploaded file | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | overwrite exist file | -- 响应体 +- **响应体** - ``` json + ```json { - "code" : 0, - "message" : null, - "data" : null, - "traceId" : "" + "code": 0, + "message": null, + "data": null, + "traceId": "" } ``` -## 终止(取消)分块上传 +--- -- API: DELETE /generic/separate/block/{project}/{repo}/{path} -- API 名称: abort_block_upload -- 功能说明： - - 中文：终止(取消)分块上传 - - English：abort block upload +## 终止（取消）分块上传 -- 请求体 -此接口请求体为空 +- **API**: `DELETE /generic/separate/block/{project}/{repo}/{path}` +- **API 名称**: `abort_block_upload` +- **功能说明**： + - 中文：终止（取消）分块上传 + - English: abort block upload -- 请求字段说明 +- **请求体** - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| + 此接口请求体为空。 -- 请求头 +- **请求字段说明** -- 响应体 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | - ``` json +- **请求头** + + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------ | ------ | -------- | ------ | ----------- | ----------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + +- **响应体** + + ```json { - "code" : 0, - "message" : null, - "data" : null, - "traceId" : null + "code": 0, + "message": null, + "data": null, + "traceId": null } ``` +--- + ## 查询已上传的分块列表 -- API: GET /generic/separate/block/{project}/{repo}/{path} -- API 名称: list_upload_block -- 功能说明： - - 中文：查询已上传的分块列表 - - English：list upload block +- **API**: `GET /generic/separate/block/{project}/{repo}/{path}` +- **API 名称**: `list_upload_block` +- **功能说明**： + - 中文：查询已上传的分块列表 + - English: list uploaded blocks + +- **请求体** -- 请求体 - 此接口请求体为空 + 此接口请求体为空。 -- 请求字段说明 +- **请求字段说明** - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | -- 请求头 +- **请求头** - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |Authorization|string|否|无|Basic Auth认证头，Basic base64(username:password)|Basic Auth header| + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------ | ------ | -------- | ------ | ------------------------------------------------------------------------- | -------------------- | + | Authorization | string | 否 | 无 | Basic Auth 认证头，格式：`Basic base64(username:password)` | Basic Auth header | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | -- 响应体 +- **响应体** - ``` json + ```json { - "code" : 0, - "message" : null, - "data" : [ { - "size" : 10240, - "sha256" : "d17f25ecfbcc7857f7bebea469308be0b2580943e96d13a3ad98a13675c4bfc2", - "startPos" : xxx - }, { - "size" : 10240, - "sha256" : "cc399d73903f06ee694032ab0538f05634ff7e1ce5e8e50ac330a871484f34cf", - "startPos" : xxx - } ], - "traceId" : null + "code": 0, + "message": null, + "data": [ + { + "size": 10240, + "sha256": "d17f25ecfbcc7857f7bebea469308be0b2580943e96d13a3ad98a13675c4bfc2", + "startPos": 0, + "version": "1.0" + }, + { + "size": 10240, + "sha256": "cc399d73903f06ee694032ab0538f05634ff7e1ce5e8e50ac330a871484f34cf", + "startPos": 10240, + "version": "1.0" + } + ], + "traceId": null } ``` -- 响应字段说明 +- **响应字段说明** + + | 字段 | 类型 | 说明 | Description | + | -------- | ------ | ------------------------------------- | -------------------------- | + | code | int | 错误编码。0 表示成功，>0 表示失败错误 | 0: success, other: failure | + | message | string | 错误消息 | the failure message | + | data | list | 分块列表 | block list | + | traceId | string | 请求跟踪 id | trace id | - |字段|类型|说明|Description| - |---|---|---|---| - |code|bool|错误编码。 0表示success，>0表示失败错误|0:success, other: failure| - |message|string|错误消息|the failure message| - |data|list|分块列表|block list| - |traceId|string|请求跟踪id|trace id| +- **分块信息字段说明** -- 分块信息字段说明 + | 字段 | 类型 | 说明 | Description | + | --------- | ------ | ------------ | -------------------------- | + | size | long | 分块大小 | block size | + | sha256 | string | 分块 SHA256 | block sha256 checksum | + | startPos | long | 分块起始位置 | block start position | + | version | string | 分块版本信息 | block version | - |字段| 类型 | 说明 |Description| - |---|-----|----|---| - |size| long |分块大小|block size| - |sha256| string| 分块sha256|block sha256 checksum| - |startPos| long| 分块起始位置|block sequence| \ No newline at end of file +--- \ No newline at end of file diff --git a/docs/apidoc/generic/SeparateBlock.md b/docs/apidoc/generic/SeparateBlock.md index 431de2b9c1..264f5cf39d 100644 --- a/docs/apidoc/generic/SeparateBlock.md +++ b/docs/apidoc/generic/SeparateBlock.md @@ -1,205 +1,239 @@ -# Generic通用制品仓库分块文件操作 +# Generic 通用制品仓库分块文件操作 -[toc] +[TOC] ## 初始化分块上传 -- API: POST /generic/separate/block/{project}/{repo}/{path} -- API 名称: start_block_upload -- 功能说明： - - 中文：初始化分块上传 - - English：start block upload +- **API**: `POST /generic/separate/block/{project}/{repo}/{path}` +- **API 名称**: `start_block_upload` +- **功能说明**： + - 中文：初始化分块上传 + - English: start block upload -- 请求体 -此接口请求体为空 +- **请求体** -- 请求字段说明 + 此接口请求体为空。 - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| +- **请求字段说明** -- 请求头 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| -- |X-BKREPO-SIZE|string|是|0L|文件大小|上传文件的总大小| - |X-BKREPO-MD5|string|否|无|文件md5|file md5| - |X-BKREPO-OVERWRITE|boolean|否|false|是否覆盖已存在文件|overwrite exist file| +- **请求头** -- 响应体 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------- | ------- | -------- | ------ | ------------------ | --------------------- | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | overwrite exist file | -``` json +- **响应体** + + ```json { - "code" : 0, - "message" : null, - "data" : null, - "traceId" : null + "code": 0, + "message": null, + "data": { + "uploadId": "8be31384f82a45b0aafb6c6add29e94f/VERSION", + "expireSeconds": 43200 + }, + "traceId": null } -``` + ``` + +--- ## 上传分块文件 -- API: PUT /generic/separate/{project}/{repo}/{path} -- API 名称: block_upload -- 功能说明： - - 中文：分块上传通用制品文件 - - English：upload generic artifact file block +- **API**: `PUT /generic/{project}/{repo}/{path}` +- **API 名称**: `block_upload` +- **功能说明**： + - 中文：分块上传通用制品文件 + - English: upload generic artifact file block + +- **请求体** -- 请求体 -[文件流] + [文件流] -- 请求字段说明 +- **请求字段说明** - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | -- 请求头 +- **请求头** - |字段|类型| 是否必须 |默认值| 说明 | Description | - |---|---|-|---|------|---------| - |X-BKREPO-SEQUENCE|int|是|无| 分块序号(从1开始), SEQUENCE 和 OFFSET 二者不可同时为空 |block sequence(start from 1) | -- |X-BKREPO-OFFSET|int|是|无| 分块偏移量,SEQUENCE 和 OFFSET 二者不可同时为空| block offset(start from 0)| - |X-BKREPO-SHA256|string|否|无| 文件sha256|file sha256| - |X-BKREPO-MD5|string|否|无| 文件md5| file md5| + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------- | ------- | -------- | ------ | -------------------------------------------------------------------- | ------------------------------ | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + | X-BKREPO-SEQUENCE | int | 是 | 无 | 分块序号（从 1 开始），`SEQUENCE` 和 `OFFSET` 二者不可同时为空 | block sequence (start from 1) | + | X-BKREPO-OFFSET | int | 是 | 无 | 分块偏移量，`SEQUENCE` 和 `OFFSET` 二者不可同时为空 | block offset (start from 0) | + | X-BKREPO-SHA256 | string | 否 | 无 | 文件 SHA256 | file sha256 | + | X-BKREPO-MD5 | string | 否 | 无 | 文件 MD5 | file md5 | -- 响应体 +- **响应体** - ``` json + ```json { - "code" : 0, - "message" : null, - "data" : null, - "traceId" : null + "code": 0, + "message": null, + "data": null, + "traceId": null } ``` +--- + ## 完成分块上传 -- API: PUT /generic/separate/block/{project}/{repo}/{path} -- API 名称: complete_block_upload -- 功能说明： - - 中文：完成化分块上传 - - English：complete block upload +- **API**: `PUT /generic/separate/block/{project}/{repo}/{path}` +- **API 名称**: `complete_block_upload` +- **功能说明**： + - 中文：完成分块上传 + - English: complete block upload + +- **请求体** + + 此接口请求体为空。 -- 请求体 -此接口请求体为空 +- **请求字段说明** -- 请求字段说明 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| +- **请求头** -- 请求头 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------ | ------- | -------- | ------ | ------------------ | ------------------------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + | X-BKREPO-SIZE | string | 是 | 0L | 文件大小 | total size of the uploaded file | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | overwrite exist file | -- 响应体 +- **响应体** - ``` json + ```json { - "code" : 0, - "message" : null, - "data" : null, - "traceId" : "" + "code": 0, + "message": null, + "data": null, + "traceId": "" } ``` -## 终止(取消)分块上传 +--- -- API: DELETE /generic/separate/block/{project}/{repo}/{path} -- API 名称: abort_block_upload -- 功能说明： - - 中文：终止(取消)分块上传 - - English：abort block upload +## 终止（取消）分块上传 -- 请求体 -此接口请求体为空 +- **API**: `DELETE /generic/separate/block/{project}/{repo}/{path}` +- **API 名称**: `abort_block_upload` +- **功能说明**： + - 中文：终止（取消）分块上传 + - English: abort block upload -- 请求字段说明 +- **请求体** - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| + 此接口请求体为空。 -- 请求头 +- **请求字段说明** -- 响应体 + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | - ``` json +- **请求头** + + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------ | ------ | -------- | ------ | ----------- | ----------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + +- **响应体** + + ```json { - "code" : 0, - "message" : null, - "data" : null, - "traceId" : null + "code": 0, + "message": null, + "data": null, + "traceId": null } ``` +--- + ## 查询已上传的分块列表 -- API: GET /generic/separate/block/{project}/{repo}/{path} -- API 名称: list_upload_block -- 功能说明： - - 中文：查询已上传的分块列表 - - English：list upload block +- **API**: `GET /generic/separate/block/{project}/{repo}/{path}` +- **API 名称**: `list_upload_block` +- **功能说明**： + - 中文：查询已上传的分块列表 + - English: list uploaded blocks + +- **请求体** -- 请求体 - 此接口请求体为空 + 此接口请求体为空。 -- 请求字段说明 +- **请求字段说明** - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |project|string|是|无|项目名称|project name| - |repo|string|是|无|仓库名称|repo name| - |path|string|是|无|完整路径|full path| + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------- | ------ | -------- | ------ | -------- | ------------ | + | project | string | 是 | 无 | 项目名称 | project name | + | repo | string | 是 | 无 | 仓库名称 | repo name | + | path | string | 是 | 无 | 完整路径 | full path | -- 请求头 +- **请求头** - |字段|类型|是否必须|默认值|说明|Description| - |---|---|---|---|---|---| - |Authorization|string|否|无|Basic Auth认证头，Basic base64(username:password)|Basic Auth header| + | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | + | ------------------ | ------ | -------- | ------ | ------------------------------------------------------------------------- | -------------------- | + | Authorization | string | 否 | 无 | Basic Auth 认证头，格式：`Basic base64(username:password)` | Basic Auth header | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | -- 响应体 +- **响应体** - ``` json + ```json { - "code" : 0, - "message" : null, - "data" : [ { - "size" : 10240, - "sha256" : "d17f25ecfbcc7857f7bebea469308be0b2580943e96d13a3ad98a13675c4bfc2", - "startPos" : xxx - }, { - "size" : 10240, - "sha256" : "cc399d73903f06ee694032ab0538f05634ff7e1ce5e8e50ac330a871484f34cf", - "startPos" : xxx - } ], - "traceId" : null + "code": 0, + "message": null, + "data": [ + { + "size": 10240, + "sha256": "d17f25ecfbcc7857f7bebea469308be0b2580943e96d13a3ad98a13675c4bfc2", + "startPos": 0, + "version": "1.0" + }, + { + "size": 10240, + "sha256": "cc399d73903f06ee694032ab0538f05634ff7e1ce5e8e50ac330a871484f34cf", + "startPos": 10240, + "version": "1.0" + } + ], + "traceId": null } ``` -- 响应字段说明 +- **响应字段说明** + + | 字段 | 类型 | 说明 | Description | + | -------- | ------ | ------------------------------------- | -------------------------- | + | code | int | 错误编码。0 表示成功，>0 表示失败错误 | 0: success, other: failure | + | message | string | 错误消息 | the failure message | + | data | list | 分块列表 | block list | + | traceId | string | 请求跟踪 id | trace id | - |字段|类型|说明|Description| - |---|---|---|---| - |code|bool|错误编码。 0表示success，>0表示失败错误|0:success, other: failure| - |message|string|错误消息|the failure message| - |data|list|分块列表|block list| - |traceId|string|请求跟踪id|trace id| +- **分块信息字段说明** -- 分块信息字段说明 + | 字段 | 类型 | 说明 | Description | + | --------- | ------ | ------------ | -------------------------- | + | size | long | 分块大小 | block size | + | sha256 | string | 分块 SHA256 | block sha256 checksum | + | startPos | long | 分块起始位置 | block start position | + | version | string | 分块版本信息 | block version | - |字段| 类型 | 说明 |Description| - |---|-----|----|---| - |size| long |分块大小|block size| - |sha256| string| 分块sha256|block sha256 checksum| - |startPos| long| 分块起始位置|block sequence| \ No newline at end of file +--- \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt index aef9425d14..3a9a34505b 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt @@ -50,13 +50,6 @@ interface ArtifactRepository { */ fun upload(context: ArtifactUploadContext) - /** - * 分块上传构件 - * - * @param context 构件上传上下文 - */ - fun newUpload(context: ArtifactUploadContext) {} - /** * 下载构件 * diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/local/LocalRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/local/LocalRepository.kt index db93d1536b..b6a610bd7a 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/local/LocalRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/local/LocalRepository.kt @@ -31,7 +31,6 @@ package com.tencent.bkrepo.common.artifact.repository.local -import com.tencent.bkrepo.common.api.exception.MethodNotAllowedException import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext @@ -106,34 +105,4 @@ abstract class LocalRepository : AbstractArtifactRepository() { operator = context.userId ) } - - /** - * 分块上传 - */ - override fun newUpload(context: ArtifactUploadContext) { - try { - this.onNewUploadBefore(context) - this.onNewUpload(context) - this.onUploadSuccess(context) - }catch (exception: RuntimeException){ - this.onUploadFailed(context, exception) - }finally { - this.onUploadFinished(context) - } - } - - /** - * 分块上传前回调 - */ - open fun onNewUploadBefore(context: ArtifactUploadContext) { - artifactMetrics.uploadingCount.incrementAndGet() - } - - - /** - * 分块上传构件 - */ - open fun onNewUpload(context: ArtifactUploadContext) { - throw MethodNotAllowedException() - } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt index d926367645..150ab4eddd 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt @@ -40,15 +40,15 @@ import org.springframework.stereotype.Repository @Conditional(SyncCondition::class) class BlockNodeDao : HashShardingMongoDao(){ - fun updateBlock(query: Query, update: Update): UpdateResult { - if (logger.isDebugEnabled) { - logger.debug("Mongo Dao updateFirst: [$query], [$update]") - } - return determineMongoTemplate() - .updateFirst( - query, - update, - TBlockNode::class.java, - determineCollectionName(query)) + fun updateBlock(query: Query, update: Update): UpdateResult { + if (logger.isDebugEnabled) { + logger.debug("Mongo Dao updateFirst: [$query], [$update]") } + return determineMongoTemplate() + .updateFirst( + query, + update, + TBlockNode::class.java, + determineCollectionName(query)) + } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt index 4ec14a7c41..e3c6239f12 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt @@ -55,7 +55,8 @@ data class TBlockNode( val repoName: String, val size: Long, val endPos: Long = startPos + size - 1, - var deleted: LocalDateTime? = null + var deleted: LocalDateTime? = null, + val version: String? = null ) { companion object { const val BLOCK_IDX = "start_pos_idx" diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index b7bdbcdb34..061a3c7a6f 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -47,6 +47,17 @@ interface BlockNodeService { createdDate: String ): List + /** + * 查询出当前版本内的分块 + */ + fun listBlocksInVersion( + projectId: String, + repoName: String, + fullPath: String, + createdDate: String? = null, + version: String + ): List + /** * 创建分块 * */ @@ -60,11 +71,14 @@ interface BlockNodeService { * */ fun updateBlock( blockNode: TBlockNode, + startPos: Long, + endPos: Long, ) /** * 删除旧分块，即删除非指定的nodeCurrentSha256的分块。 * 如果未指定nodeCurrentSha256，则删除节点所有分块 + * 如果指定version，则删除该版本对应的分块，未指定则删除所有分块 * @param projectId 项目id * @param repoName 仓库名 * @param fullPath 文件路径 @@ -72,7 +86,8 @@ interface BlockNodeService { fun deleteBlocks( projectId: String, repoName: String, - fullPath: String + fullPath: String, + version: String? = null ) /** diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index 8b62839c14..3f3f47cca5 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -69,18 +69,21 @@ class BlockNodeServiceImpl( } } - override fun updateBlock(blockNode: TBlockNode) { + override fun updateBlock( + blockNode: TBlockNode, + startPos: Long, + endPos: Long, + ) { with(blockNode) { - val criteria = Criteria.where(TBlockNode::id.name).`is`(id) - .and(TBlockNode::nodeFullPath.name).`is`(nodeFullPath) - .and(TBlockNode::projectId.name).`is`(projectId) - .and(TBlockNode::repoName.name).`is`(repoName) - val update = Update() - .set(TBlockNode::startPos.name, startPos) - .set(TBlockNode::endPos.name, endPos) + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, nodeFullPath,false) + criteria.and(TBlockNode::id).isEqualTo(id) + .and(TBlockNode::version).isEqualTo(version) + val update = BlockNodeQueryHelper.updateVersionBlocks(startPos, endPos) blockNodeDao.updateBlock(Query(criteria), update) logger.info("Update block node[$projectId/$repoName/$nodeFullPath-$startPos], sha256[$sha256] success.") } + + } override fun listBlocks( @@ -94,15 +97,33 @@ class BlockNodeServiceImpl( return blockNodeDao.find(query) } + override fun listBlocksInVersion( + projectId: String, + repoName: String, + fullPath: String, + createdDate: String?, + version: String + ): List { + val query = + BlockNodeQueryHelper.listQueryInVersion(projectId, repoName, fullPath, createdDate, version) + return blockNodeDao.find(query) + } + override fun deleteBlocks( projectId: String, repoName: String, - fullPath: String + fullPath: String, + version: String? ) { val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + if (version != null) { + criteria.and(TBlockNode::version.name).`is`(version) + }else{ + criteria.and(TBlockNode::version.name).`is`(null) + } val update = BlockNodeQueryHelper.deleteUpdate() blockNodeDao.updateMulti(Query(criteria), update) - logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") + logger.info("Delete node blocks[$projectId/$repoName$fullPath] success. Version: $version") } override fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt index 396e6b14e5..ba66f01dbb 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt @@ -63,6 +63,25 @@ object BlockNodeQueryHelper { return query } + fun listQueryInVersion( + projectId: String, + repoName: String, + fullPath: String, + createdDate: String?, + version: String, + ):Query { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId).isEqualTo(projectId) + .and(TBlockNode::repoName).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + .and(TBlockNode::version).isEqualTo(version) + createdDate?.let { + criteria.and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) + } + val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) + return query + } + fun fullPathCriteria(projectId: String, repoName: String, fullPath: String, deep: Boolean): Criteria { val criteria = if (deep) { where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") @@ -100,4 +119,11 @@ object BlockNodeQueryHelper { return Update().set(TBlockNode::deleted.name, null) } + fun updateVersionBlocks(startPos: Long, endPos: Long): Update { + return Update().set(TBlockNode::version.name, null) + .set(TBlockNode::createdDate.name, LocalDateTime.now()) + .set(TBlockNode::startPos.name, startPos) + .set(TBlockNode::endPos.name, endPos) + } + } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt index 73fa78fca9..7230b47b59 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/NodeDeleteHelper.kt @@ -6,21 +6,19 @@ import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.and import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where -import java.time.LocalDateTime object NodeDeleteHelper { fun buildCriteria( projectId: String, repoName: String, fullPath: String, - deletedDate: LocalDateTime?= null ): Criteria { val normalizedFullPath = PathUtils.normalizeFullPath(fullPath) val normalizedPath = PathUtils.toPath(normalizedFullPath) val escapedPath = PathUtils.escapeRegex(normalizedPath) val criteria = where(TNode::projectId).isEqualTo(projectId) .and(TNode::repoName).isEqualTo(repoName) - .and(TNode::deleted).isEqualTo(deletedDate) + .and(TNode::deleted).isEqualTo(null) .orOperator( where(TNode::fullPath).regex("^$escapedPath"), where(TNode::fullPath).isEqualTo(normalizedFullPath) diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt index d2f9c3b457..bea78e9921 100644 --- a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt +++ b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt @@ -33,3 +33,5 @@ const val JWT_CLAIMS_REPOSITORY = "repository" const val JWT_CLAIMS_PERMIT = "permit" const val FS_ATTR_KEY = "fs:attr" +const val VERSION_KEY = "VERSION" + diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt index d6fb717047..8be21f3dc5 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt @@ -49,9 +49,9 @@ enum class GenericMessageCode(private val businessCode: Int, private val key: St PIPELINE_REPO_MANUAL_UPLOAD_NOT_ALLOWED(10, "generic.pipeline-repo.manual-upload.not-allowed"), PIPELINE_ARTIFACT_PATH_ILLEGAL(11, "generic.pipeline.artifact.path.illegal"), CHUNKED_ARTIFACT_BROKEN(12, "generic.chunked.artifact.broken"), - BLOCK_FILE_NODE_NOT_CREATE(13, "generic.block.file.node.not-create"), + BLOCK_FILE_NODE_CREATE_FAIL(13, "generic.block.file.node.create.fail"), BLOCK_HEAD_NOT_FOUND(14, "generic.block.node.head.not-found"), - BLOCK_LIST_PATH_IS_FOLDER(15, "generic.block.list.path.is.folder") + BLOCK_UPDATE_LIST_IS_NULL(15, "generic.block.update.list.is.null") ; override fun getBusinessCode() = businessCode diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt index b31b5371bc..3f15422437 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt @@ -5,11 +5,13 @@ import io.swagger.annotations.ApiModelProperty @ApiModel("新分块信息") -data class NewBlockInfo( +data class SeparateBlockInfo( @ApiModelProperty("分块大小") val size: Long, @ApiModelProperty("分块sha256") val sha256: String, @ApiModelProperty("分块起始位置") - val startPos: Long + val startPos: Long, + @ApiModelProperty("分块版本") + val version: String? ) \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index cb1eb1fc1b..e2072f97f0 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -87,6 +87,7 @@ import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.common.storage.message.StorageErrorException import com.tencent.bkrepo.common.storage.pojo.FileInfo +import com.tencent.bkrepo.fs.server.constant.VERSION_KEY import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext import com.tencent.bkrepo.generic.constant.BKREPO_META import com.tencent.bkrepo.generic.constant.BKREPO_META_PREFIX @@ -177,39 +178,35 @@ class GenericLocalRepository( } } - override fun onNewUploadBefore(context: ArtifactUploadContext){ - // 上传前校验 - super.onUploadBefore(context) - // 检查BlockNodeBase是否存在 - checkBlockBaseNode(context.artifactInfo) - // 通用上传前检查 - baseUploadBefore(context) - } - override fun onUpload(context: ArtifactUploadContext) { val uploadId = context.request.getHeader(HEADER_UPLOAD_ID) - val sequence = context.request.getHeader(HEADER_SEQUENCE)?.toInt() - val uploadType = HeaderUtils.getHeader(HEADER_UPLOAD_TYPE) - if (isBlockUpload(uploadId, sequence)) { - this.blockUpload(uploadId, sequence!!, context) - context.response.contentType = MediaTypes.APPLICATION_JSON - context.response.writer.println(ResponseBuilder.success().toJsonString()) - } else if (isChunkedUpload(uploadType)) { - chunkedUpload(context) - } else { - val nodeDetail = storageManager.storeArtifactFile( - buildNodeCreateRequest(context), - context.getArtifactFile(), - context.storageCredentials - ) - context.response.contentType = MediaTypes.APPLICATION_JSON - context.response.addHeader(X_CHECKSUM_MD5, context.getArtifactMd5()) - context.response.addHeader(X_CHECKSUM_SHA256, context.getArtifactSha256()) - context.response.writer.println(ResponseBuilder.success(nodeDetail).toJsonString()) + if (isSeparateUpload(uploadId)){ + onSeparateUpload(context, uploadId) + } + else{ + val sequence = context.request.getHeader(HEADER_SEQUENCE)?.toInt() + val uploadType = HeaderUtils.getHeader(HEADER_UPLOAD_TYPE) + if (isBlockUpload(uploadId, sequence)) { + this.blockUpload(uploadId, sequence!!, context) + context.response.contentType = MediaTypes.APPLICATION_JSON + context.response.writer.println(ResponseBuilder.success().toJsonString()) + } else if (isChunkedUpload(uploadType)) { + chunkedUpload(context) + } else { + val nodeDetail = storageManager.storeArtifactFile( + buildNodeCreateRequest(context), + context.getArtifactFile(), + context.storageCredentials + ) + context.response.contentType = MediaTypes.APPLICATION_JSON + context.response.addHeader(X_CHECKSUM_MD5, context.getArtifactMd5()) + context.response.addHeader(X_CHECKSUM_SHA256, context.getArtifactSha256()) + context.response.writer.println(ResponseBuilder.success(nodeDetail).toJsonString()) + } } } - override fun onNewUpload(context: ArtifactUploadContext) { + private fun onSeparateUpload(context: ArtifactUploadContext, uploadId: String) { with(context) { val bArtifactFile = getArtifactFile() @@ -222,11 +219,13 @@ class GenericLocalRepository( createdBy = userId, createdDate = LocalDateTime.now(), nodeFullPath = artifactInfo.getArtifactFullPath(), - startPos = offset ?: sequence ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND), + startPos = offset ?: sequence + ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND), sha256 = sha256, projectId = projectId, repoName = repoName, - size = bArtifactFile.getSize() + size = bArtifactFile.getSize(), + version = uploadId ) val stored = storageService.store(sha256, bArtifactFile, storageCredentials) @@ -248,6 +247,10 @@ class GenericLocalRepository( } } + private fun isSeparateUpload(uploadId: String): Boolean { + return uploadId.substringAfter("/") == VERSION_KEY + } + override fun onUploadSuccess(context: ArtifactUploadContext) { super.onUploadSuccess(context) if (HttpContextHolder.getRequestOrNull()?.getParameter(PARAM_REPLICATE).toBoolean()) { @@ -308,6 +311,7 @@ class GenericLocalRepository( val overwrite = HeaderUtils.getBooleanHeader(HEADER_OVERWRITE) val uploadId = HeaderUtils.getHeader(HEADER_UPLOAD_ID) val sequence = HeaderUtils.getHeader(HEADER_SEQUENCE)?.toInt() + ?: HeaderUtils.getHeader(HEADER_OFFSET)?.toInt() val uploadType = HeaderUtils.getHeader(HEADER_UPLOAD_TYPE) if (!overwrite && !isBlockUpload(uploadId, sequence) && !isChunkedUpload(uploadType)) { with(context.artifactInfo) { @@ -318,14 +322,6 @@ class GenericLocalRepository( } } - private fun checkBlockBaseNode(artifactInfo: ArtifactInfo) { - nodeService.getNodeDetail(artifactInfo) - ?: run { - logger.error("Node detail not found for artifact: $artifactInfo") - throw ErrorCodeException(GenericMessageCode.BLOCK_FILE_NODE_NOT_CREATE, artifactInfo) - } - } - private fun checkIfOverwritePipelineArtifact(context: ArtifactUploadContext) { val pipelineSource = context.repoName == PIPELINE || context.repoName == CUSTOM if (!pipelineSource) { diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt index ac172fae9b..1da67cab1d 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt @@ -38,7 +38,6 @@ import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE @@ -47,15 +46,16 @@ import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.BLOCK_MAPPING_URI -import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.GENERIC_MAPPING_URI -import com.tencent.bkrepo.generic.pojo.NewBlockInfo +import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_ID +import com.tencent.bkrepo.generic.pojo.SeparateBlockInfo +import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo import com.tencent.bkrepo.generic.service.UploadService -import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.PutMapping import org.springframework.web.bind.annotation.RequestAttribute +import org.springframework.web.bind.annotation.RequestHeader import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RestController @@ -67,35 +67,11 @@ class SeparateBlockController( @Permission(ResourceType.NODE, PermissionAction.WRITE) @PostMapping(BLOCK_MAPPING_URI) - fun newStartBlockUpload( + fun startSeparateBlockUpload( @RequestAttribute userId: String, @ArtifactPathVariable artifactInfo: GenericArtifactInfo, - ): Response { - uploadService.newStartBlockUpload(userId, artifactInfo) - return ResponseBuilder.success() - } - - @AuditEntry( - actionId = NODE_CREATE_ACTION - ) - @ActionAuditRecord( - actionId = NODE_CREATE_ACTION, - instance = AuditInstanceRecord( - resourceType = NODE_RESOURCE, - instanceIds = "#artifactInfo?.getArtifactFullPath()", - instanceNames = "#artifactInfo?.getArtifactFullPath()" - ), - attributes = [ - AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), - AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") - ], - scopeId = "#artifactInfo?.projectId", - content = ActionAuditContent.NODE_UPLOAD_CONTENT - ) - @PutMapping(GENERIC_MAPPING_URI) - @Permission(ResourceType.NODE, PermissionAction.WRITE) - fun newBlockUpload(@ArtifactPathVariable artifactInfo: GenericArtifactInfo, file: ArtifactFile) { - uploadService.newBlockUpload(artifactInfo, file) + ): Response { + return ResponseBuilder.success(uploadService.startSeparateBlockUpload(userId, artifactInfo)) } @AuditEntry( @@ -117,30 +93,33 @@ class SeparateBlockController( ) @Permission(ResourceType.NODE, PermissionAction.WRITE) @PutMapping(BLOCK_MAPPING_URI) - fun completeNewBlockUpload( + fun completeSeparateBlockUpload( @RequestAttribute userId: String, + @RequestHeader(HEADER_UPLOAD_ID) uploadId: String, @ArtifactPathVariable artifactInfo: GenericArtifactInfo, ): Response { - uploadService.completeNewBlockUpload(userId, artifactInfo) + uploadService.completeSeparateBlockUpload(userId, uploadId, artifactInfo) return ResponseBuilder.success() } @Permission(ResourceType.NODE, PermissionAction.WRITE) @DeleteMapping(BLOCK_MAPPING_URI) - fun abortNewBlockUpload( + fun abortSeparateBlockUpload( @RequestAttribute userId: String, + @RequestHeader(HEADER_UPLOAD_ID) uploadId: String, @ArtifactPathVariable artifactInfo: GenericArtifactInfo, ): Response { - uploadService.abortNewBlockUpload(userId, artifactInfo) + uploadService.abortSeparateBlockUpload(userId, uploadId, artifactInfo) return ResponseBuilder.success() } @Permission(ResourceType.REPO, PermissionAction.READ) @GetMapping(BLOCK_MAPPING_URI) - fun listNewBlock( + fun listSeparateBlock( @RequestAttribute userId: String, + @RequestHeader(HEADER_UPLOAD_ID) uploadId: String, @ArtifactPathVariable artifactInfo: GenericArtifactInfo, - ): Response> { - return ResponseBuilder.success(uploadService.newListBlock(userId, artifactInfo)) + ): Response> { + return ResponseBuilder.success(uploadService.separateListBlock(userId, uploadId, artifactInfo)) } } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 81b45d90b2..b9e98b6af9 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -32,24 +32,21 @@ package com.tencent.bkrepo.generic.service import com.tencent.bk.audit.context.ActionAuditContext +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.artifact.api.ArtifactFile -import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder -import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService -import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.model.NodeAttribute -import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService @@ -62,6 +59,7 @@ import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.message.StorageErrorException import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY +import com.tencent.bkrepo.fs.server.constant.VERSION_KEY import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo import com.tencent.bkrepo.generic.artifact.GenericLocalRepository import com.tencent.bkrepo.generic.constant.GenericMessageCode @@ -69,7 +67,7 @@ import com.tencent.bkrepo.generic.constant.HEADER_EXPIRES import com.tencent.bkrepo.generic.constant.HEADER_FILE_SIZE import com.tencent.bkrepo.generic.constant.HEADER_OVERWRITE import com.tencent.bkrepo.generic.pojo.BlockInfo -import com.tencent.bkrepo.generic.pojo.NewBlockInfo +import com.tencent.bkrepo.generic.pojo.SeparateBlockInfo import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest @@ -92,11 +90,6 @@ class UploadService( repository.upload(context) } - fun newBlockUpload(artifactInfo: GenericArtifactInfo, file: ArtifactFile) { - val context = ArtifactUploadContext(file) - repository.newUpload(context) - } - fun delete(userId: String, artifactInfo: GenericArtifactInfo) { val context = ArtifactRemoveContext() repository.remove(context) @@ -132,7 +125,33 @@ class UploadService( } } - fun newStartBlockUpload(userId: String, artifactInfo: GenericArtifactInfo){ + fun startSeparateBlockUpload(userId: String, artifactInfo: GenericArtifactInfo): UploadTransactionInfo { + with(artifactInfo) { + // 获取请求头中是否允许覆盖的参数 + val overwrite = getBooleanHeader(HEADER_OVERWRITE) + // 获取当前节点信息 + val node = nodeService.getNodeDetail(this) + // 如果不允许覆盖且节点已经存在，抛出异常 + if (!overwrite && node != null) { + throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, getArtifactName()) + } + + // 生成唯一的 blockId，作为上传会话的标识 + val blockId = StringPool.uniqueId() + val uploadId = "$blockId/VERSION" + + // 创建上传事务信息，设置过期时间 + val uploadTransaction = UploadTransactionInfo( + uploadId = uploadId, + expireSeconds = TRANSACTION_EXPIRES + ) + // 记录上传启动的日志 + logger.info("User[${SecurityUtils.getPrincipal()}] start block upload [$artifactInfo] success, version: $uploadId.") + return uploadTransaction + } + } + + fun blockBaseNodeCreate(userId: String, artifactInfo: GenericArtifactInfo, uploadId: String){ val attributes = NodeAttribute( uid = NodeAttribute.NOBODY, gid = NodeAttribute.NOBODY, @@ -140,8 +159,14 @@ class UploadService( ) val fsAttr = MetadataModel( key = FS_ATTR_KEY, - value = attributes + value = attributes, + ) + val versionMetadata = MetadataModel( + key = VERSION_KEY, + value = uploadId ) + val fileSize = getLongHeader(HEADER_FILE_SIZE).takeIf { it > 0L } + ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND) val request = NodeCreateRequest( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, @@ -150,11 +175,10 @@ class UploadService( sha256 = FAKE_SHA256, md5 = FAKE_MD5, operator = userId, - size = getLongHeader(HEADER_FILE_SIZE).takeIf { it > 0L } - ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND) , + size = fileSize, overwrite = getBooleanHeader(HEADER_OVERWRITE), expires = getLongHeader(HEADER_EXPIRES), - nodeMetadata = listOf(fsAttr) + nodeMetadata = listOf(fsAttr, versionMetadata), ) ActionAuditContext.current().setInstance(request) nodeService.createNode(request) @@ -168,8 +192,13 @@ class UploadService( logger.info("User[${SecurityUtils.getPrincipal()}] abort upload block [$artifactInfo] success.") } - fun abortNewBlockUpload(userId: String, artifactInfo: GenericArtifactInfo) { - delete(userId, artifactInfo) + fun abortSeparateBlockUpload(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo) { + val deleteBlockList = separateListBlock(userId, uploadId, artifactInfo) + val storageCredentials = getStorageCredentials(artifactInfo) + deleteBlockList.forEach { + storageService.delete(it.sha256, storageCredentials) + } + blockNodeService.deleteBlocks(artifactInfo.projectId, artifactInfo.repoName, artifactInfo.getArtifactFullPath(), uploadId) } fun completeBlockUpload( @@ -217,17 +246,45 @@ class UploadService( logger.info("User[${SecurityUtils.getPrincipal()}] complete upload [$artifactInfo] success.") } - fun completeNewBlockUpload(userId: String, artifactInfo: GenericArtifactInfo) { - // 获取并按起始位置排序块信息列表 - val blockInfoList = listBlocks(artifactInfo).sortedBy { it.startPos } + fun completeSeparateBlockUpload(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo) { + + // 创建新的基础节点（Base Node） + try { + blockBaseNodeCreate(userId, artifactInfo, uploadId) + } + catch (e: Exception) { + logger.error("Create block base node failed, file path [${artifactInfo.getArtifactFullPath()}], " + + "version : $uploadId") + abortSeparateBlockUpload(userId, uploadId, artifactInfo) + throw ErrorCodeException(GenericMessageCode.BLOCK_FILE_NODE_CREATE_FAIL, artifactInfo) + } + + // 删除旧Block + blockNodeService.deleteBlocks( + artifactInfo.projectId, + artifactInfo.repoName, + artifactInfo.getArtifactFullPath() + ) - // 获取节点详情，如果节点不存在则抛出异常 + // 获取节点并验证版本信息 val node = ArtifactContextHolder.getNodeDetail(artifactInfo) - ?: throw ErrorCodeException(GenericMessageCode.BLOCK_FILE_NODE_NOT_CREATE, artifactInfo) + ?.takeIf { it.metadata[VERSION_KEY] == uploadId } + ?: run { + logger.error("Failed to retrieve node or version mismatch for uploadId: $uploadId") + throw ErrorCodeException(GenericMessageCode.BLOCK_FILE_NODE_CREATE_FAIL, artifactInfo) + } + + // 获取并按起始位置排序块信息列表 + val blockInfoList = blockNodeService.listBlocksInVersion( + node.projectId, + node.repoName, + node.fullPath, + version = uploadId + ).sortedBy { it.startPos } - // 如果节点是文件夹，抛出异常 - if (node.folder) { - throw ErrorCodeException(GenericMessageCode.BLOCK_LIST_PATH_IS_FOLDER, artifactInfo) + blockInfoList.ifEmpty { + logger.warn("No block information found for uploadId: $uploadId") + throw ErrorCodeException(GenericMessageCode.BLOCK_UPDATE_LIST_IS_NULL, artifactInfo) } var offset = 0L // 用于记录当前偏移量 @@ -235,28 +292,20 @@ class UploadService( // 遍历块信息列表，创建对应的块节点 blockInfoList.forEach { blockInfo -> val blockSize = blockInfo.size - val blockNode = TBlockNode( - id = blockInfo.id, - createdBy = blockInfo.createdBy, - createdDate = blockInfo.createdDate, - nodeFullPath = blockInfo.nodeFullPath, - startPos = offset, - endPos = offset + blockSize - 1, - sha256 = blockInfo.sha256, - projectId = blockInfo.projectId, - repoName = blockInfo.repoName, - size = blockSize - ) - - blockNodeService.updateBlock(blockNode) + blockNodeService.updateBlock(blockInfo, offset, offset + blockSize - 1) offset += blockInfo.size // 更新偏移量 } // 验证节点大小是否与块总大小一致 if (node.size != offset) { - abortNewBlockUpload(userId, artifactInfo) + abortSeparateBlockUpload(userId, uploadId, artifactInfo) throw ErrorCodeException(GenericMessageCode.NODE_DATA_HAS_CHANGED, artifactInfo) } + + // 上传完成，记录日志 + logger.info("User [$userId] successfully completed block upload [uploadId: $uploadId], " + + "file path [${artifactInfo.getArtifactFullPath()}].") + } fun listBlock(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo): List { @@ -269,47 +318,20 @@ class UploadService( } } - fun newListBlock(userId: String, artifactInfo: GenericArtifactInfo): List { + fun separateListBlock(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo): List { - val blockInfoList = listBlocks(artifactInfo) + val blockInfoList = blockNodeService.listBlocksInVersion( + artifactInfo.projectId, + artifactInfo.repoName, + artifactInfo.getArtifactFullPath(), + version = uploadId + ) return blockInfoList.map { blockInfo -> - NewBlockInfo(blockInfo.size, blockInfo.sha256, blockInfo.startPos) + SeparateBlockInfo(blockInfo.size, blockInfo.sha256, blockInfo.startPos, blockInfo.version) } } - private fun listBlocks(artifactInfo: GenericArtifactInfo): List - { - val node = ArtifactContextHolder.getNodeDetail(artifactInfo) - val context = ArtifactDownloadContext() - - if (node == null) { - // 安全调用 repositoryDetail，防止空指针异常 - if (context.repositoryDetail.category == RepositoryCategory.LOCAL) { - throw NodeNotFoundException(artifactInfo.getArtifactFullPath()) - } else { - // 当 node 为 null 且不是 LOCAL 仓库时，返回空列表 - return emptyList() - } - } - - if (node.folder) { - throw ErrorCodeException( - GenericMessageCode.BLOCK_LIST_PATH_IS_FOLDER, - artifactInfo - ) - } - - val blockInfoList = blockNodeService.listBlocks( - Range.full(node.size), - node.projectId, - node.repoName, - node.fullPath, - node.createdDate - ) - return blockInfoList - } - private fun checkUploadId(uploadId: String, storageCredentials: StorageCredentials?) { if (!storageService.checkBlockId(uploadId, storageCredentials)) { throw ErrorCodeException(GenericMessageCode.UPLOAD_ID_NOT_FOUND, uploadId) diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties index cab827727b..41e238bb16 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties @@ -41,6 +41,6 @@ generic.pipeline-repo.manual-upload.not-allowed=Manual upload to pipeline reposi generic.pipeline.metadata.incomplete=Pipeline metadata is incomplete. {0} is missing generic.pipeline.artifact.path.illegal=Note: The path parameter is incorrect. The specified file[{0}] belongs to the artifact of the pipeline repository[{1}] generic.chunked.artifact.broken=Chunked artifact broken -generic.block.file.node.not-create=Block file information node [{0}] is not created +generic.block.file.node.create.fail=Block file information node [{0}] is not created or the version is inconsistent generic.block.node.head.not-found=Both HEADER_OFFSET and HEADER_SEQUENCE are null, HEADER_FILE_SIZE are null -generic.block.list.path.is.folder=The list block path is the directory +generic.block.update.list.is.null=The update list blocks is the empty diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties index 25bdca905a..46ad89ddef 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties @@ -41,6 +41,6 @@ generic.pipeline-repo.manual-upload.not-allowed=禁止手动上传到流水线 generic.pipeline.metadata.incomplete=流水线元数据不完整. 缺少{0} generic.pipeline.artifact.path.illegal=注意：path参数出错，指定文件[{0}]属于流水线仓库[{1}]的制品 generic.chunked.artifact.broken=分块上传制品文件已损坏 -generic.block.file.node.not-create=分块文件信息节点[{0}]未创建 +generic.block.file.node.create.fail=分块文件信息节点[{0}]未创建或版本不一致 generic.block.node.head.not-found=请求头HEADER_OFFSET和HEADER_SEQUENCE不可同时为空, HEADER_FILE_SIZE不可为空 -generic.block.list.path.is.folder=获取分块路径是目录 +generic.block.update.list.is.null=更新分块列表为空 diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties index afffee61c5..296b6b9f32 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties @@ -41,6 +41,6 @@ generic.pipeline-repo.manual-upload.not-allowed=禁止手動上傳到流水線 generic.pipeline.metadata.incomplete=流水線元數據不完整，缺少{0} generic.pipeline.artifact.path.illegal=注意：path參數出錯，指定文件[{0}]屬於流水線倉庫[{1}]的製品 generic.chunked.artifact.broken=製品已損壞 -generic.block.file.node.not-create=分塊文件信息節點[{0}]未創建 +generic.block.file.node.not-create=分塊文件信息節點[{0}]未創建或版本不一致 generic.block.node.head.not-found=請求頭 HEADER_OFFSET 和 HEADER_SEQUENCE 不可同時為空, HEADER_FILE_SIZE 不可為空 -generic.block.list.path.is.folder=獲取分塊路徑是目錄 \ No newline at end of file +generic.block.update.list.is.null=更新分塊列表為空 \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConstants.kt b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConstants.kt index 7751712861..fe46cd1673 100644 --- a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConstants.kt +++ b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/SeparateTestConstants.kt @@ -5,3 +5,4 @@ const val UT_REPO_NAME = "ut-repo" const val UT_USER = "ut-user" const val UT_SHA256 = "ut1000000000000000000000000" const val BLOCK_SIZE = 10 * 1024 * 1024L // 10MB +const val UT_VERSION = "ut-version/version" diff --git a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt index 35a65c5abd..9ebdb0f7f1 100644 --- a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt +++ b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt @@ -17,6 +17,7 @@ import com.tencent.com.bkrepo.generic.UT_PROJECT_ID import com.tencent.com.bkrepo.generic.UT_REPO_NAME import com.tencent.com.bkrepo.generic.UT_SHA256 import com.tencent.com.bkrepo.generic.UT_USER +import com.tencent.com.bkrepo.generic.UT_VERSION import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName @@ -97,7 +98,7 @@ class BlockNodeServiceTest { fun testBlockUpload() { setupBlocks() val blocks = listBlocks(createdDate) - assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE) + assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE, UT_VERSION) } @DisplayName("测试BlockCompletion") @@ -105,13 +106,19 @@ class BlockNodeServiceTest { fun testBlockCompletion() { setupBlocks() val blocks = listBlocks(createdDate) - assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE) + assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE, UT_VERSION) // 完成上传 completeUpload(blocks) - val completeBlocks = listBlocks(createdDate) - assertBlocks(completeBlocks, expectedSize = 2, blockSize = BLOCK_SIZE) + val completeBlocks = blockNodeService.listBlocks( + range, + UT_PROJECT_ID, + UT_REPO_NAME, + "/file", + createdDate.toString() + ) + assertBlocks(completeBlocks, expectedSize = 2, blockSize = BLOCK_SIZE, null) Assertions.assertEquals(0, completeBlocks[0].startPos) Assertions.assertEquals(BLOCK_SIZE, completeBlocks[1].startPos) } @@ -121,13 +128,14 @@ class BlockNodeServiceTest { fun testBlockAbort() { setupBlocks() val blocks = listBlocks(createdDate) - assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE) + assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE, UT_VERSION) // 中止上传 blockNodeService.deleteBlocks( projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME, - fullPath = "/file" + fullPath = "/file", + version = UT_VERSION ) val deleteBlocksQuery = deleteBlocksQuery("/file", UT_PROJECT_ID, UT_REPO_NAME, createdDate) @@ -192,29 +200,32 @@ class BlockNodeServiceTest { sha256 = "$UT_SHA256$i", projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME, - size = BLOCK_SIZE + size = BLOCK_SIZE, + version = UT_VERSION ) val artifactFile = createTempArtifactFile() storageService.store(blockNode.sha256, artifactFile, storageCredentials) blockNodeService.createBlock(blockNode, storageCredentials) } private fun listBlocks(createdDate: LocalDateTime, fullPath: String = "/file"): List { - return blockNodeService.listBlocks( - range = range, + return blockNodeService.listBlocksInVersion( projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME, fullPath = fullPath, - createdDate = createdDate.toString() + createdDate = createdDate.toString(), + version = UT_VERSION, ) } private fun assertBlocks(blocks: List, expectedSize: Int, - blockSize: Long) { + blockSize: Long, + version: String?,) { Assertions.assertEquals(expectedSize, blocks.size) blocks.forEach { block -> Assertions.assertEquals(blockSize, block.size) Assertions.assertTrue(storageService.exist(block.sha256, storageCredentials)) + Assertions.assertEquals(block.version, version) } } @@ -222,11 +233,7 @@ class BlockNodeServiceTest { var offset = 0L blocks.sortedBy { it.startPos }.forEach { blockInfo -> val blockSize = blockInfo.size - val blockNode = blockInfo.copy( - startPos = offset, - endPos = offset + blockSize - 1 - ) - blockNodeService.updateBlock(blockNode) + blockNodeService.updateBlock(blockInfo, offset, offset + blockSize - 1) offset += blockSize // 更新偏移量 } } From 8e0f621065398d4660afd0695826500d73727739 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Thu, 19 Dec 2024 17:06:41 +0800 Subject: [PATCH 10/28] =?UTF-8?q?fix:=20=E4=BB=A3=E7=A0=81=E8=A7=84?= =?UTF-8?q?=E8=8C=83=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{NewBlockInfo.kt => SeparateBlockInfo.kt} | 0 .../bkrepo/generic/service/UploadService.kt | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) rename src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/{NewBlockInfo.kt => SeparateBlockInfo.kt} (100%) diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/SeparateBlockInfo.kt similarity index 100% rename from src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/NewBlockInfo.kt rename to src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/SeparateBlockInfo.kt diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index b9e98b6af9..ff10a7a9be 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -146,7 +146,9 @@ class UploadService( expireSeconds = TRANSACTION_EXPIRES ) // 记录上传启动的日志 - logger.info("User[${SecurityUtils.getPrincipal()}] start block upload [$artifactInfo] success, version: $uploadId.") + logger.info( + "User[${SecurityUtils.getPrincipal()}] start block upload [$artifactInfo] success, " + + "version: $uploadId.") return uploadTransaction } } @@ -198,7 +200,11 @@ class UploadService( deleteBlockList.forEach { storageService.delete(it.sha256, storageCredentials) } - blockNodeService.deleteBlocks(artifactInfo.projectId, artifactInfo.repoName, artifactInfo.getArtifactFullPath(), uploadId) + blockNodeService.deleteBlocks( + artifactInfo.projectId, + artifactInfo.repoName, + artifactInfo.getArtifactFullPath(), + uploadId) } fun completeBlockUpload( @@ -318,7 +324,11 @@ class UploadService( } } - fun separateListBlock(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo): List { + fun separateListBlock( + userId: String, + uploadId: String, + artifactInfo: GenericArtifactInfo + ): List { val blockInfoList = blockNodeService.listBlocksInVersion( artifactInfo.projectId, From 11f5525e7835f5b705cfa8958fa7278cfdad5eef Mon Sep 17 00:00:00 2001 From: zzdjx Date: Wed, 25 Dec 2024 19:33:08 +0800 Subject: [PATCH 11/28] =?UTF-8?q?fix:=20=E4=BB=A3=E7=A0=81=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E5=8F=8A=E9=83=A8=E5=88=86=E9=80=BB=E8=BE=91=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/blocknode/impl/BlockNodeServiceImpl.kt | 6 +----- .../bkrepo/generic/artifact/GenericLocalRepository.kt | 3 +-- .../tencent/bkrepo/generic/service/UploadService.kt | 10 +++------- .../src/main/resources/i18n/messages_zh_CN.properties | 2 +- .../src/main/resources/i18n/messages_zh_TW.properties | 2 +- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index 3f3f47cca5..1a7bc8019f 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -116,11 +116,7 @@ class BlockNodeServiceImpl( version: String? ) { val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) - if (version != null) { - criteria.and(TBlockNode::version.name).`is`(version) - }else{ - criteria.and(TBlockNode::version.name).`is`(null) - } + .apply { and(TBlockNode::version.name).isEqualTo(version) } val update = BlockNodeQueryHelper.deleteUpdate() blockNodeDao.updateMulti(Query(criteria), update) logger.info("Delete node blocks[$projectId/$repoName$fullPath] success. Version: $version") diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index e2072f97f0..39aaec7843 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -182,8 +182,7 @@ class GenericLocalRepository( val uploadId = context.request.getHeader(HEADER_UPLOAD_ID) if (isSeparateUpload(uploadId)){ onSeparateUpload(context, uploadId) - } - else{ + } else{ val sequence = context.request.getHeader(HEADER_SEQUENCE)?.toInt() val uploadType = HeaderUtils.getHeader(HEADER_UPLOAD_TYPE) if (isBlockUpload(uploadId, sequence)) { diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index ff10a7a9be..701d7f0f78 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -195,11 +195,6 @@ class UploadService( } fun abortSeparateBlockUpload(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo) { - val deleteBlockList = separateListBlock(userId, uploadId, artifactInfo) - val storageCredentials = getStorageCredentials(artifactInfo) - deleteBlockList.forEach { - storageService.delete(it.sha256, storageCredentials) - } blockNodeService.deleteBlocks( artifactInfo.projectId, artifactInfo.repoName, @@ -276,8 +271,9 @@ class UploadService( val node = ArtifactContextHolder.getNodeDetail(artifactInfo) ?.takeIf { it.metadata[VERSION_KEY] == uploadId } ?: run { - logger.error("Failed to retrieve node or version mismatch for uploadId: $uploadId") - throw ErrorCodeException(GenericMessageCode.BLOCK_FILE_NODE_CREATE_FAIL, artifactInfo) + logger.warn("Version mismatch for uploadId: $uploadId") + abortSeparateBlockUpload(userId, uploadId, artifactInfo) + return } // 获取并按起始位置排序块信息列表 diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties index 46ad89ddef..2ec5901bc3 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties @@ -43,4 +43,4 @@ generic.pipeline.artifact.path.illegal=注意：path参数出错，指定文件[ generic.chunked.artifact.broken=分块上传制品文件已损坏 generic.block.file.node.create.fail=分块文件信息节点[{0}]未创建或版本不一致 generic.block.node.head.not-found=请求头HEADER_OFFSET和HEADER_SEQUENCE不可同时为空, HEADER_FILE_SIZE不可为空 -generic.block.update.list.is.null=更新分块列表为空 +generic.block.update.list.is.null=更新分块列表为空 \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties index 296b6b9f32..af0eaec3da 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties @@ -41,6 +41,6 @@ generic.pipeline-repo.manual-upload.not-allowed=禁止手動上傳到流水線 generic.pipeline.metadata.incomplete=流水線元數據不完整，缺少{0} generic.pipeline.artifact.path.illegal=注意：path參數出錯，指定文件[{0}]屬於流水線倉庫[{1}]的製品 generic.chunked.artifact.broken=製品已損壞 -generic.block.file.node.not-create=分塊文件信息節點[{0}]未創建或版本不一致 +generic.block.file.node.create.fail=分塊文件信息節點[{0}]未創建或版本不一致 generic.block.node.head.not-found=請求頭 HEADER_OFFSET 和 HEADER_SEQUENCE 不可同時為空, HEADER_FILE_SIZE 不可為空 generic.block.update.list.is.null=更新分塊列表為空 \ No newline at end of file From b940e8c5fb04624655e757e801bba38e738d791c Mon Sep 17 00:00:00 2001 From: zzdjx Date: Thu, 26 Dec 2024 18:35:23 +0800 Subject: [PATCH 12/28] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8DisSeparateUpload?= =?UTF-8?q?=20uploadId=20null=E5=BC=82=E5=B8=B8=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/bkrepo/generic/artifact/GenericLocalRepository.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index 39aaec7843..f6ad7fcc9e 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -246,8 +246,8 @@ class GenericLocalRepository( } } - private fun isSeparateUpload(uploadId: String): Boolean { - return uploadId.substringAfter("/") == VERSION_KEY + private fun isSeparateUpload(uploadId: String?): Boolean { + return uploadId?.substringAfter("/") == VERSION_KEY } override fun onUploadSuccess(context: ArtifactUploadContext) { From 1ca4460b449694d085be19c666754f2d8172a705 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 11:31:52 +0800 Subject: [PATCH 13/28] =?UTF-8?q?feat:=20=E5=88=86=E5=9D=97=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E5=B9=B6=E5=8F=91=E5=9C=BA=E6=99=AF=E4=B8=8B=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/metadata/constant/Constant.kt | 1 + .../metadata/dao/blocknode/BlockNodeDao.kt | 18 +------ .../common/metadata/model/TBlockNode.kt | 2 +- .../service/blocknode/BlockNodeService.kt | 19 ++++--- .../blocknode/impl/BlockNodeServiceImpl.kt | 48 ++++++++++------- .../service/node/NodeDeleteOperation.kt | 11 ++++ .../service/node/impl/NodeBaseService.kt | 21 +++++++- .../service/node/impl/NodeDeleteSupport.kt | 32 +++++++++++ .../service/node/impl/NodeServiceImpl.kt | 10 ++++ .../node/impl/center/CenterNodeServiceImpl.kt | 4 +- .../node/impl/edge/EdgeNodeServiceImpl.kt | 10 ++++ .../metadata/util/BlockNodeQueryHelper.kt | 14 ++--- .../bkrepo/fs/server/constant/Constants.kt | 2 +- .../generic/artifact/GenericArtifactInfo.kt | 1 + .../bkrepo/generic/constant/Constants.kt | 5 ++ .../artifact/GenericLocalRepository.kt | 49 ++++++++--------- .../controller/SeparateBlockController.kt | 12 ++--- .../bkrepo/generic/service/UploadService.kt | 54 ++++++++++--------- .../resources/i18n/messages_en.properties | 2 +- .../resources/i18n/messages_zh_CN.properties | 2 +- .../resources/i18n/messages_zh_TW.properties | 2 +- .../pojo/node/service/NodeCreateRequest.kt | 4 +- 22 files changed, 202 insertions(+), 121 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt index 3f63c95f1d..4f9aa8a032 100644 --- a/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt +++ b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt @@ -32,3 +32,4 @@ const val ID = "_id" const val FAKE_SHA256 = "0000000000000000000000000000000000000000000000000000000000000000" const val FAKE_MD5 = "00000000000000000000000000000000" +const val FAKE_SEPARATE = "00000000FIRSTUPLOAD" diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt index 150ab4eddd..3dc8412b1b 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt @@ -27,28 +27,12 @@ package com.tencent.bkrepo.common.metadata.dao.blocknode -import com.mongodb.client.result.UpdateResult import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.mongo.dao.sharding.HashShardingMongoDao import org.springframework.context.annotation.Conditional -import org.springframework.data.mongodb.core.query.Query -import org.springframework.data.mongodb.core.query.Update import org.springframework.stereotype.Repository @Repository @Conditional(SyncCondition::class) -class BlockNodeDao : HashShardingMongoDao(){ - - fun updateBlock(query: Query, update: Update): UpdateResult { - if (logger.isDebugEnabled) { - logger.debug("Mongo Dao updateFirst: [$query], [$update]") - } - return determineMongoTemplate() - .updateFirst( - query, - update, - TBlockNode::class.java, - determineCollectionName(query)) - } -} +class BlockNodeDao : HashShardingMongoDao() diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt index e3c6239f12..e09ae25231 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt @@ -56,7 +56,7 @@ data class TBlockNode( val size: Long, val endPos: Long = startPos + size - 1, var deleted: LocalDateTime? = null, - val version: String? = null + val uploadId: String? = null ) { companion object { const val BLOCK_IDX = "start_pos_idx" diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index 061a3c7a6f..1c997772fb 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -50,12 +50,12 @@ interface BlockNodeService { /** * 查询出当前版本内的分块 */ - fun listBlocksInVersion( + fun listBlocksInUploadId( projectId: String, repoName: String, fullPath: String, createdDate: String? = null, - version: String + uploadId: String ): List /** @@ -69,10 +69,11 @@ interface BlockNodeService { /** * 更新分块 * */ - fun updateBlock( - blockNode: TBlockNode, - startPos: Long, - endPos: Long, + fun updateBlockUploadId( + projectId: String, + repoName: String, + fullPath: String, + uploadId: String ) /** @@ -87,7 +88,11 @@ interface BlockNodeService { projectId: String, repoName: String, fullPath: String, - version: String? = null + uploadId: String? = null + ) + + fun deleteBlock( + blockNode: TBlockNode ) /** diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index 1a7bc8019f..b458df526d 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -69,21 +69,18 @@ class BlockNodeServiceImpl( } } - override fun updateBlock( - blockNode: TBlockNode, - startPos: Long, - endPos: Long, + override fun updateBlockUploadId( + projectId: String, + repoName: String, + fullPath: String, + uploadId: String ) { - with(blockNode) { - val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, nodeFullPath,false) - criteria.and(TBlockNode::id).isEqualTo(id) - .and(TBlockNode::version).isEqualTo(version) - val update = BlockNodeQueryHelper.updateVersionBlocks(startPos, endPos) - blockNodeDao.updateBlock(Query(criteria), update) - logger.info("Update block node[$projectId/$repoName/$nodeFullPath-$startPos], sha256[$sha256] success.") - } - - + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath,false) + criteria.and(TBlockNode::uploadId).isEqualTo(uploadId) + val update = Update().set(TBlockNode::uploadId.name, null) + .set(TBlockNode::createdDate.name, LocalDateTime.now()) + blockNodeDao.updateMulti(Query(criteria), update) + logger.info("Update block node[$projectId/$repoName/$fullPath--/uploadId: $uploadId] success.") } override fun listBlocks( @@ -97,15 +94,15 @@ class BlockNodeServiceImpl( return blockNodeDao.find(query) } - override fun listBlocksInVersion( + override fun listBlocksInUploadId( projectId: String, repoName: String, fullPath: String, createdDate: String?, - version: String + uploadId: String ): List { val query = - BlockNodeQueryHelper.listQueryInVersion(projectId, repoName, fullPath, createdDate, version) + BlockNodeQueryHelper.listQueryInUploadId(projectId, repoName, fullPath, createdDate, uploadId) return blockNodeDao.find(query) } @@ -113,13 +110,24 @@ class BlockNodeServiceImpl( projectId: String, repoName: String, fullPath: String, - version: String? + uploadId: String? ) { val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) - .apply { and(TBlockNode::version.name).isEqualTo(version) } + .apply { and(TBlockNode::uploadId.name).isEqualTo(uploadId) } val update = BlockNodeQueryHelper.deleteUpdate() blockNodeDao.updateMulti(Query(criteria), update) - logger.info("Delete node blocks[$projectId/$repoName$fullPath] success. Version: $version") + logger.info("Delete node blocks[$projectId/$repoName$fullPath] success. UPLOADID: $uploadId") + } + + override fun deleteBlock(blockNode: TBlockNode) { + with(blockNode) { + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, nodeFullPath, false) + .and(TBlockNode::startPos.name).isEqualTo(startPos) + .and(TBlockNode::uploadId.name).isEqualTo(uploadId) + val update = BlockNodeQueryHelper.deleteUpdate() + blockNodeDao.updateFirst(Query(criteria), update) + logger.info("Delete single node block[$projectId/$repoName$nodeFullPath] success. Id: $id, UPLOADID: $uploadId") + } } override fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt index a33f46b09d..3af59cebcb 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt @@ -84,4 +84,15 @@ interface NodeDeleteOperation { path: String, decreaseVolume: Boolean = true ): NodeDeleteResult + + /** + * 删除旧node + */ + fun deleteOldNode( + projectId: String, + repoName: String, + fullPath: String, + operator: String, + nodeId: String + ): NodeDeleteResult } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index eee31154b0..bfb163372c 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -43,6 +43,7 @@ import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties import com.tencent.bkrepo.common.metadata.config.RepositoryProperties import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SEPARATE import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao @@ -70,6 +71,7 @@ import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent import com.tencent.bkrepo.common.stream.constant.BinderType import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier +import com.tencent.bkrepo.fs.server.constant.UPLOADID_KEY import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -195,7 +197,7 @@ abstract class NodeBaseService( mkdirs(projectId, repoName, PathUtils.resolveParent(fullPath), operator) // 创建节点 val node = buildTNode(this) - doCreate(node) + doCreate(node, separate = separate) afterCreate(repo, node) logger.info("Create node[/$projectId/$repoName$fullPath], sha256[$sha256] success.") return convertToDetail(node)!! @@ -355,7 +357,7 @@ abstract class NodeBaseService( } } - open fun doCreate(node: TNode, repository: TRepository? = null): TNode { + open fun doCreate(node: TNode, repository: TRepository? = null, separate: Boolean = false): TNode { try { nodeDao.insert(node) if (!node.folder) { @@ -367,6 +369,9 @@ abstract class NodeBaseService( } } catch (exception: DuplicateKeyException) { logger.warn("Insert node[$node] error: [${exception.message}]") + if (separate){ + throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, node.fullPath) + } } return node @@ -417,6 +422,18 @@ abstract class NodeBaseService( throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, fullPath) } else if (existNode.folder || this.folder) { throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, fullPath) + } else if (separate) { + val currentVersion = createRequest.metadata!![UPLOADID_KEY].toString() + val oldNodeId = currentVersion.substringAfter("/") + if (oldNodeId == FAKE_SEPARATE){ + return + } + val deleteRes = deleteOldNode(projectId, repoName, fullPath, operator, oldNodeId) + if (deleteRes.deletedNumber == 0L){ + logger.warn("Delete block base node[$fullPath] by [$operator] error: node was deleted") + throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) + } + quotaService.decreaseUsedVolume(projectId, repoName, deleteRes.deletedSize) } else { val changeSize = this.size?.minus(existNode.size) ?: -existNode.size quotaService.checkRepoQuota(projectId, repoName, changeSize) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 43d72b803a..3592d09575 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -45,6 +45,7 @@ import com.tencent.bkrepo.common.metadata.util.NodeDeleteHelper.buildCriteria import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildDeletedEvent import com.tencent.bkrepo.common.metadata.util.NodeEventFactory.buildNodeCleanEvent import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper +import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.router.api.RouterControllerClient import org.slf4j.LoggerFactory import org.springframework.dao.DuplicateKeyException @@ -185,6 +186,37 @@ open class NodeDeleteSupport( return nodeDeleteResult } + override fun deleteOldNode( + projectId: String, + repoName: String, + fullPath: String, + operator: String, + nodeId: String + ): NodeDeleteResult { + require(!PathUtils.isRoot(fullPath)) { "Cannot delete root node." } + val deleteTime = LocalDateTime.now() + + val criteria = buildCriteria(projectId, repoName, fullPath).apply { + and(ID).isEqualTo(nodeId) + } + val query = Query(criteria) + + // 删除旧节点 + val updateDefinition = NodeQueryHelper.nodeDeleteUpdate(operator, deleteTime) + val updateResult = nodeDao.updateFirst(query, updateDefinition) + val deletedNum = updateResult.modifiedCount + + if (deletedNum == 0L) return NodeDeleteResult(0L, 0L, deleteTime) + + // 计算删除的文件大小 + val deletedSize = nodeBaseService.aggregateComputeSize(criteria.apply { + and(TNode::deleted.name).isEqualTo(deleteTime) + }) + quotaService.decreaseUsedVolume(projectId, repoName, deletedSize) + + return NodeDeleteResult(deletedNum, deletedSize, deleteTime) + } + private fun delete( query: Query, operator: String, diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt index 082c2f3f7d..1ab762491a 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt @@ -166,6 +166,16 @@ class NodeServiceImpl( return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume) } + override fun deleteOldNode( + projectId: String, + repoName: String, + fullPath: String, + operator: String, + nodeId: String + ): NodeDeleteResult { + return NodeDeleteSupport(this).deleteOldNode(projectId, repoName, fullPath, operator, nodeId) + } + @Transactional(rollbackFor = [Throwable::class]) override fun moveNode(moveRequest: NodeMoveCopyRequest): NodeDetail { return NodeMoveCopySupport(this).moveNode(moveRequest) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index d1b7beeba1..0690393472 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -156,9 +156,9 @@ class CenterNodeServiceImpl( return node } - override fun doCreate(node: TNode, repository: TRepository?): TNode { + override fun doCreate(node: TNode, repository: TRepository?, separate: Boolean): TNode { if (SecurityUtils.getClusterName().isNullOrBlank()) { - return super.doCreate(node, repository) + return super.doCreate(node, repository, false) } try { nodeDao.insert(node) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt index 83fa9ea862..daeff4e72e 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -201,6 +201,16 @@ class EdgeNodeServiceImpl( return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume) } + override fun deleteOldNode( + projectId: String, + repoName: String, + fullPath: String, + operator: String, + nodeId: String + ): NodeDeleteResult { + return NodeDeleteSupport(this).deleteOldNode(projectId, repoName, fullPath, operator, nodeId) + } + @Transactional(rollbackFor = [Throwable::class]) override fun moveNode(moveRequest: NodeMoveCopyRequest): NodeDetail { ignoreException( diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt index ba66f01dbb..e1da2dc301 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt @@ -59,22 +59,23 @@ object BlockNodeQueryHelper { TBlockNode::startPos.gt(range.end), TBlockNode::endPos.lt(range.start) ) + .and(TBlockNode::uploadId).isEqualTo(null) val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) return query } - fun listQueryInVersion( + fun listQueryInUploadId( projectId: String, repoName: String, fullPath: String, createdDate: String?, - version: String, + uploadId: String, ):Query { val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) .and(TBlockNode::projectId).isEqualTo(projectId) .and(TBlockNode::repoName).isEqualTo(repoName) .and(TBlockNode::deleted).isEqualTo(null) - .and(TBlockNode::version).isEqualTo(version) + .and(TBlockNode::uploadId).isEqualTo(uploadId) createdDate?.let { criteria.and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) } @@ -119,11 +120,4 @@ object BlockNodeQueryHelper { return Update().set(TBlockNode::deleted.name, null) } - fun updateVersionBlocks(startPos: Long, endPos: Long): Update { - return Update().set(TBlockNode::version.name, null) - .set(TBlockNode::createdDate.name, LocalDateTime.now()) - .set(TBlockNode::startPos.name, startPos) - .set(TBlockNode::endPos.name, endPos) - } - } diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt index bea78e9921..f58c73f399 100644 --- a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt +++ b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt @@ -33,5 +33,5 @@ const val JWT_CLAIMS_REPOSITORY = "repository" const val JWT_CLAIMS_PERMIT = "permit" const val FS_ATTR_KEY = "fs:attr" -const val VERSION_KEY = "VERSION" +const val UPLOADID_KEY = "UPLOADID" diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericArtifactInfo.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericArtifactInfo.kt index 570c6482f1..57c8237f37 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericArtifactInfo.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericArtifactInfo.kt @@ -43,6 +43,7 @@ class GenericArtifactInfo( const val BLOCK_MAPPING_URI = "/block/{projectId}/{repoName}/**" const val DELTA_MAPPING_URI = "/delta/{projectId}/{repoName}/**" const val BATCH_MAPPING_URI = "/batch/{projectId}/{repoName}" + const val SEPARATE_MAPPING_URI = "/separate/{projectId}/{repoName}/**" const val CHUNKED_UPLOAD_MAPPING_URI = "/{projectId}/{repoName}/**/chunked/uploads/**" } } diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt index 2f80458955..2d56f7aa29 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/Constants.kt @@ -62,3 +62,8 @@ const val CHUNKED_UPLOAD_CLIENT = "CHUNKED-UPLOAD-CLIENT" // block上传时直接写入文件指定位置 const val HEADER_BLOCK_APPEND = BKREPO_PREFIX + "BLOCK-APPEND" + +/** + * 分块上传版本后缀 + */ +const val SEPARATE_UPLOAD = "SEPARATE-UPLOAD" \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index f6ad7fcc9e..8d800697a2 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -87,7 +87,6 @@ import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.common.storage.message.StorageErrorException import com.tencent.bkrepo.common.storage.pojo.FileInfo -import com.tencent.bkrepo.fs.server.constant.VERSION_KEY import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext import com.tencent.bkrepo.generic.constant.BKREPO_META import com.tencent.bkrepo.generic.constant.BKREPO_META_PREFIX @@ -103,6 +102,7 @@ import com.tencent.bkrepo.generic.constant.HEADER_SIZE import com.tencent.bkrepo.generic.constant.HEADER_OVERWRITE import com.tencent.bkrepo.generic.constant.HEADER_BLOCK_APPEND import com.tencent.bkrepo.generic.constant.HEADER_EXPIRES +import com.tencent.bkrepo.generic.constant.SEPARATE_UPLOAD import com.tencent.bkrepo.generic.pojo.ChunkedResponseProperty import com.tencent.bkrepo.generic.util.ChunkedRequestUtil.uploadResponse import com.tencent.bkrepo.replication.api.ClusterNodeClient @@ -155,13 +155,6 @@ class GenericLocalRepository( super.onUploadBefore(context) // 若不允许覆盖, 提前检查节点是否存在 checkNodeExist(context) - // 通用上传前检查 - baseUploadBefore(context) - // 二次检查，防止接收文件过程中，有并发上传成功的情况 - checkNodeExist(context) - } - - private fun baseUploadBefore(context: ArtifactUploadContext){ // 检查是否是覆盖流水线构件 checkIfOverwritePipelineArtifact(context) // 校验sha256 @@ -176,22 +169,28 @@ class GenericLocalRepository( if (uploadMd5 != null && !calculatedMd5.equals(uploadMd5, true)) { throw ErrorCodeException(ArtifactMessageCode.DIGEST_CHECK_FAILED, "md5") } + // 二次检查，防止接收文件过程中，有并发上传成功的情况 + checkNodeExist(context) } override fun onUpload(context: ArtifactUploadContext) { val uploadId = context.request.getHeader(HEADER_UPLOAD_ID) - if (isSeparateUpload(uploadId)){ - onSeparateUpload(context, uploadId) - } else{ - val sequence = context.request.getHeader(HEADER_SEQUENCE)?.toInt() - val uploadType = HeaderUtils.getHeader(HEADER_UPLOAD_TYPE) - if (isBlockUpload(uploadId, sequence)) { + val sequence = context.request.getHeader(HEADER_SEQUENCE)?.toInt() + val uploadType = HeaderUtils.getHeader(HEADER_UPLOAD_TYPE) + + when { + isSeparateUpload(uploadType) -> { + onSeparateUpload(context, uploadId) + } + isBlockUpload(uploadId, sequence) -> { this.blockUpload(uploadId, sequence!!, context) context.response.contentType = MediaTypes.APPLICATION_JSON context.response.writer.println(ResponseBuilder.success().toJsonString()) - } else if (isChunkedUpload(uploadType)) { + } + isChunkedUpload(uploadType) -> { chunkedUpload(context) - } else { + } + else -> { val nodeDetail = storageManager.storeArtifactFile( buildNodeCreateRequest(context), context.getArtifactFile(), @@ -208,32 +207,30 @@ class GenericLocalRepository( private fun onSeparateUpload(context: ArtifactUploadContext, uploadId: String) { with(context) { - val bArtifactFile = getArtifactFile() + val blockArtifactFile = getArtifactFile() val sha256 = getArtifactSha256() - val sequence = context.request.getHeader(HEADER_SEQUENCE).toLongOrNull() val offset = context.request.getHeader(HEADER_OFFSET)?.toLongOrNull() val blockNode = TBlockNode( createdBy = userId, createdDate = LocalDateTime.now(), nodeFullPath = artifactInfo.getArtifactFullPath(), - startPos = offset ?: sequence - ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND), + startPos = offset ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND), sha256 = sha256, projectId = projectId, repoName = repoName, - size = bArtifactFile.getSize(), - version = uploadId + size = blockArtifactFile.getSize(), + uploadId = uploadId ) - val stored = storageService.store(sha256, bArtifactFile, storageCredentials) + val stored = storageService.store(sha256, blockArtifactFile, storageCredentials) try { blockNodeService.createBlock(blockNode, storageCredentials) } catch (e: Exception) { if (stored > 1) { - storageService.delete(sha256, storageCredentials) + blockNodeService.deleteBlock(blockNode) } // Log the exception for debugging purposes logger.error("Failed to create block node", e) @@ -246,8 +243,8 @@ class GenericLocalRepository( } } - private fun isSeparateUpload(uploadId: String?): Boolean { - return uploadId?.substringAfter("/") == VERSION_KEY + private fun isSeparateUpload(uploadType: String?): Boolean { + return !uploadType.isNullOrEmpty() && uploadType == SEPARATE_UPLOAD } override fun onUploadSuccess(context: ArtifactUploadContext) { diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt index 1da67cab1d..134333aca4 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/SeparateBlockController.kt @@ -45,7 +45,7 @@ import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo -import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.BLOCK_MAPPING_URI +import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo.Companion.SEPARATE_MAPPING_URI import com.tencent.bkrepo.generic.constant.HEADER_UPLOAD_ID import com.tencent.bkrepo.generic.pojo.SeparateBlockInfo import com.tencent.bkrepo.generic.pojo.UploadTransactionInfo @@ -56,17 +56,15 @@ import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.PutMapping import org.springframework.web.bind.annotation.RequestAttribute import org.springframework.web.bind.annotation.RequestHeader -import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RestController @RestController -@RequestMapping("/separate") class SeparateBlockController( private val uploadService: UploadService, ) { @Permission(ResourceType.NODE, PermissionAction.WRITE) - @PostMapping(BLOCK_MAPPING_URI) + @PostMapping(SEPARATE_MAPPING_URI) fun startSeparateBlockUpload( @RequestAttribute userId: String, @ArtifactPathVariable artifactInfo: GenericArtifactInfo, @@ -92,7 +90,7 @@ class SeparateBlockController( content = ActionAuditContent.NODE_UPLOAD_CONTENT ) @Permission(ResourceType.NODE, PermissionAction.WRITE) - @PutMapping(BLOCK_MAPPING_URI) + @PutMapping(SEPARATE_MAPPING_URI) fun completeSeparateBlockUpload( @RequestAttribute userId: String, @RequestHeader(HEADER_UPLOAD_ID) uploadId: String, @@ -103,7 +101,7 @@ class SeparateBlockController( } @Permission(ResourceType.NODE, PermissionAction.WRITE) - @DeleteMapping(BLOCK_MAPPING_URI) + @DeleteMapping(SEPARATE_MAPPING_URI) fun abortSeparateBlockUpload( @RequestAttribute userId: String, @RequestHeader(HEADER_UPLOAD_ID) uploadId: String, @@ -114,7 +112,7 @@ class SeparateBlockController( } @Permission(ResourceType.REPO, PermissionAction.READ) - @GetMapping(BLOCK_MAPPING_URI) + @GetMapping(SEPARATE_MAPPING_URI) fun listSeparateBlock( @RequestAttribute userId: String, @RequestHeader(HEADER_UPLOAD_ID) uploadId: String, diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 701d7f0f78..6211cf8a1c 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -45,6 +45,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveConte import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SEPARATE import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.model.NodeAttribute import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService @@ -59,7 +60,7 @@ import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.message.StorageErrorException import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY -import com.tencent.bkrepo.fs.server.constant.VERSION_KEY +import com.tencent.bkrepo.fs.server.constant.UPLOADID_KEY import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo import com.tencent.bkrepo.generic.artifact.GenericLocalRepository import com.tencent.bkrepo.generic.constant.GenericMessageCode @@ -129,16 +130,17 @@ class UploadService( with(artifactInfo) { // 获取请求头中是否允许覆盖的参数 val overwrite = getBooleanHeader(HEADER_OVERWRITE) - // 获取当前节点信息 + val node = nodeService.getNodeDetail(this) + + val oldNodeId = node?.nodeInfo?.id ?: FAKE_SEPARATE // 如果不允许覆盖且节点已经存在，抛出异常 - if (!overwrite && node != null) { + if (node != null && !overwrite) { throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, getArtifactName()) } // 生成唯一的 blockId，作为上传会话的标识 - val blockId = StringPool.uniqueId() - val uploadId = "$blockId/VERSION" + val uploadId = "${StringPool.uniqueId()}/$oldNodeId" // 创建上传事务信息，设置过期时间 val uploadTransaction = UploadTransactionInfo( @@ -164,7 +166,7 @@ class UploadService( value = attributes, ) val versionMetadata = MetadataModel( - key = VERSION_KEY, + key = UPLOADID_KEY, value = uploadId ) val fileSize = getLongHeader(HEADER_FILE_SIZE).takeIf { it > 0L } @@ -181,6 +183,8 @@ class UploadService( overwrite = getBooleanHeader(HEADER_OVERWRITE), expires = getLongHeader(HEADER_EXPIRES), nodeMetadata = listOf(fsAttr, versionMetadata), + separate = true, + metadata = mapOf(UPLOADID_KEY to uploadId) ) ActionAuditContext.current().setInstance(request) nodeService.createNode(request) @@ -240,7 +244,8 @@ class UploadService( overwrite = getBooleanHeader(HEADER_OVERWRITE), operator = userId, expires = getLongHeader(HEADER_EXPIRES), - nodeMetadata = repository.resolveMetadata(HttpContextHolder.getRequest()) + nodeMetadata = repository.resolveMetadata(HttpContextHolder.getRequest()), + separate = true ) ActionAuditContext.current().setInstance(request) nodeService.createNode(request) @@ -257,7 +262,7 @@ class UploadService( logger.error("Create block base node failed, file path [${artifactInfo.getArtifactFullPath()}], " + "version : $uploadId") abortSeparateBlockUpload(userId, uploadId, artifactInfo) - throw ErrorCodeException(GenericMessageCode.BLOCK_FILE_NODE_CREATE_FAIL, artifactInfo) + throw e } // 删除旧Block @@ -269,7 +274,7 @@ class UploadService( // 获取节点并验证版本信息 val node = ArtifactContextHolder.getNodeDetail(artifactInfo) - ?.takeIf { it.metadata[VERSION_KEY] == uploadId } + ?.takeIf { it.metadata[UPLOADID_KEY] == uploadId } ?: run { logger.warn("Version mismatch for uploadId: $uploadId") abortSeparateBlockUpload(userId, uploadId, artifactInfo) @@ -277,30 +282,31 @@ class UploadService( } // 获取并按起始位置排序块信息列表 - val blockInfoList = blockNodeService.listBlocksInVersion( + val blockInfoList = blockNodeService.listBlocksInUploadId( node.projectId, node.repoName, node.fullPath, - version = uploadId - ).sortedBy { it.startPos } + uploadId = uploadId + ) blockInfoList.ifEmpty { logger.warn("No block information found for uploadId: $uploadId") throw ErrorCodeException(GenericMessageCode.BLOCK_UPDATE_LIST_IS_NULL, artifactInfo) } - var offset = 0L // 用于记录当前偏移量 + // 计算所有块的总大小 + val totalSize = blockInfoList.sumOf { it.size } - // 遍历块信息列表，创建对应的块节点 - blockInfoList.forEach { blockInfo -> - val blockSize = blockInfo.size - blockNodeService.updateBlock(blockInfo, offset, offset + blockSize - 1) - offset += blockInfo.size // 更新偏移量 - } + // 更新节点版本信息为null + blockNodeService.updateBlockUploadId( + artifactInfo.projectId, + artifactInfo.repoName, + artifactInfo.getArtifactFullPath(), + uploadId + ) // 验证节点大小是否与块总大小一致 - if (node.size != offset) { - abortSeparateBlockUpload(userId, uploadId, artifactInfo) + if (node.size != totalSize) { throw ErrorCodeException(GenericMessageCode.NODE_DATA_HAS_CHANGED, artifactInfo) } @@ -326,15 +332,15 @@ class UploadService( artifactInfo: GenericArtifactInfo ): List { - val blockInfoList = blockNodeService.listBlocksInVersion( + val blockInfoList = blockNodeService.listBlocksInUploadId( artifactInfo.projectId, artifactInfo.repoName, artifactInfo.getArtifactFullPath(), - version = uploadId + uploadId = uploadId ) return blockInfoList.map { blockInfo -> - SeparateBlockInfo(blockInfo.size, blockInfo.sha256, blockInfo.startPos, blockInfo.version) + SeparateBlockInfo(blockInfo.size, blockInfo.sha256, blockInfo.startPos, blockInfo.uploadId) } } diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties index 41e238bb16..2647c184dd 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties @@ -42,5 +42,5 @@ generic.pipeline.metadata.incomplete=Pipeline metadata is incomplete. {0} is mis generic.pipeline.artifact.path.illegal=Note: The path parameter is incorrect. The specified file[{0}] belongs to the artifact of the pipeline repository[{1}] generic.chunked.artifact.broken=Chunked artifact broken generic.block.file.node.create.fail=Block file information node [{0}] is not created or the version is inconsistent -generic.block.node.head.not-found=Both HEADER_OFFSET and HEADER_SEQUENCE are null, HEADER_FILE_SIZE are null +generic.block.node.head.not-found=Both HEADER_OFFSET and HEADER_FILE_SIZE are null. generic.block.update.list.is.null=The update list blocks is the empty diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties index 2ec5901bc3..83ada238b1 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties @@ -42,5 +42,5 @@ generic.pipeline.metadata.incomplete=流水线元数据不完整. 缺少{0} generic.pipeline.artifact.path.illegal=注意：path参数出错，指定文件[{0}]属于流水线仓库[{1}]的制品 generic.chunked.artifact.broken=分块上传制品文件已损坏 generic.block.file.node.create.fail=分块文件信息节点[{0}]未创建或版本不一致 -generic.block.node.head.not-found=请求头HEADER_OFFSET和HEADER_SEQUENCE不可同时为空, HEADER_FILE_SIZE不可为空 +generic.block.node.head.not-found=请求头HEADER_OFFSET和HEADER_FILE_SIZE不可为空 generic.block.update.list.is.null=更新分块列表为空 \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties index af0eaec3da..90e51a091e 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties @@ -42,5 +42,5 @@ generic.pipeline.metadata.incomplete=流水線元數據不完整，缺少{0} generic.pipeline.artifact.path.illegal=注意：path參數出錯，指定文件[{0}]屬於流水線倉庫[{1}]的製品 generic.chunked.artifact.broken=製品已損壞 generic.block.file.node.create.fail=分塊文件信息節點[{0}]未創建或版本不一致 -generic.block.node.head.not-found=請求頭 HEADER_OFFSET 和 HEADER_SEQUENCE 不可同時為空, HEADER_FILE_SIZE 不可為空 +generic.block.node.head.not-found=請求頭 HEADER_OFFSET 和 HEADER_FILE_SIZE 不可為空 generic.block.update.list.is.null=更新分塊列表為空 \ No newline at end of file diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeCreateRequest.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeCreateRequest.kt index f9f23a53c8..5cb327e0a1 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeCreateRequest.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeCreateRequest.kt @@ -73,5 +73,7 @@ data class NodeCreateRequest( override val createdBy: String? = null, override var createdDate: LocalDateTime? = null, override val lastModifiedBy: String? = null, - override var lastModifiedDate: LocalDateTime? = null + override var lastModifiedDate: LocalDateTime? = null, + @ApiModelProperty("是否SEPARATE_UPLOAD") + val separate: Boolean = false ) : NodeRequest, ServiceRequest, AuditableRequest From 2804cbbf71f29c7b6110d7dd81292774271048b7 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 11:52:36 +0800 Subject: [PATCH 14/28] =?UTF-8?q?fix:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../blocknode/impl/BlockNodeServiceImpl.kt | 3 +- .../service/node/impl/NodeBaseService.kt | 56 +++++++++++-------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index b458df526d..5cc2bcbe1f 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -126,7 +126,8 @@ class BlockNodeServiceImpl( .and(TBlockNode::uploadId.name).isEqualTo(uploadId) val update = BlockNodeQueryHelper.deleteUpdate() blockNodeDao.updateFirst(Query(criteria), update) - logger.info("Delete single node block[$projectId/$repoName$nodeFullPath] success. Id: $id, UPLOADID: $uploadId") + logger.info("Delete single node block[$projectId/$repoName$nodeFullPath] success. " + + "Id: $id, UPLOADID: $uploadId") } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index bfb163372c..1925e530ad 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -417,31 +417,41 @@ abstract class NodeBaseService( open fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { with(createRequest) { val existNode = nodeDao.findNode(projectId, repoName, fullPath) - if (existNode != null) { - if (!overwrite) { - throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, fullPath) - } else if (existNode.folder || this.folder) { - throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, fullPath) - } else if (separate) { - val currentVersion = createRequest.metadata!![UPLOADID_KEY].toString() - val oldNodeId = currentVersion.substringAfter("/") - if (oldNodeId == FAKE_SEPARATE){ - return - } - val deleteRes = deleteOldNode(projectId, repoName, fullPath, operator, oldNodeId) - if (deleteRes.deletedNumber == 0L){ - logger.warn("Delete block base node[$fullPath] by [$operator] error: node was deleted") - throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) - } - quotaService.decreaseUsedVolume(projectId, repoName, deleteRes.deletedSize) - } else { - val changeSize = this.size?.minus(existNode.size) ?: -existNode.size - quotaService.checkRepoQuota(projectId, repoName, changeSize) - deleteByFullPathWithoutDecreaseVolume(projectId, repoName, fullPath, operator) - quotaService.decreaseUsedVolume(projectId, repoName, existNode.size) + + // 如果节点不存在，进行配额检查后直接返回 + if (existNode == null) { + quotaService.checkRepoQuota(projectId, repoName, this.size ?: 0) + return + } + + // 如果不允许覆盖，抛出节点已存在异常 + if (!overwrite) { + throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, fullPath) + } + + // 如果存在文件夹冲突，抛出节点冲突异常 + if (existNode.folder || this.folder) { + throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, fullPath) + } + + // 根据 separate 参数执行不同的逻辑 + if (separate) { + val currentVersion = createRequest.metadata!![UPLOADID_KEY].toString() + val oldNodeId = currentVersion.substringAfter("/") + if (oldNodeId == FAKE_SEPARATE) { + return } + val deleteRes = deleteOldNode(projectId, repoName, fullPath, operator, oldNodeId) + if (deleteRes.deletedNumber == 0L) { + logger.warn("Delete block base node[$fullPath] by [$operator] error: node was deleted") + throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) + } + quotaService.decreaseUsedVolume(projectId, repoName, deleteRes.deletedSize) } else { - quotaService.checkRepoQuota(projectId, repoName, this.size ?: 0) + val changeSize = this.size?.minus(existNode.size) ?: -existNode.size + quotaService.checkRepoQuota(projectId, repoName, changeSize) + deleteByFullPathWithoutDecreaseVolume(projectId, repoName, fullPath, operator) + quotaService.decreaseUsedVolume(projectId, repoName, existNode.size) } } } From 020d69b3e733c37edbac1ca65bca925cdd4be1c3 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 12:09:44 +0800 Subject: [PATCH 15/28] =?UTF-8?q?fix:=20test=E6=96=B9=E6=B3=95=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../generic/service/BlockNodeServiceTest.kt | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt index 9ebdb0f7f1..f54f042288 100644 --- a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt +++ b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt @@ -135,7 +135,7 @@ class BlockNodeServiceTest { projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME, fullPath = "/file", - version = UT_VERSION + uploadId = UT_VERSION ) val deleteBlocksQuery = deleteBlocksQuery("/file", UT_PROJECT_ID, UT_REPO_NAME, createdDate) @@ -196,24 +196,24 @@ class BlockNodeServiceTest { createdBy = UT_USER, createdDate = LocalDateTime.now(), nodeFullPath = fullPath, - startPos = i.toLong(), + startPos = i * BLOCK_SIZE, sha256 = "$UT_SHA256$i", projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME, size = BLOCK_SIZE, - version = UT_VERSION + uploadId = UT_VERSION ) val artifactFile = createTempArtifactFile() storageService.store(blockNode.sha256, artifactFile, storageCredentials) blockNodeService.createBlock(blockNode, storageCredentials) } private fun listBlocks(createdDate: LocalDateTime, fullPath: String = "/file"): List { - return blockNodeService.listBlocksInVersion( + return blockNodeService.listBlocksInUploadId( projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME, fullPath = fullPath, createdDate = createdDate.toString(), - version = UT_VERSION, + uploadId = UT_VERSION, ) } @@ -225,16 +225,19 @@ class BlockNodeServiceTest { blocks.forEach { block -> Assertions.assertEquals(blockSize, block.size) Assertions.assertTrue(storageService.exist(block.sha256, storageCredentials)) - Assertions.assertEquals(block.version, version) + Assertions.assertEquals(block.uploadId, version) } } private fun completeUpload(blocks: List) { - var offset = 0L - blocks.sortedBy { it.startPos }.forEach { blockInfo -> - val blockSize = blockInfo.size - blockNodeService.updateBlock(blockInfo, offset, offset + blockSize - 1) - offset += blockSize // 更新偏移量 + val block = blocks.first() + block.uploadId?.let { + blockNodeService.updateBlockUploadId( + block.projectId, + block.repoName, + block.nodeFullPath, + it + ) } } From 71ac5047ca3424b3c057c454a598899e4da6681c Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 15:21:53 +0800 Subject: [PATCH 16/28] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/node/impl/center/CenterNodeServiceImpl.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index 0690393472..6fa10627a0 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.config.RepositoryProperties +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao import com.tencent.bkrepo.common.metadata.model.TMetadata @@ -174,6 +175,10 @@ class CenterNodeServiceImpl( override fun createNode(createRequest: NodeCreateRequest): NodeDetail { with(createRequest) { + if (sha256 == FAKE_SHA256 && separate) + { + return super.createNode(createRequest) + } val srcCluster = SecurityUtils.getClusterName() ?: clusterProperties.self.name.toString() val normalizeFullPath = PathUtils.normalizeFullPath(fullPath) val existNode = nodeDao.findNode(projectId, repoName, normalizeFullPath) From b7c980d55a8f1855cfa16f1e57a5ea6a42857569 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 15:51:41 +0800 Subject: [PATCH 17/28] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/bkrepo/generic/service/UploadService.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 6211cf8a1c..fe67d2720f 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -245,7 +245,6 @@ class UploadService( operator = userId, expires = getLongHeader(HEADER_EXPIRES), nodeMetadata = repository.resolveMetadata(HttpContextHolder.getRequest()), - separate = true ) ActionAuditContext.current().setInstance(request) nodeService.createNode(request) From 3ae6dacc62d47ae31195101abd0acdaa66ba9fd8 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 16:29:50 +0800 Subject: [PATCH 18/28] =?UTF-8?q?fix:=20=E6=96=87=E6=A1=A3=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/apidoc-user/generic/SeparateBlock.md | 267 +++++++++++------- docs/apidoc/generic/SeparateBlock.md | 267 +++++++++++------- .../repository/core/ArtifactRepository.kt | 1 - .../service/blocknode/BlockNodeService.kt | 2 +- .../bkrepo/generic/service/UploadService.kt | 2 +- 5 files changed, 322 insertions(+), 217 deletions(-) diff --git a/docs/apidoc-user/generic/SeparateBlock.md b/docs/apidoc-user/generic/SeparateBlock.md index 264f5cf39d..b93fa94f6d 100644 --- a/docs/apidoc-user/generic/SeparateBlock.md +++ b/docs/apidoc-user/generic/SeparateBlock.md @@ -1,32 +1,32 @@ -# Generic 通用制品仓库分块文件操作 +# 通用制品仓库分块文件操作指南 -[TOC] +[toc] ## 初始化分块上传 -- **API**: `POST /generic/separate/block/{project}/{repo}/{path}` -- **API 名称**: `start_block_upload` -- **功能说明**： +- **接口地址**: `POST /generic/separate/{project}/{repo}/{path}` +- **接口名称**: `start_block_upload` +- **功能说明**: - 中文：初始化分块上传 - - English: start block upload + - English: Start block upload -- **请求体** - - 此接口请求体为空。 +### 请求参数 -- **请求字段说明** +- **路径参数** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------- | ------- | -------- | ------ | ------------------ | --------------------- | - | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | overwrite exist file | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- | ------ | ----------------------- | ---------------------- | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite exist file | + +### 响应参数 - **响应体** @@ -35,44 +35,59 @@ "code": 0, "message": null, "data": { - "uploadId": "8be31384f82a45b0aafb6c6add29e94f/VERSION", + "uploadId": "8be31384f82a45b0aafb6c6add29e94f/xxxxxxxx", "expireSeconds": 43200 }, "traceId": null } ``` +- **字段说明** + + | 字段 | 类型 | 描述 | Description | + | ------------- | ------- | --------------------------- | -------------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | object | 返回数据 | Response data | + | ├─ uploadId | string | 分块上传 ID | Block upload ID | + | ├─ expireSeconds | int | 上传 ID 过期时间，单位：秒 | Upload ID expiration in seconds | + | traceId | string | 请求跟踪 ID | Trace ID | + --- ## 上传分块文件 -- **API**: `PUT /generic/{project}/{repo}/{path}` -- **API 名称**: `block_upload` -- **功能说明**： +- **接口地址**: `PUT /generic/{project}/{repo}/{path}` +- **接口名称**: `block_upload` +- **功能说明**: - 中文：分块上传通用制品文件 - - English: upload generic artifact file block + - English: Upload generic artifact file block -- **请求体** - - [文件流] +### 请求参数 -- **请求字段说明** +- **路径参数** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------- | ------- | -------- | ------ | -------------------------------------------------------------------- | ------------------------------ | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | - | X-BKREPO-SEQUENCE | int | 是 | 无 | 分块序号（从 1 开始），`SEQUENCE` 和 `OFFSET` 二者不可同时为空 | block sequence (start from 1) | - | X-BKREPO-OFFSET | int | 是 | 无 | 分块偏移量，`SEQUENCE` 和 `OFFSET` 二者不可同时为空 | block offset (start from 0) | - | X-BKREPO-SHA256 | string | 否 | 无 | 文件 SHA256 | file sha256 | - | X-BKREPO-MD5 | string | 否 | 无 | 文件 MD5 | file md5 | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- | ------ | -------------------------------------------------- | ------------------------------------ | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | X-BKREPO-OFFSET | int | 是 | 无 | 分块偏移量，起始值为 0 | Block offset (start from 0) | + | X-BKREPO-SHA256 | string | 否 | 无 | 分块文件的 SHA256 校验值 | SHA256 checksum of the block | + | X-BKREPO-MD5 | string | 否 | 无 | 分块文件的 MD5 校验值 | MD5 checksum of the block | + | UPLOAD-TYPE | string | 是 | 无 | 上传类型，值为 `SEPARATE-UPLOAD` | Upload type (`SEPARATE-UPLOAD`) | + +- **请求体** + + - 文件流（binary data） + +### 响应参数 - **响应体** @@ -85,35 +100,48 @@ } ``` +- **字段说明** + + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | --------------------------- | -------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | + --- ## 完成分块上传 -- **API**: `PUT /generic/separate/block/{project}/{repo}/{path}` -- **API 名称**: `complete_block_upload` -- **功能说明**： +- **接口地址**: `PUT /generic/separate/{project}/{repo}/{path}` +- **接口名称**: `complete_block_upload` +- **功能说明**: - 中文：完成分块上传 - - English: complete block upload + - English: Complete block upload -- **请求体** +### 请求参数 - 此接口请求体为空。 +- **路径参数** -- **请求字段说明** - - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------ | ------- | -------- | ------ | ------------------ | ------------------------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | - | X-BKREPO-SIZE | string | 是 | 0L | 文件大小 | total size of the uploaded file | - | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | overwrite exist file | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- | ------ | ----------------------- | ---------------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | X-BKREPO-SIZE | string | 是 | 0 | 文件总大小 | Total size of the file | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite exist file | + +- **请求体** + + - 此接口请求体为空。 + +### 响应参数 - **响应体** @@ -126,33 +154,46 @@ } ``` +- **字段说明** + + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | --------------------------- | -------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | + --- ## 终止（取消）分块上传 -- **API**: `DELETE /generic/separate/block/{project}/{repo}/{path}` -- **API 名称**: `abort_block_upload` -- **功能说明**： +- **接口地址**: `DELETE /generic/separate/{project}/{repo}/{path}` +- **接口名称**: `abort_block_upload` +- **功能说明**: - 中文：终止（取消）分块上传 - - English: abort block upload - -- **请求体** + - English: Abort block upload - 此接口请求体为空。 +### 请求参数 -- **请求字段说明** +- **路径参数** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------ | ------ | -------- | ------ | ----------- | ----------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------ | ---- | ------ | ------------- | ---------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + +- **请求体** + + - 此接口请求体为空。 + +### 响应参数 - **响应体** @@ -165,34 +206,46 @@ } ``` +- **字段说明** + + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | --------------------------- | -------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | + --- ## 查询已上传的分块列表 -- **API**: `GET /generic/separate/block/{project}/{repo}/{path}` -- **API 名称**: `list_upload_block` -- **功能说明**： +- **接口地址**: `GET /generic/separate/{project}/{repo}/{path}` +- **接口名称**: `list_upload_block` +- **功能说明**: - 中文：查询已上传的分块列表 - - English: list uploaded blocks - -- **请求体** + - English: List uploaded blocks - 此接口请求体为空。 +### 请求参数 -- **请求字段说明** +- **路径参数** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------ | ------ | -------- | ------ | ------------------------------------------------------------------------- | -------------------- | - | Authorization | string | 否 | 无 | Basic Auth 认证头，格式：`Basic base64(username:password)` | Basic Auth header | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------ | ---- | ------ | ------------- | ---------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + +- **请求体** + + - 此接口请求体为空。 + +### 响应参数 - **响应体** @@ -203,37 +256,37 @@ "data": [ { "size": 10240, - "sha256": "d17f25ecfbcc7857f7bebea469308be0b2580943e96d13a3ad98a13675c4bfc2", + "sha256": "000000000000000000000000000000000000000000000", "startPos": 0, - "version": "1.0" + "uploadId": "1.0" }, { "size": 10240, - "sha256": "cc399d73903f06ee694032ab0538f05634ff7e1ce5e8e50ac330a871484f34cf", + "sha256": "000000000000000000000000000000000000000000000", "startPos": 10240, - "version": "1.0" + "uploadId": "1.0" } ], "traceId": null } ``` -- **响应字段说明** +- **字段说明** - | 字段 | 类型 | 说明 | Description | - | -------- | ------ | ------------------------------------- | -------------------------- | - | code | int | 错误编码。0 表示成功，>0 表示失败错误 | 0: success, other: failure | - | message | string | 错误消息 | the failure message | - | data | list | 分块列表 | block list | - | traceId | string | 请求跟踪 id | trace id | + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | --------------------------- | -------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | array | 分块列表 | Block list | + | traceId | string | 请求跟踪 ID | Trace ID | - **分块信息字段说明** - | 字段 | 类型 | 说明 | Description | - | --------- | ------ | ------------ | -------------------------- | - | size | long | 分块大小 | block size | - | sha256 | string | 分块 SHA256 | block sha256 checksum | - | startPos | long | 分块起始位置 | block start position | - | version | string | 分块版本信息 | block version | + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | -------------- | -------------------------- | + | size | long | 分块大小 | Block size | + | sha256 | string | 分块 SHA256 值 | Block SHA256 checksum | + | startPos | long | 分块起始位置 | Block start position | + | uploadId | string | 分块版本信息 | Block upload ID | --- \ No newline at end of file diff --git a/docs/apidoc/generic/SeparateBlock.md b/docs/apidoc/generic/SeparateBlock.md index 264f5cf39d..b93fa94f6d 100644 --- a/docs/apidoc/generic/SeparateBlock.md +++ b/docs/apidoc/generic/SeparateBlock.md @@ -1,32 +1,32 @@ -# Generic 通用制品仓库分块文件操作 +# 通用制品仓库分块文件操作指南 -[TOC] +[toc] ## 初始化分块上传 -- **API**: `POST /generic/separate/block/{project}/{repo}/{path}` -- **API 名称**: `start_block_upload` -- **功能说明**： +- **接口地址**: `POST /generic/separate/{project}/{repo}/{path}` +- **接口名称**: `start_block_upload` +- **功能说明**: - 中文：初始化分块上传 - - English: start block upload + - English: Start block upload -- **请求体** - - 此接口请求体为空。 +### 请求参数 -- **请求字段说明** +- **路径参数** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------- | ------- | -------- | ------ | ------------------ | --------------------- | - | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | overwrite exist file | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- | ------ | ----------------------- | ---------------------- | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite exist file | + +### 响应参数 - **响应体** @@ -35,44 +35,59 @@ "code": 0, "message": null, "data": { - "uploadId": "8be31384f82a45b0aafb6c6add29e94f/VERSION", + "uploadId": "8be31384f82a45b0aafb6c6add29e94f/xxxxxxxx", "expireSeconds": 43200 }, "traceId": null } ``` +- **字段说明** + + | 字段 | 类型 | 描述 | Description | + | ------------- | ------- | --------------------------- | -------------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | object | 返回数据 | Response data | + | ├─ uploadId | string | 分块上传 ID | Block upload ID | + | ├─ expireSeconds | int | 上传 ID 过期时间，单位：秒 | Upload ID expiration in seconds | + | traceId | string | 请求跟踪 ID | Trace ID | + --- ## 上传分块文件 -- **API**: `PUT /generic/{project}/{repo}/{path}` -- **API 名称**: `block_upload` -- **功能说明**： +- **接口地址**: `PUT /generic/{project}/{repo}/{path}` +- **接口名称**: `block_upload` +- **功能说明**: - 中文：分块上传通用制品文件 - - English: upload generic artifact file block + - English: Upload generic artifact file block -- **请求体** - - [文件流] +### 请求参数 -- **请求字段说明** +- **路径参数** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------- | ------- | -------- | ------ | -------------------------------------------------------------------- | ------------------------------ | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | - | X-BKREPO-SEQUENCE | int | 是 | 无 | 分块序号（从 1 开始），`SEQUENCE` 和 `OFFSET` 二者不可同时为空 | block sequence (start from 1) | - | X-BKREPO-OFFSET | int | 是 | 无 | 分块偏移量，`SEQUENCE` 和 `OFFSET` 二者不可同时为空 | block offset (start from 0) | - | X-BKREPO-SHA256 | string | 否 | 无 | 文件 SHA256 | file sha256 | - | X-BKREPO-MD5 | string | 否 | 无 | 文件 MD5 | file md5 | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- | ------ | -------------------------------------------------- | ------------------------------------ | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | X-BKREPO-OFFSET | int | 是 | 无 | 分块偏移量，起始值为 0 | Block offset (start from 0) | + | X-BKREPO-SHA256 | string | 否 | 无 | 分块文件的 SHA256 校验值 | SHA256 checksum of the block | + | X-BKREPO-MD5 | string | 否 | 无 | 分块文件的 MD5 校验值 | MD5 checksum of the block | + | UPLOAD-TYPE | string | 是 | 无 | 上传类型，值为 `SEPARATE-UPLOAD` | Upload type (`SEPARATE-UPLOAD`) | + +- **请求体** + + - 文件流（binary data） + +### 响应参数 - **响应体** @@ -85,35 +100,48 @@ } ``` +- **字段说明** + + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | --------------------------- | -------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | + --- ## 完成分块上传 -- **API**: `PUT /generic/separate/block/{project}/{repo}/{path}` -- **API 名称**: `complete_block_upload` -- **功能说明**： +- **接口地址**: `PUT /generic/separate/{project}/{repo}/{path}` +- **接口名称**: `complete_block_upload` +- **功能说明**: - 中文：完成分块上传 - - English: complete block upload + - English: Complete block upload -- **请求体** +### 请求参数 - 此接口请求体为空。 +- **路径参数** -- **请求字段说明** - - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------ | ------- | -------- | ------ | ------------------ | ------------------------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | - | X-BKREPO-SIZE | string | 是 | 0L | 文件大小 | total size of the uploaded file | - | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | overwrite exist file | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- | ------ | ----------------------- | ---------------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | X-BKREPO-SIZE | string | 是 | 0 | 文件总大小 | Total size of the file | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite exist file | + +- **请求体** + + - 此接口请求体为空。 + +### 响应参数 - **响应体** @@ -126,33 +154,46 @@ } ``` +- **字段说明** + + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | --------------------------- | -------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | + --- ## 终止（取消）分块上传 -- **API**: `DELETE /generic/separate/block/{project}/{repo}/{path}` -- **API 名称**: `abort_block_upload` -- **功能说明**： +- **接口地址**: `DELETE /generic/separate/{project}/{repo}/{path}` +- **接口名称**: `abort_block_upload` +- **功能说明**: - 中文：终止（取消）分块上传 - - English: abort block upload - -- **请求体** + - English: Abort block upload - 此接口请求体为空。 +### 请求参数 -- **请求字段说明** +- **路径参数** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------ | ------ | -------- | ------ | ----------- | ----------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------ | ---- | ------ | ------------- | ---------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + +- **请求体** + + - 此接口请求体为空。 + +### 响应参数 - **响应体** @@ -165,34 +206,46 @@ } ``` +- **字段说明** + + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | --------------------------- | -------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | + --- ## 查询已上传的分块列表 -- **API**: `GET /generic/separate/block/{project}/{repo}/{path}` -- **API 名称**: `list_upload_block` -- **功能说明**： +- **接口地址**: `GET /generic/separate/{project}/{repo}/{path}` +- **接口名称**: `list_upload_block` +- **功能说明**: - 中文：查询已上传的分块列表 - - English: list uploaded blocks - -- **请求体** + - English: List uploaded blocks - 此接口请求体为空。 +### 请求参数 -- **请求字段说明** +- **路径参数** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------- | ------ | -------- | ------ | -------- | ------------ | - | project | string | 是 | 无 | 项目名称 | project name | - | repo | string | 是 | 无 | 仓库名称 | repo name | - | path | string | 是 | 无 | 完整路径 | full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ---------------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 是否必须 | 默认值 | 说明 | Description | - | ------------------ | ------ | -------- | ------ | ------------------------------------------------------------------------- | -------------------- | - | Authorization | string | 否 | 无 | Basic Auth 认证头，格式：`Basic base64(username:password)` | Basic Auth header | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 id | block upload id | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------ | ---- | ------ | ------------- | ---------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + +- **请求体** + + - 此接口请求体为空。 + +### 响应参数 - **响应体** @@ -203,37 +256,37 @@ "data": [ { "size": 10240, - "sha256": "d17f25ecfbcc7857f7bebea469308be0b2580943e96d13a3ad98a13675c4bfc2", + "sha256": "000000000000000000000000000000000000000000000", "startPos": 0, - "version": "1.0" + "uploadId": "1.0" }, { "size": 10240, - "sha256": "cc399d73903f06ee694032ab0538f05634ff7e1ce5e8e50ac330a871484f34cf", + "sha256": "000000000000000000000000000000000000000000000", "startPos": 10240, - "version": "1.0" + "uploadId": "1.0" } ], "traceId": null } ``` -- **响应字段说明** +- **字段说明** - | 字段 | 类型 | 说明 | Description | - | -------- | ------ | ------------------------------------- | -------------------------- | - | code | int | 错误编码。0 表示成功，>0 表示失败错误 | 0: success, other: failure | - | message | string | 错误消息 | the failure message | - | data | list | 分块列表 | block list | - | traceId | string | 请求跟踪 id | trace id | + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | --------------------------- | -------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, other: failure | + | message | string | 错误消息 | The failure message | + | data | array | 分块列表 | Block list | + | traceId | string | 请求跟踪 ID | Trace ID | - **分块信息字段说明** - | 字段 | 类型 | 说明 | Description | - | --------- | ------ | ------------ | -------------------------- | - | size | long | 分块大小 | block size | - | sha256 | string | 分块 SHA256 | block sha256 checksum | - | startPos | long | 分块起始位置 | block start position | - | version | string | 分块版本信息 | block version | + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | -------------- | -------------------------- | + | size | long | 分块大小 | Block size | + | sha256 | string | 分块 SHA256 值 | Block SHA256 checksum | + | startPos | long | 分块起始位置 | Block start position | + | uploadId | string | 分块版本信息 | Block upload ID | --- \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt index 3a9a34505b..578badc2f2 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/ArtifactRepository.kt @@ -84,5 +84,4 @@ interface ArtifactRepository { * @param context 构件迁移上下文 */ fun migrate(context: ArtifactMigrateContext): MigrateDetail - } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index 1c997772fb..99f62c5335 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -79,7 +79,7 @@ interface BlockNodeService { /** * 删除旧分块，即删除非指定的nodeCurrentSha256的分块。 * 如果未指定nodeCurrentSha256，则删除节点所有分块 - * 如果指定version，则删除该版本对应的分块，未指定则删除所有分块 + * 如果指定uploadId，则删除该uploadId对应的分块，未指定则删除所有分块 * @param projectId 项目id * @param repoName 仓库名 * @param fullPath 文件路径 diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index fe67d2720f..f6f7b11cf0 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -271,7 +271,7 @@ class UploadService( artifactInfo.getArtifactFullPath() ) - // 获取节点并验证版本信息 + // 获取节点并验证uploadId信息 val node = ArtifactContextHolder.getNodeDetail(artifactInfo) ?.takeIf { it.metadata[UPLOADID_KEY] == uploadId } ?: run { From f33e49cbdaa2b3a2cb18f4d8d8b96a1ae49f6775 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 17:31:11 +0800 Subject: [PATCH 19/28] =?UTF-8?q?fix:=20=E5=A2=9E=E5=8A=A0=E9=83=A8?= =?UTF-8?q?=E5=88=86log=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/metadata/service/node/impl/NodeBaseService.kt | 3 ++- .../common/metadata/service/node/impl/NodeDeleteSupport.kt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index 1925e530ad..e0a133cc96 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -368,10 +368,11 @@ abstract class NodeBaseService( quotaService.increaseUsedVolume(node.projectId, node.repoName, node.size) } } catch (exception: DuplicateKeyException) { - logger.warn("Insert node[$node] error: [${exception.message}]") if (separate){ + logger.warn("Insert block base node[$node] error: [${exception.message}]") throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, node.fullPath) } + logger.warn("Insert node[$node] error: [${exception.message}]") } return node diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 3592d09575..5677f01111 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -214,6 +214,7 @@ open class NodeDeleteSupport( }) quotaService.decreaseUsedVolume(projectId, repoName, deletedSize) + logger.info("Delete old block base node: $fullPath, operator: $operator, delete time: $deleteTime success") return NodeDeleteResult(deletedNum, deletedSize, deleteTime) } From 637fac27cba8e9fe4b1e69f44abd16772b440bf8 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 18:15:06 +0800 Subject: [PATCH 20/28] =?UTF-8?q?fix:=20=E8=B0=83=E8=AF=95=E6=97=A5?= =?UTF-8?q?=E5=BF=97=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/common/metadata/service/node/impl/NodeBaseService.kt | 1 + .../metadata/service/node/impl/center/CenterNodeServiceImpl.kt | 1 + 2 files changed, 2 insertions(+) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index e0a133cc96..4d17b4d09b 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -417,6 +417,7 @@ abstract class NodeBaseService( open fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { with(createRequest) { + logger.info("checkConflictAndQuota was be called: $fullPath, separate: $separate") val existNode = nodeDao.findNode(projectId, repoName, fullPath) // 如果节点不存在，进行配额检查后直接返回 diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index 6fa10627a0..edaf363db8 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -177,6 +177,7 @@ class CenterNodeServiceImpl( with(createRequest) { if (sha256 == FAKE_SHA256 && separate) { + logger.info("Create block base node by node base service") return super.createNode(createRequest) } val srcCluster = SecurityUtils.getClusterName() ?: clusterProperties.self.name.toString() From 5cbba618009680938bbb24defdc4225e9bad0aed Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 19:54:45 +0800 Subject: [PATCH 21/28] =?UTF-8?q?fix:=20=E8=B0=83=E8=AF=95=E6=97=A5?= =?UTF-8?q?=E5=BF=97=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/metadata/service/node/impl/NodeBaseService.kt | 6 +++++- .../service/node/impl/center/CenterNodeServiceImpl.kt | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index 4d17b4d09b..07d266d309 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -185,6 +185,7 @@ abstract class NodeBaseService( @Transactional(rollbackFor = [Throwable::class]) override fun createNode(createRequest: NodeCreateRequest): NodeDetail { with(createRequest) { + logger.info("Create block base node by node base service") val fullPath = PathUtils.normalizeFullPath(fullPath) Preconditions.checkArgument(!PathUtils.isRoot(fullPath), this::fullPath.name) Preconditions.checkArgument(folder || !sha256.isNullOrBlank(), this::sha256.name) @@ -192,11 +193,13 @@ abstract class NodeBaseService( // 仓库是否存在 val repo = checkRepo(projectId, repoName) // 路径唯一性校验 + logger.info("checkConflictAndQuota was be called before: $fullPath, separate: $separate") checkConflictAndQuota(createRequest, fullPath) // 判断父目录是否存在，不存在先创建 mkdirs(projectId, repoName, PathUtils.resolveParent(fullPath), operator) // 创建节点 val node = buildTNode(this) + logger.info("Ready to create node[$node].") doCreate(node, separate = separate) afterCreate(repo, node) logger.info("Create node[/$projectId/$repoName$fullPath], sha256[$sha256] success.") @@ -359,6 +362,7 @@ abstract class NodeBaseService( open fun doCreate(node: TNode, repository: TRepository? = null, separate: Boolean = false): TNode { try { + logger.info("Do create node in NodeBaseService") nodeDao.insert(node) if (!node.folder) { // 软链接node或fs-server创建的node的sha256为FAKE_SHA256不会关联实际文件，无需增加引用数 @@ -417,7 +421,7 @@ abstract class NodeBaseService( open fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { with(createRequest) { - logger.info("checkConflictAndQuota was be called: $fullPath, separate: $separate") + logger.info("checkConflictAndQuota was be called in NodeBaseService: $fullPath, separate: $separate") val existNode = nodeDao.findNode(projectId, repoName, fullPath) // 如果节点不存在，进行配额检查后直接返回 diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index edaf363db8..9fd8b6f896 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -119,6 +119,7 @@ class CenterNodeServiceImpl( override fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { with(createRequest) { + logger.info("checkConflictAndQuota was be called in CenterNodeService: $fullPath, separate: $separate") val existNode = nodeDao.findNode(projectId, repoName, fullPath) if (existNode != null) { if (!overwrite) { @@ -158,7 +159,9 @@ class CenterNodeServiceImpl( } override fun doCreate(node: TNode, repository: TRepository?, separate: Boolean): TNode { + logger.info("Do create node in CenterNodeServiceImpl") if (SecurityUtils.getClusterName().isNullOrBlank()) { + logger.info("Create node in super do create") return super.doCreate(node, repository, false) } try { @@ -177,7 +180,6 @@ class CenterNodeServiceImpl( with(createRequest) { if (sha256 == FAKE_SHA256 && separate) { - logger.info("Create block base node by node base service") return super.createNode(createRequest) } val srcCluster = SecurityUtils.getClusterName() ?: clusterProperties.self.name.toString() From 646950651a8a5b63eb13497500459e3afec29c62 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 6 Jan 2025 21:37:42 +0800 Subject: [PATCH 22/28] =?UTF-8?q?fix:=20=E9=97=AE=E9=A2=98=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/node/impl/NodeBaseService.kt | 18 +++++--- .../node/impl/center/CenterNodeServiceImpl.kt | 43 ++++++------------- 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index 07d266d309..28df8cd02a 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -185,7 +185,6 @@ abstract class NodeBaseService( @Transactional(rollbackFor = [Throwable::class]) override fun createNode(createRequest: NodeCreateRequest): NodeDetail { with(createRequest) { - logger.info("Create block base node by node base service") val fullPath = PathUtils.normalizeFullPath(fullPath) Preconditions.checkArgument(!PathUtils.isRoot(fullPath), this::fullPath.name) Preconditions.checkArgument(folder || !sha256.isNullOrBlank(), this::sha256.name) @@ -193,13 +192,11 @@ abstract class NodeBaseService( // 仓库是否存在 val repo = checkRepo(projectId, repoName) // 路径唯一性校验 - logger.info("checkConflictAndQuota was be called before: $fullPath, separate: $separate") checkConflictAndQuota(createRequest, fullPath) // 判断父目录是否存在，不存在先创建 mkdirs(projectId, repoName, PathUtils.resolveParent(fullPath), operator) // 创建节点 val node = buildTNode(this) - logger.info("Ready to create node[$node].") doCreate(node, separate = separate) afterCreate(repo, node) logger.info("Create node[/$projectId/$repoName$fullPath], sha256[$sha256] success.") @@ -362,7 +359,6 @@ abstract class NodeBaseService( open fun doCreate(node: TNode, repository: TRepository? = null, separate: Boolean = false): TNode { try { - logger.info("Do create node in NodeBaseService") nodeDao.insert(node) if (!node.folder) { // 软链接node或fs-server创建的node的sha256为FAKE_SHA256不会关联实际文件，无需增加引用数 @@ -421,7 +417,6 @@ abstract class NodeBaseService( open fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { with(createRequest) { - logger.info("checkConflictAndQuota was be called in NodeBaseService: $fullPath, separate: $separate") val existNode = nodeDao.findNode(projectId, repoName, fullPath) // 如果节点不存在，进行配额检查后直接返回 @@ -440,18 +435,25 @@ abstract class NodeBaseService( throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, fullPath) } + // 子类的附加检查方法 + additionalCheck(existNode) + // 根据 separate 参数执行不同的逻辑 if (separate) { - val currentVersion = createRequest.metadata!![UPLOADID_KEY].toString() + // 删除旧节点 并 检查旧节点是否删除 防止并发删除 + val currentVersion = metadata!![UPLOADID_KEY].toString() val oldNodeId = currentVersion.substringAfter("/") + if (oldNodeId == FAKE_SEPARATE) { return } + val deleteRes = deleteOldNode(projectId, repoName, fullPath, operator, oldNodeId) if (deleteRes.deletedNumber == 0L) { logger.warn("Delete block base node[$fullPath] by [$operator] error: node was deleted") throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) } + logger.info("Delete block base node[$fullPath] by [$operator] success: $oldNodeId.") quotaService.decreaseUsedVolume(projectId, repoName, deleteRes.deletedSize) } else { val changeSize = this.size?.minus(existNode.size) ?: -existNode.size @@ -462,6 +464,10 @@ abstract class NodeBaseService( } } + open fun additionalCheck(existNode: TNode) { + // 默认不做任何操作 + } + private fun incrementFileReference(node: TNode, repository: TRepository?): Boolean { if (!validateParameter(node)) return false return try { diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index 9fd8b6f896..c92e52e1d1 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -45,6 +45,8 @@ import com.tencent.bkrepo.common.metadata.model.TRepository import com.tencent.bkrepo.common.metadata.pojo.node.RestoreContext import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService +import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService.Companion import com.tencent.bkrepo.common.metadata.service.node.impl.NodeServiceImpl import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.QuotaService @@ -117,28 +119,6 @@ class CenterNodeServiceImpl( return repo } - override fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { - with(createRequest) { - logger.info("checkConflictAndQuota was be called in CenterNodeService: $fullPath, separate: $separate") - val existNode = nodeDao.findNode(projectId, repoName, fullPath) - if (existNode != null) { - if (!overwrite) { - throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, fullPath) - } else if (existNode.folder || this.folder) { - throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, fullPath) - } else { - ClusterUtils.checkIsSrcCluster(existNode.clusterNames) - val changeSize = this.size?.minus(existNode.size) ?: -existNode.size - quotaService.checkRepoQuota(projectId, repoName, changeSize) - deleteByFullPathWithoutDecreaseVolume(projectId, repoName, fullPath, operator) - quotaService.decreaseUsedVolume(projectId, repoName, existNode.size) - } - } else { - quotaService.checkRepoQuota(projectId, repoName, this.size ?: 0) - } - } - } - override fun deleteByFullPathWithoutDecreaseVolume( projectId: String, repoName: String, fullPath: String, operator: String ) { @@ -159,10 +139,8 @@ class CenterNodeServiceImpl( } override fun doCreate(node: TNode, repository: TRepository?, separate: Boolean): TNode { - logger.info("Do create node in CenterNodeServiceImpl") if (SecurityUtils.getClusterName().isNullOrBlank()) { - logger.info("Create node in super do create") - return super.doCreate(node, repository, false) + return super.doCreate(node, repository, separate) } try { nodeDao.insert(node) @@ -170,6 +148,10 @@ class CenterNodeServiceImpl( quotaService.increaseUsedVolume(node.projectId, node.repoName, node.size) } } catch (exception: DuplicateKeyException) { + if (separate){ + logger.warn("Insert block base node[$node] error: [${exception.message}]") + throw ErrorCodeException(ArtifactMessageCode.NODE_CONFLICT, node.fullPath) + } logger.warn("Insert node[$node] error: [${exception.message}]") } @@ -178,15 +160,11 @@ class CenterNodeServiceImpl( override fun createNode(createRequest: NodeCreateRequest): NodeDetail { with(createRequest) { - if (sha256 == FAKE_SHA256 && separate) - { - return super.createNode(createRequest) - } val srcCluster = SecurityUtils.getClusterName() ?: clusterProperties.self.name.toString() val normalizeFullPath = PathUtils.normalizeFullPath(fullPath) val existNode = nodeDao.findNode(projectId, repoName, normalizeFullPath) ?: return super.createNode(createRequest) - if (sha256 == existNode.sha256) { + if (sha256 == existNode.sha256 && sha256 != FAKE_SHA256) { val clusterNames = existNode.clusterNames.orEmpty().toMutableSet() clusterNames.add(srcCluster) val query = NodeQueryHelper.nodeQuery(projectId, repoName, normalizeFullPath) @@ -277,6 +255,11 @@ class CenterNodeServiceImpl( return CenterNodeRenameSupport(this, clusterProperties).renameNode(renameRequest) } + override fun additionalCheck(existNode: TNode) { + // center 检查请求来源cluster是否是资源的唯一拥有者 + ClusterUtils.checkIsSrcCluster(existNode.clusterNames) + } + companion object { private val logger = LoggerFactory.getLogger(CenterNodeServiceImpl::class.java) From 28ba5d7f30d1b5a9a2d130d1d1ec867918471795 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Tue, 7 Jan 2025 11:06:31 +0800 Subject: [PATCH 23/28] =?UTF-8?q?fix:=20=E9=97=AE=E9=A2=98=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../metadata/service/node/impl/center/CenterNodeServiceImpl.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index c92e52e1d1..73ad28c786 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -45,8 +45,6 @@ import com.tencent.bkrepo.common.metadata.model.TRepository import com.tencent.bkrepo.common.metadata.pojo.node.RestoreContext import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService -import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService -import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService.Companion import com.tencent.bkrepo.common.metadata.service.node.impl.NodeServiceImpl import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.QuotaService From 7b5343583200fdb606cf898ae54e02d725b50a5d Mon Sep 17 00:00:00 2001 From: zzdjx Date: Thu, 9 Jan 2025 18:18:27 +0800 Subject: [PATCH 24/28] =?UTF-8?q?feat:=20blocks=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E8=BF=87=E6=9C=9F=E9=80=BB=E8=BE=91=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/apidoc-user/generic/SeparateBlock.md | 202 ++++++++++-------- docs/apidoc/generic/SeparateBlock.md | 202 ++++++++++-------- .../core/AbstractArtifactRepository.kt | 4 - .../common/metadata/model/TBlockNode.kt | 3 +- .../service/blocknode/BlockNodeService.kt | 5 - .../blocknode/impl/BlockNodeServiceImpl.kt | 18 +- .../service/node/NodeDeleteOperation.kt | 3 +- .../service/node/impl/NodeBaseService.kt | 17 +- .../service/node/impl/NodeDeleteSupport.kt | 6 +- .../service/node/impl/NodeServiceImpl.kt | 4 +- .../impl/center/CenterNodeDeleteSupport.kt | 49 ++++- .../node/impl/center/CenterNodeServiceImpl.kt | 16 ++ .../node/impl/edge/EdgeNodeServiceImpl.kt | 4 +- .../metadata/util/BlockNodeQueryHelper.kt | 8 +- .../generic/constant/GenericMessageCode.kt | 2 +- .../bkrepo/generic/pojo/SeparateBlockInfo.kt | 4 +- .../artifact/GenericLocalRepository.kt | 38 ++-- .../bkrepo/generic/service/UploadService.kt | 22 +- .../resources/i18n/messages_en.properties | 2 +- .../resources/i18n/messages_zh_CN.properties | 2 +- .../resources/i18n/messages_zh_TW.properties | 2 +- .../generic/service/BlockNodeServiceTest.kt | 12 +- .../task/other/ExpiredBlockNodeMarkupJob.kt | 91 ++++++++ .../ExpiredBlockNodeMarkupJobProperties.kt | 8 + 24 files changed, 444 insertions(+), 280 deletions(-) create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredBlockNodeMarkupJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ExpiredBlockNodeMarkupJobProperties.kt diff --git a/docs/apidoc-user/generic/SeparateBlock.md b/docs/apidoc-user/generic/SeparateBlock.md index b93fa94f6d..fc2efd6a59 100644 --- a/docs/apidoc-user/generic/SeparateBlock.md +++ b/docs/apidoc-user/generic/SeparateBlock.md @@ -1,8 +1,8 @@ # 通用制品仓库分块文件操作指南 -[toc] +[TOC] -## 初始化分块上传 +## 一、初始化分块上传 - **接口地址**: `POST /generic/separate/{project}/{repo}/{path}` - **接口名称**: `start_block_upload` @@ -14,17 +14,18 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------- | ---- | ------ | ----------------------- | ---------------------- | - | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite exist file | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- |-------| ------------------------- | --------------------------------- | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite existing file | + | X-BKREPO-EXPIRES | long | 否 | 3600 * 8 | 上传 ID 过期时间，单位秒） | Upload ID expiration in seconds | ### 响应参数 @@ -44,18 +45,18 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | ------------- | ------- | --------------------------- | -------------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | object | 返回数据 | Response data | - | ├─ uploadId | string | 分块上传 ID | Block upload ID | - | ├─ expireSeconds | int | 上传 ID 过期时间，单位：秒 | Upload ID expiration in seconds | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | --------------- | ------- | ------------------------------- | ------------------------------------ | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | object | 返回数据 | Response data | + | ├── uploadId | string | 分块上传 ID | Block upload ID | + | ├── expireSeconds | long | 上传 ID 过期时间，单位：秒 | Upload ID expiration in seconds | + | traceId | string | 请求跟踪 ID | Trace ID | --- -## 上传分块文件 +## 二、上传分块文件 - **接口地址**: `PUT /generic/{project}/{repo}/{path}` - **接口名称**: `block_upload` @@ -67,25 +68,26 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------- | ---- | ------ | -------------------------------------------------- | ------------------------------------ | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | - | X-BKREPO-OFFSET | int | 是 | 无 | 分块偏移量，起始值为 0 | Block offset (start from 0) | - | X-BKREPO-SHA256 | string | 否 | 无 | 分块文件的 SHA256 校验值 | SHA256 checksum of the block | - | X-BKREPO-MD5 | string | 否 | 无 | 分块文件的 MD5 校验值 | MD5 checksum of the block | - | UPLOAD-TYPE | string | 是 | 无 | 上传类型，值为 `SEPARATE-UPLOAD` | Upload type (`SEPARATE-UPLOAD`) | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- |-----| --------------- | ------------------------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | X-BKREPO-OFFSET | long | 是 | 无 | 分块偏移量，起始值为 0 | Block offset (starting from 0) | + | X-BKREPO-SHA256 | string | 否 | 无 | 分块文件的 SHA256 校验值 | SHA256 checksum of the block | + | X-BKREPO-MD5 | string | 否 | 无 | 分块文件的 MD5 校验值 | MD5 checksum of the block | + | UPLOAD-TYPE | string | 是 | 无 | 上传类型，值为 `SEPARATE-UPLOAD` | Upload type (`SEPARATE-UPLOAD`) | + | X-BKREPO-EXPIRES | long | 否 | 3600 * 8 | 分块过期时间，单位秒| Block expiration in seconds | - **请求体** - - 文件流（binary data） + - 文件流（二进制数据） ### 响应参数 @@ -102,16 +104,16 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | --------------------------- | -------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | null | 返回数据（为空） | Response data (null) | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | ------- | ------ | ------------------------------- | ---------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | --- -## 完成分块上传 +## 三、完成分块上传 - **接口地址**: `PUT /generic/separate/{project}/{repo}/{path}` - **接口名称**: `complete_block_upload` @@ -123,19 +125,19 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------- | ---- | ------ | ----------------------- | ---------------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | - | X-BKREPO-SIZE | string | 是 | 0 | 文件总大小 | Total size of the file | - | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite exist file | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- | ------ | ------------------------------- | ---------------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | X-BKREPO-SIZE | long | 是 | 0 | 文件总大小 | Total size of the file | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite existing file| - **请求体** @@ -156,16 +158,16 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | --------------------------- | -------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | null | 返回数据（为空） | Response data (null) | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | ------- | ------ | ------------------------------- | ---------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | --- -## 终止（取消）分块上传 +## 四、终止（取消）分块上传 - **接口地址**: `DELETE /generic/separate/{project}/{repo}/{path}` - **接口名称**: `abort_block_upload` @@ -177,17 +179,17 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------ | ---- | ------ | ------------- | ---------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------ | ---- | ------ | --------------------- | ---------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | - **请求体** @@ -208,19 +210,19 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | --------------------------- | -------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | null | 返回数据（为空） | Response data (null) | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | ------- | ------ | ------------------------------- | ---------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | --- -## 查询已上传的分块列表 +## 五、查询已上传的分块列表 - **接口地址**: `GET /generic/separate/{project}/{repo}/{path}` -- **接口名称**: `list_upload_block` +- **接口名称**: `list_uploaded_blocks` - **功能说明**: - 中文：查询已上传的分块列表 - English: List uploaded blocks @@ -229,17 +231,17 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------ | ---- | ------ | ------------- | ---------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------ | ---- | ------ | --------------------- | ---------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | - **请求体** @@ -256,13 +258,13 @@ "data": [ { "size": 10240, - "sha256": "000000000000000000000000000000000000000000000", + "sha256": "abc123def456...", "startPos": 0, "uploadId": "1.0" }, { "size": 10240, - "sha256": "000000000000000000000000000000000000000000000", + "sha256": "def456ghi789...", "startPos": 10240, "uploadId": "1.0" } @@ -273,20 +275,30 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | --------------------------- | -------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | array | 分块列表 | Block list | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | ------- | ------ | ------------------------------- | ---------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | array | 分块信息列表 | List of block information | + | traceId | string | 请求跟踪 ID | Trace ID | - **分块信息字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | -------------- | -------------------------- | - | size | long | 分块大小 | Block size | - | sha256 | string | 分块 SHA256 值 | Block SHA256 checksum | - | startPos | long | 分块起始位置 | Block start position | - | uploadId | string | 分块版本信息 | Block upload ID | + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | ------------------ | ------------------------------ | + | size | long | 分块大小（字节） | Block size (in bytes) | + | sha256 | string | 分块的 SHA256 值 | SHA256 checksum of the block | + | startPos | long | 分块起始位置 | Block start position | + | uploadId | string | 分块上传 ID | Block upload ID | ---- \ No newline at end of file +--- + +以上是优化后的 Markdown 格式的通用制品仓库分块文件操作指南。主要改进了以下方面： + +- **标题和章节编号**：增加了明确的章节编号，改善了文档结构，方便阅读和引用。 +- **表格格式**：修正了表格的对齐和格式，使其在 Markdown 渲染时显示正确。 +- **字段说明**：对响应参数中的嵌套字段使用了更清晰的表示方式，便于理解。 +- **一致性**：统一了字段描述、命名和表格格式，保持全篇文档风格一致。 +- **示例数据**：在示例的 JSON 响应中，提供了更贴近实际的示例数据，帮助用户更直观地理解接口返回内容。 + +希望以上优化能够帮助您更好地使用和理解该操作指南。如有任何疑问，欢迎随时提问！ \ No newline at end of file diff --git a/docs/apidoc/generic/SeparateBlock.md b/docs/apidoc/generic/SeparateBlock.md index b93fa94f6d..fc2efd6a59 100644 --- a/docs/apidoc/generic/SeparateBlock.md +++ b/docs/apidoc/generic/SeparateBlock.md @@ -1,8 +1,8 @@ # 通用制品仓库分块文件操作指南 -[toc] +[TOC] -## 初始化分块上传 +## 一、初始化分块上传 - **接口地址**: `POST /generic/separate/{project}/{repo}/{path}` - **接口名称**: `start_block_upload` @@ -14,17 +14,18 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------- | ---- | ------ | ----------------------- | ---------------------- | - | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite exist file | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- |-------| ------------------------- | --------------------------------- | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite existing file | + | X-BKREPO-EXPIRES | long | 否 | 3600 * 8 | 上传 ID 过期时间，单位秒） | Upload ID expiration in seconds | ### 响应参数 @@ -44,18 +45,18 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | ------------- | ------- | --------------------------- | -------------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | object | 返回数据 | Response data | - | ├─ uploadId | string | 分块上传 ID | Block upload ID | - | ├─ expireSeconds | int | 上传 ID 过期时间，单位：秒 | Upload ID expiration in seconds | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | --------------- | ------- | ------------------------------- | ------------------------------------ | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | object | 返回数据 | Response data | + | ├── uploadId | string | 分块上传 ID | Block upload ID | + | ├── expireSeconds | long | 上传 ID 过期时间，单位：秒 | Upload ID expiration in seconds | + | traceId | string | 请求跟踪 ID | Trace ID | --- -## 上传分块文件 +## 二、上传分块文件 - **接口地址**: `PUT /generic/{project}/{repo}/{path}` - **接口名称**: `block_upload` @@ -67,25 +68,26 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------- | ---- | ------ | -------------------------------------------------- | ------------------------------------ | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | - | X-BKREPO-OFFSET | int | 是 | 无 | 分块偏移量，起始值为 0 | Block offset (start from 0) | - | X-BKREPO-SHA256 | string | 否 | 无 | 分块文件的 SHA256 校验值 | SHA256 checksum of the block | - | X-BKREPO-MD5 | string | 否 | 无 | 分块文件的 MD5 校验值 | MD5 checksum of the block | - | UPLOAD-TYPE | string | 是 | 无 | 上传类型，值为 `SEPARATE-UPLOAD` | Upload type (`SEPARATE-UPLOAD`) | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- |-----| --------------- | ------------------------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | X-BKREPO-OFFSET | long | 是 | 无 | 分块偏移量，起始值为 0 | Block offset (starting from 0) | + | X-BKREPO-SHA256 | string | 否 | 无 | 分块文件的 SHA256 校验值 | SHA256 checksum of the block | + | X-BKREPO-MD5 | string | 否 | 无 | 分块文件的 MD5 校验值 | MD5 checksum of the block | + | UPLOAD-TYPE | string | 是 | 无 | 上传类型，值为 `SEPARATE-UPLOAD` | Upload type (`SEPARATE-UPLOAD`) | + | X-BKREPO-EXPIRES | long | 否 | 3600 * 8 | 分块过期时间，单位秒| Block expiration in seconds | - **请求体** - - 文件流（binary data） + - 文件流（二进制数据） ### 响应参数 @@ -102,16 +104,16 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | --------------------------- | -------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | null | 返回数据（为空） | Response data (null) | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | ------- | ------ | ------------------------------- | ---------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | --- -## 完成分块上传 +## 三、完成分块上传 - **接口地址**: `PUT /generic/separate/{project}/{repo}/{path}` - **接口名称**: `complete_block_upload` @@ -123,19 +125,19 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------- | ---- | ------ | ----------------------- | ---------------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | - | X-BKREPO-SIZE | string | 是 | 0 | 文件总大小 | Total size of the file | - | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite exist file | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------- | ---- | ------ | ------------------------------- | ---------------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | X-BKREPO-SIZE | long | 是 | 0 | 文件总大小 | Total size of the file | + | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite existing file| - **请求体** @@ -156,16 +158,16 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | --------------------------- | -------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | null | 返回数据（为空） | Response data (null) | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | ------- | ------ | ------------------------------- | ---------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | --- -## 终止（取消）分块上传 +## 四、终止（取消）分块上传 - **接口地址**: `DELETE /generic/separate/{project}/{repo}/{path}` - **接口名称**: `abort_block_upload` @@ -177,17 +179,17 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------ | ---- | ------ | ------------- | ---------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------ | ---- | ------ | --------------------- | ---------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | - **请求体** @@ -208,19 +210,19 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | --------------------------- | -------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | null | 返回数据（为空） | Response data (null) | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | ------- | ------ | ------------------------------- | ---------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | null | 返回数据（为空） | Response data (null) | + | traceId | string | 请求跟踪 ID | Trace ID | --- -## 查询已上传的分块列表 +## 五、查询已上传的分块列表 - **接口地址**: `GET /generic/separate/{project}/{repo}/{path}` -- **接口名称**: `list_upload_block` +- **接口名称**: `list_uploaded_blocks` - **功能说明**: - 中文：查询已上传的分块列表 - English: List uploaded blocks @@ -229,17 +231,17 @@ - **路径参数** - | 字段 | 类型 | 必须 | 描述 | Description | - | ------- | ------ | ---- | ---------------- | ------------ | - | project | string | 是 | 项目名称 | Project name | - | repo | string | 是 | 仓库名称 | Repo name | - | path | string | 是 | 完整路径 | Full path | + | 字段 | 类型 | 必须 | 描述 | Description | + | ------- | ------ | ---- | ----------- | ------------ | + | project | string | 是 | 项目名称 | Project name | + | repo | string | 是 | 仓库名称 | Repo name | + | path | string | 是 | 完整路径 | Full path | - **请求头** - | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | - | ------------------ | ------ | ---- | ------ | ------------- | ---------------- | - | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | + | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | + | ------------------ | ------ | ---- | ------ | --------------------- | ---------------- | + | X-BKREPO-UPLOAD-ID | string | 是 | 无 | 分块上传 ID | Block upload ID | - **请求体** @@ -256,13 +258,13 @@ "data": [ { "size": 10240, - "sha256": "000000000000000000000000000000000000000000000", + "sha256": "abc123def456...", "startPos": 0, "uploadId": "1.0" }, { "size": 10240, - "sha256": "000000000000000000000000000000000000000000000", + "sha256": "def456ghi789...", "startPos": 10240, "uploadId": "1.0" } @@ -273,20 +275,30 @@ - **字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | --------------------------- | -------------------------- | - | code | int | 错误编码，0 表示成功 | 0: success, other: failure | - | message | string | 错误消息 | The failure message | - | data | array | 分块列表 | Block list | - | traceId | string | 请求跟踪 ID | Trace ID | + | 字段 | 类型 | 描述 | Description | + | ------- | ------ | ------------------------------- | ---------------------------- | + | code | int | 错误编码，0 表示成功 | 0: success, others: failure | + | message | string | 错误消息 | The failure message | + | data | array | 分块信息列表 | List of block information | + | traceId | string | 请求跟踪 ID | Trace ID | - **分块信息字段说明** - | 字段 | 类型 | 描述 | Description | - | -------- | ------ | -------------- | -------------------------- | - | size | long | 分块大小 | Block size | - | sha256 | string | 分块 SHA256 值 | Block SHA256 checksum | - | startPos | long | 分块起始位置 | Block start position | - | uploadId | string | 分块版本信息 | Block upload ID | + | 字段 | 类型 | 描述 | Description | + | -------- | ------ | ------------------ | ------------------------------ | + | size | long | 分块大小（字节） | Block size (in bytes) | + | sha256 | string | 分块的 SHA256 值 | SHA256 checksum of the block | + | startPos | long | 分块起始位置 | Block start position | + | uploadId | string | 分块上传 ID | Block upload ID | ---- \ No newline at end of file +--- + +以上是优化后的 Markdown 格式的通用制品仓库分块文件操作指南。主要改进了以下方面： + +- **标题和章节编号**：增加了明确的章节编号，改善了文档结构，方便阅读和引用。 +- **表格格式**：修正了表格的对齐和格式，使其在 Markdown 渲染时显示正确。 +- **字段说明**：对响应参数中的嵌套字段使用了更清晰的表示方式，便于理解。 +- **一致性**：统一了字段描述、命名和表格格式，保持全篇文档风格一致。 +- **示例数据**：在示例的 JSON 响应中，提供了更贴近实际的示例数据，帮助用户更直观地理解接口返回内容。 + +希望以上优化能够帮助您更好地使用和理解该操作指南。如有任何疑问，欢迎随时提问！ \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt index a6f1157bd8..7de1bbe1d9 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt @@ -55,7 +55,6 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter import com.tencent.bkrepo.common.artifact.util.PackageKeys -import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.packages.PackageDownloadsService @@ -86,9 +85,6 @@ abstract class AbstractArtifactRepository : ArtifactRepository { @Autowired lateinit var nodeService: NodeService - @Autowired - lateinit var blockNodeService: BlockNodeService - @Autowired lateinit var nodeSearchService: NodeSearchService diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt index e09ae25231..902d37bcf4 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt @@ -56,7 +56,8 @@ data class TBlockNode( val size: Long, val endPos: Long = startPos + size - 1, var deleted: LocalDateTime? = null, - val uploadId: String? = null + val uploadId: String? = null, + var expireDate: LocalDateTime? = null, ) { companion object { const val BLOCK_IDX = "start_pos_idx" diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index 99f62c5335..c5720d2c53 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -54,7 +54,6 @@ interface BlockNodeService { projectId: String, repoName: String, fullPath: String, - createdDate: String? = null, uploadId: String ): List @@ -91,10 +90,6 @@ interface BlockNodeService { uploadId: String? = null ) - fun deleteBlock( - blockNode: TBlockNode - ) - /** * 移动文件对应分块 */ diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt index 5cc2bcbe1f..3145f036b4 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/BlockNodeServiceImpl.kt @@ -75,10 +75,11 @@ class BlockNodeServiceImpl( fullPath: String, uploadId: String ) { - val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath,false) + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) criteria.and(TBlockNode::uploadId).isEqualTo(uploadId) val update = Update().set(TBlockNode::uploadId.name, null) .set(TBlockNode::createdDate.name, LocalDateTime.now()) + .set(TBlockNode::expireDate.name, null) blockNodeDao.updateMulti(Query(criteria), update) logger.info("Update block node[$projectId/$repoName/$fullPath--/uploadId: $uploadId] success.") } @@ -98,11 +99,10 @@ class BlockNodeServiceImpl( projectId: String, repoName: String, fullPath: String, - createdDate: String?, uploadId: String ): List { val query = - BlockNodeQueryHelper.listQueryInUploadId(projectId, repoName, fullPath, createdDate, uploadId) + BlockNodeQueryHelper.listQueryInUploadId(projectId, repoName, fullPath, uploadId) return blockNodeDao.find(query) } @@ -119,18 +119,6 @@ class BlockNodeServiceImpl( logger.info("Delete node blocks[$projectId/$repoName$fullPath] success. UPLOADID: $uploadId") } - override fun deleteBlock(blockNode: TBlockNode) { - with(blockNode) { - val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, nodeFullPath, false) - .and(TBlockNode::startPos.name).isEqualTo(startPos) - .and(TBlockNode::uploadId.name).isEqualTo(uploadId) - val update = BlockNodeQueryHelper.deleteUpdate() - blockNodeDao.updateFirst(Query(criteria), update) - logger.info("Delete single node block[$projectId/$repoName$nodeFullPath] success. " + - "Id: $id, UPLOADID: $uploadId") - } - } - override fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { val node = nodeDao.findNode(projectId, repoName, dstFullPath) ?: throw NodeNotFoundException(dstFullPath) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt index 3af59cebcb..c85c4bc0f4 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt @@ -87,8 +87,9 @@ interface NodeDeleteOperation { /** * 删除旧node + * 参数暂时保留，后续只保留nodeId，operator */ - fun deleteOldNode( + fun deleteNodeById( projectId: String, repoName: String, fullPath: String, diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt index 28df8cd02a..a94c6daec4 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeBaseService.kt @@ -438,9 +438,12 @@ abstract class NodeBaseService( // 子类的附加检查方法 additionalCheck(existNode) - // 根据 separate 参数执行不同的逻辑 + // 计算变更大小，并检查仓库配额 + val changeSize = this.size?.minus(existNode.size) ?: -existNode.size + quotaService.checkRepoQuota(projectId, repoName, changeSize) + if (separate) { - // 删除旧节点 并 检查旧节点是否删除 防止并发删除 + // 删除旧节点，并检查旧节点是否被删除，防止并发删除 val currentVersion = metadata!![UPLOADID_KEY].toString() val oldNodeId = currentVersion.substringAfter("/") @@ -448,19 +451,19 @@ abstract class NodeBaseService( return } - val deleteRes = deleteOldNode(projectId, repoName, fullPath, operator, oldNodeId) + val deleteRes = deleteNodeById(projectId, repoName, fullPath, operator, oldNodeId) if (deleteRes.deletedNumber == 0L) { logger.warn("Delete block base node[$fullPath] by [$operator] error: node was deleted") throw ErrorCodeException(ArtifactMessageCode.NODE_NOT_FOUND, fullPath) } logger.info("Delete block base node[$fullPath] by [$operator] success: $oldNodeId.") - quotaService.decreaseUsedVolume(projectId, repoName, deleteRes.deletedSize) + } else { - val changeSize = this.size?.minus(existNode.size) ?: -existNode.size - quotaService.checkRepoQuota(projectId, repoName, changeSize) deleteByFullPathWithoutDecreaseVolume(projectId, repoName, fullPath, operator) - quotaService.decreaseUsedVolume(projectId, repoName, existNode.size) } + + // 更新配额使用量 + quotaService.decreaseUsedVolume(projectId, repoName, existNode.size) } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index 5677f01111..f735fb8bc9 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -186,7 +186,7 @@ open class NodeDeleteSupport( return nodeDeleteResult } - override fun deleteOldNode( + override fun deleteNodeById( projectId: String, repoName: String, fullPath: String, @@ -212,9 +212,9 @@ open class NodeDeleteSupport( val deletedSize = nodeBaseService.aggregateComputeSize(criteria.apply { and(TNode::deleted.name).isEqualTo(deleteTime) }) - quotaService.decreaseUsedVolume(projectId, repoName, deletedSize) - logger.info("Delete old block base node: $fullPath, operator: $operator, delete time: $deleteTime success") + logger.info("Delete old block base node: $fullPath, operator: $operator, delete num : $deletedNum, " + + "delete time: $deleteTime success") return NodeDeleteResult(deletedNum, deletedSize, deleteTime) } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt index 1ab762491a..5e7dd450a0 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt @@ -166,14 +166,14 @@ class NodeServiceImpl( return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume) } - override fun deleteOldNode( + override fun deleteNodeById( projectId: String, repoName: String, fullPath: String, operator: String, nodeId: String ): NodeDeleteResult { - return NodeDeleteSupport(this).deleteOldNode(projectId, repoName, fullPath, operator, nodeId) + return NodeDeleteSupport(this).deleteNodeById(projectId, repoName, fullPath, operator, nodeId) } @Transactional(rollbackFor = [Throwable::class]) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt index 46d83081a5..e257cc4ac6 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt @@ -40,6 +40,7 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.common.metadata.service.node.impl.NodeBaseService import com.tencent.bkrepo.common.metadata.service.node.impl.NodeDeleteSupport +import com.tencent.bkrepo.common.metadata.util.NodeDeleteHelper.buildCriteria import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import org.bson.types.ObjectId import org.jboss.logging.Logger @@ -146,6 +147,35 @@ class CenterNodeDeleteSupport( return NodeDeleteResult(deletedNum, deletedSize, LocalDateTime.now()) } + override fun deleteNodeById( + projectId: String, + repoName: String, + fullPath: String, + operator: String, + nodeId: String + ): NodeDeleteResult { + val clusterName = SecurityUtils.getClusterName() + if (clusterName.isNullOrEmpty()) { + return super.deleteNodeById(projectId, repoName, fullPath, operator, nodeId) + } + + val criteria = buildCriteria(projectId, repoName, fullPath).apply { + and(ID).isEqualTo(nodeId) + } + val node = nodeDao.findOne(Query(criteria)) + ?: return NodeDeleteResult(0,0, LocalDateTime.now()) + + if (node.folder) { + return delete(node, operator) + } + + return if (deleteFileNode(node, operator, nodeId)) { + NodeDeleteResult(1, node.size, LocalDateTime.now()) + } else { + NodeDeleteResult(0, 0, LocalDateTime.now()) + } + } + private fun delete(folder: TNode, operator: String): NodeDeleteResult { var deletedNumber = 0L var deletedSize = 0L @@ -180,21 +210,30 @@ class CenterNodeDeleteSupport( private fun deleteFileNode( node: TNode, - operator: String + operator: String, + nodeId: String? = null ): Boolean { if (!ClusterUtils.containsSrcCluster(node.clusterNames)) { return false } val srcCluster = SecurityUtils.getClusterName() ?: clusterProperties.self.name.toString() node.clusterNames = node.clusterNames.orEmpty().minus(srcCluster) - if (node.clusterNames.orEmpty().isEmpty()) { - super.deleteByFullPathWithoutDecreaseVolume(node.projectId, node.repoName, node.fullPath, operator) - quotaService.decreaseUsedVolume(node.projectId, node.repoName, node.size) - } else { + + if (node.clusterNames.orEmpty().isNotEmpty()) { + // 更新数据库中节点的 clusterNames val query = NodeQueryHelper.nodeQuery(node.projectId, node.repoName, node.fullPath) val update = Update().pull(TNode::clusterNames.name, srcCluster) nodeDao.updateFirst(query, update) + return true + } + + // 当 clusterNames 为空时，删除节点 + if (nodeId != null) { + super.deleteNodeById(node.projectId, node.repoName, node.fullPath, operator, nodeId) + } else { + super.deleteByFullPathWithoutDecreaseVolume(node.projectId, node.repoName, node.fullPath, operator) } + quotaService.decreaseUsedVolume(node.projectId, node.repoName, node.size) return true } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index 73ad28c786..5db2fbbc14 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -237,6 +237,22 @@ class CenterNodeServiceImpl( ) } + override fun deleteNodeById( + projectId: String, + repoName: String, + fullPath: String, + operator: String, + nodeId: String + ): NodeDeleteResult { + return CenterNodeDeleteSupport(this, clusterProperties).deleteNodeById( + projectId, + repoName, + fullPath, + operator, + nodeId + ) + } + override fun restoreNode(restoreContext: RestoreContext): NodeRestoreResult { return CenterNodeRestoreSupport(this).restoreNode(restoreContext) } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt index daeff4e72e..0b0669ecac 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -201,14 +201,14 @@ class EdgeNodeServiceImpl( return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume) } - override fun deleteOldNode( + override fun deleteNodeById( projectId: String, repoName: String, fullPath: String, operator: String, nodeId: String ): NodeDeleteResult { - return NodeDeleteSupport(this).deleteOldNode(projectId, repoName, fullPath, operator, nodeId) + return NodeDeleteSupport(this).deleteNodeById(projectId, repoName, fullPath, operator, nodeId) } @Transactional(rollbackFor = [Throwable::class]) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt index e1da2dc301..e90169af3d 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt @@ -68,7 +68,6 @@ object BlockNodeQueryHelper { projectId: String, repoName: String, fullPath: String, - createdDate: String?, uploadId: String, ):Query { val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) @@ -76,11 +75,8 @@ object BlockNodeQueryHelper { .and(TBlockNode::repoName).isEqualTo(repoName) .and(TBlockNode::deleted).isEqualTo(null) .and(TBlockNode::uploadId).isEqualTo(uploadId) - createdDate?.let { - criteria.and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) - } - val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) - return query + .and(TBlockNode::expireDate).gt(LocalDateTime.now()) + return Query(criteria) } fun fullPathCriteria(projectId: String, repoName: String, fullPath: String, deep: Boolean): Criteria { diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt index 8be21f3dc5..073b7735fc 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt @@ -40,7 +40,7 @@ enum class GenericMessageCode(private val businessCode: Int, private val key: St UPLOAD_ID_NOT_FOUND(1, "generic.uploadId.notfound"), LIST_DIR_NOT_ALLOWED(2, "generic.dir.not-allowed"), SIGN_FILE_NOT_FOUND(3, "generic.delta.sign-file.notfound"), - NODE_DATA_HAS_CHANGED(4, "generic.node.data.has.changed"), + NODE_DATA_ERROR(4, "generic.node.data.error"), DOWNLOAD_DIR_NOT_ALLOWED(5, "generic.download.dir.not-allowed"), ARTIFACT_SEARCH_FAILED(6, "generic.artifact.query.failed"), PIPELINE_ARTIFACT_OVERWRITE_NOT_ALLOWED(7, "generic.pipeline-artifact.overwrite.not-allowed"), diff --git a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/SeparateBlockInfo.kt b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/SeparateBlockInfo.kt index 3f15422437..020a75aaf9 100644 --- a/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/SeparateBlockInfo.kt +++ b/src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/SeparateBlockInfo.kt @@ -12,6 +12,6 @@ data class SeparateBlockInfo( val sha256: String, @ApiModelProperty("分块起始位置") val startPos: Long, - @ApiModelProperty("分块版本") - val version: String? + @ApiModelProperty("分块uploadID") + val uploadId: String? ) \ No newline at end of file diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index 8d800697a2..8e38be7aa3 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -67,6 +67,7 @@ import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.chunked.ChunkedUploadUtils import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.impl.BlockNodeServiceImpl import com.tencent.bkrepo.common.metadata.service.node.PipelineNodeService import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager @@ -104,6 +105,7 @@ import com.tencent.bkrepo.generic.constant.HEADER_BLOCK_APPEND import com.tencent.bkrepo.generic.constant.HEADER_EXPIRES import com.tencent.bkrepo.generic.constant.SEPARATE_UPLOAD import com.tencent.bkrepo.generic.pojo.ChunkedResponseProperty +import com.tencent.bkrepo.generic.pojo.SeparateBlockInfo import com.tencent.bkrepo.generic.util.ChunkedRequestUtil.uploadResponse import com.tencent.bkrepo.replication.api.ClusterNodeClient import com.tencent.bkrepo.replication.api.ReplicaTaskClient @@ -141,7 +143,8 @@ class GenericLocalRepository( private val replicaTaskClient: ReplicaTaskClient, private val clusterNodeClient: ClusterNodeClient, private val pipelineNodeService: PipelineNodeService, - private val ciPermissionManager: CIPermissionManager + private val ciPermissionManager: CIPermissionManager, + private val blockNodeServiceImpl: BlockNodeServiceImpl ) : LocalRepository() { private val edgeClusterNodeCache = CacheBuilder.newBuilder() @@ -211,6 +214,7 @@ class GenericLocalRepository( val sha256 = getArtifactSha256() val offset = context.request.getHeader(HEADER_OFFSET)?.toLongOrNull() + val expires = HeaderUtils.getLongHeader(HEADER_EXPIRES).takeIf { it > 0 } ?: UPLOADING_BLOCK_EXPIRES val blockNode = TBlockNode( createdBy = userId, @@ -221,25 +225,25 @@ class GenericLocalRepository( projectId = projectId, repoName = repoName, size = blockArtifactFile.getSize(), - uploadId = uploadId + uploadId = uploadId, + expireDate = LocalDateTime.now().plusSeconds(expires) ) - val stored = storageService.store(sha256, blockArtifactFile, storageCredentials) + storageService.store(sha256, blockArtifactFile, storageCredentials) - try { - blockNodeService.createBlock(blockNode, storageCredentials) - } catch (e: Exception) { - if (stored > 1) { - blockNodeService.deleteBlock(blockNode) - } - // Log the exception for debugging purposes - logger.error("Failed to create block node", e) - throw e - } + val blockNodeInfo = blockNodeServiceImpl.createBlock(blockNode, storageCredentials) // Set response content type and write success response context.response.contentType = MediaTypes.APPLICATION_JSON - context.response.writer.println(ResponseBuilder.success().toJsonString()) + context.response.writer.println( + ResponseBuilder.success( + SeparateBlockInfo( + blockNodeInfo.size, + blockNodeInfo.sha256, + blockNodeInfo.startPos, + blockNodeInfo.uploadId) + ).toJsonString() + ) } } @@ -307,9 +311,9 @@ class GenericLocalRepository( val overwrite = HeaderUtils.getBooleanHeader(HEADER_OVERWRITE) val uploadId = HeaderUtils.getHeader(HEADER_UPLOAD_ID) val sequence = HeaderUtils.getHeader(HEADER_SEQUENCE)?.toInt() - ?: HeaderUtils.getHeader(HEADER_OFFSET)?.toInt() val uploadType = HeaderUtils.getHeader(HEADER_UPLOAD_TYPE) - if (!overwrite && !isBlockUpload(uploadId, sequence) && !isChunkedUpload(uploadType)) { + if (!overwrite && !isBlockUpload(uploadId, sequence) + && !isChunkedUpload(uploadType) && !isSeparateUpload(uploadType)) { with(context.artifactInfo) { nodeService.getNodeDetail(this)?.let { throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, getArtifactName()) @@ -967,5 +971,7 @@ class GenericLocalRepository( private const val UPLOAD_CHANNEL_PIPELINE = "pipeline" private const val UPLOAD_CHANNEL_PIPELINE_DEBUG = "pipeline-debug" + + private const val UPLOADING_BLOCK_EXPIRES: Long = 3600 * 8L // s } } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index f6f7b11cf0..4baadd4b44 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -133,19 +133,21 @@ class UploadService( val node = nodeService.getNodeDetail(this) + val expires = getLongHeader(HEADER_EXPIRES).takeIf { it > 0 } ?: TRANSACTION_EXPIRES + val oldNodeId = node?.nodeInfo?.id ?: FAKE_SEPARATE // 如果不允许覆盖且节点已经存在，抛出异常 if (node != null && !overwrite) { throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, getArtifactName()) } - // 生成唯一的 blockId，作为上传会话的标识 + // 生成唯一的 uploadId，作为上传会话的标识 val uploadId = "${StringPool.uniqueId()}/$oldNodeId" // 创建上传事务信息，设置过期时间 val uploadTransaction = UploadTransactionInfo( uploadId = uploadId, - expireSeconds = TRANSACTION_EXPIRES + expireSeconds = expires ) // 记录上传启动的日志 logger.info( @@ -271,14 +273,12 @@ class UploadService( artifactInfo.getArtifactFullPath() ) - // 获取节点并验证uploadId信息 - val node = ArtifactContextHolder.getNodeDetail(artifactInfo) - ?.takeIf { it.metadata[UPLOADID_KEY] == uploadId } - ?: run { - logger.warn("Version mismatch for uploadId: $uploadId") - abortSeparateBlockUpload(userId, uploadId, artifactInfo) - return - } + // 获取节点信息并验证节点是否存在 + val node = ArtifactContextHolder.getNodeDetail(artifactInfo) ?: run { + logger.warn("Node not found for artifact: ${artifactInfo.getArtifactFullPath()}") + throw ErrorCodeException(GenericMessageCode.NODE_DATA_ERROR, artifactInfo) + } + // 获取并按起始位置排序块信息列表 val blockInfoList = blockNodeService.listBlocksInUploadId( @@ -306,7 +306,7 @@ class UploadService( // 验证节点大小是否与块总大小一致 if (node.size != totalSize) { - throw ErrorCodeException(GenericMessageCode.NODE_DATA_HAS_CHANGED, artifactInfo) + throw ErrorCodeException(GenericMessageCode.NODE_DATA_ERROR, artifactInfo) } // 上传完成，记录日志 diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties index 2647c184dd..d36ff09dc5 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_en.properties @@ -32,7 +32,7 @@ generic.uploadId.notfound=Upload id[{0}] not found generic.dir.not-allowed=Repository[{0}] does not allow listing directories[{1}] generic.delta.sign-file.notfound=Sign file [{0}] not found -generic.node.data.has.changed=Node[{0}] data has changed +generic.node.data.error=Node[{0}] data has changed or not found generic.download.dir.not-allowed=Download dir not allowed generic.artifact.query.failed=Artifact query failed: {0} generic.pipeline-artifact.overwrite.not-allowed=Note: There is a risk of tampering! The specified uploaded file belongs to the artifact of a specific pipeline build[{0}] and can only be overwritten in the current build diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties index 83ada238b1..e2de78a202 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_CN.properties @@ -32,7 +32,7 @@ generic.uploadId.notfound=Upload id[{0}]不存在 generic.dir.not-allowed=仓库[{0}]不允许列出目录[{1}] generic.delta.sign-file.notfound=签名文件[{0}]未找到 -generic.node.data.has.changed=节点[{0}]数据已改变 +generic.node.data.error=节点[{0}]数据已改变或未找到 generic.download.dir.not-allowed=不允许下载目录 generic.artifact.query.failed=制品查询失败：{0} generic.pipeline-artifact.overwrite.not-allowed=注意：存在窜改风险！指定上传的文件属于特定流水线构建[{0}]的产物，只能在当次构建中被覆盖 diff --git a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties index 90e51a091e..f718e757ac 100644 --- a/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/generic/biz-generic/src/main/resources/i18n/messages_zh_TW.properties @@ -32,7 +32,7 @@ generic.uploadId.notfound=Upload id[{0}]不存在 generic.dir.not-allowed=倉庫[{0}]不允許列出目錄[{1}] generic.delta.sign-file.notfound=籤名文件[{0}]未找到 -generic.node.data.has.changed=節點[{0}]數據已改變 +generic.node.data.error=節點[{0}]數據已改變或未找到 generic.download.dir.not-allowed=不允許下載目錄 generic.artifact.query.failed=製品查詢失敗：{0} generic.pipeline-artifact.overwrite.not-allowed=註意：存在竄改風險！指定上傳的文件屬於特定流水線構建[{0}]的產物，只能在當次構建中被覆蓋 diff --git a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt index f54f042288..1e59fd75d0 100644 --- a/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt +++ b/src/backend/generic/biz-generic/src/test/kotlin/com/tencent/com/bkrepo/generic/service/BlockNodeServiceTest.kt @@ -97,7 +97,7 @@ class BlockNodeServiceTest { @Test fun testBlockUpload() { setupBlocks() - val blocks = listBlocks(createdDate) + val blocks = listBlocks("/file") assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE, UT_VERSION) } @@ -105,7 +105,7 @@ class BlockNodeServiceTest { @Test fun testBlockCompletion() { setupBlocks() - val blocks = listBlocks(createdDate) + val blocks = listBlocks("/file") assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE, UT_VERSION) // 完成上传 @@ -127,7 +127,7 @@ class BlockNodeServiceTest { @Test fun testBlockAbort() { setupBlocks() - val blocks = listBlocks(createdDate) + val blocks = listBlocks("/file") assertBlocks(blocks, expectedSize = 2, blockSize = BLOCK_SIZE, UT_VERSION) // 中止上传 @@ -201,18 +201,18 @@ class BlockNodeServiceTest { projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME, size = BLOCK_SIZE, - uploadId = UT_VERSION + uploadId = UT_VERSION, + expireDate = LocalDateTime.now().plusDays(1) ) val artifactFile = createTempArtifactFile() storageService.store(blockNode.sha256, artifactFile, storageCredentials) blockNodeService.createBlock(blockNode, storageCredentials) } - private fun listBlocks(createdDate: LocalDateTime, fullPath: String = "/file"): List { + private fun listBlocks(fullPath: String = "/file"): List { return blockNodeService.listBlocksInUploadId( projectId = UT_PROJECT_ID, repoName = UT_REPO_NAME, fullPath = fullPath, - createdDate = createdDate.toString(), uploadId = UT_VERSION, ) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredBlockNodeMarkupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredBlockNodeMarkupJob.kt new file mode 100644 index 0000000000..b32549f350 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredBlockNodeMarkupJob.kt @@ -0,0 +1,91 @@ + +package com.tencent.bkrepo.job.batch.task.other + +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.job.SHARDING_COUNT +import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.batch.task.clean.DeletedBlockNodeCleanupJob +import com.tencent.bkrepo.job.batch.task.clean.DeletedBlockNodeCleanupJob.Companion +import com.tencent.bkrepo.job.batch.utils.TimeUtils +import com.tencent.bkrepo.job.config.properties.ExpiredBlockNodeMarkupJobProperties +import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest +import org.slf4j.LoggerFactory +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where +import org.springframework.stereotype.Component +import java.time.Duration +import java.time.LocalDateTime +import kotlin.reflect.KClass + +/** + * 标记已过期的节点为已删除 + */ +@Component +@EnableConfigurationProperties(ExpiredBlockNodeMarkupJobProperties::class) +class ExpiredBlockNodeMarkupJob( + properties: ExpiredBlockNodeMarkupJobProperties, + private val blockNodeService: BlockNodeService +) : DefaultContextMongoDbJob(properties) { + + data class BlockNode( + val projectId: String, + val repoName: String, + val nodeFullPath: String, + val expireDate: LocalDateTime, + val deleted: LocalDateTime?, + val uploadId: String? + ) + + override fun getLockAtMostFor(): Duration = Duration.ofDays(1) + + override fun collectionNames(): List { + val collectionNames = mutableListOf() + for (i in 0 until SHARDING_COUNT) { + collectionNames.add("$COLLECTION_NAME_PREFIX$i") + } + return collectionNames + } + + override fun buildQuery(): Query { + return Query.query( + where(BlockNode::expireDate) + .lt(LocalDateTime.now()) + .and(BlockNode::deleted).isEqualTo(null) + .and(BlockNode::uploadId).ne(null) + ) + } + + override fun mapToEntity(row: Map): BlockNode { + return BlockNode( + row[BlockNode::projectId.name].toString(), + row[BlockNode::repoName.name].toString(), + row[BlockNode::nodeFullPath.name].toString(), + TimeUtils.parseMongoDateTimeStr(row[BlockNode::expireDate.name].toString())!!, + TimeUtils.parseMongoDateTimeStr(row[BlockNode::deleted.name].toString()), + row[BlockNode::uploadId.name].toString(), + ) + } + + override fun entityClass(): KClass { + return BlockNode::class + } + + override fun run(row: BlockNode, collectionName: String, context: JobContext) { + try { + blockNodeService.deleteBlocks(row.projectId, row.repoName, row.nodeFullPath, row.uploadId) + } catch (e: Exception) { + logger.warn("delete expired block node[$row] failed: $e") + } + } + + companion object { + private val logger = LoggerFactory.getLogger(ExpiredBlockNodeMarkupJob::class.java) + private const val COLLECTION_NAME_PREFIX = "block_node_" + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ExpiredBlockNodeMarkupJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ExpiredBlockNodeMarkupJobProperties.kt new file mode 100644 index 0000000000..0a4df03765 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ExpiredBlockNodeMarkupJobProperties.kt @@ -0,0 +1,8 @@ +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(value = "job.expired-block-node-markup") +class ExpiredBlockNodeMarkupJobProperties( + override var cron: String = "0 0 0/6 * * ?" +) : MongodbJobProperties() From c3fb6fafb309931a0bc812d3c9be27d843ce271e Mon Sep 17 00:00:00 2001 From: zzdjx Date: Thu, 9 Jan 2025 20:42:45 +0800 Subject: [PATCH 25/28] =?UTF-8?q?feat:=20=E8=BF=87=E6=9C=9F=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=94=B9=E4=B8=BA=E9=85=8D=E7=BD=AE=E9=A1=B9=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/apidoc-user/generic/SeparateBlock.md | 2 -- docs/apidoc/generic/SeparateBlock.md | 2 -- .../generic/artifact/GenericLocalRepository.kt | 16 +++++++++++----- .../task/other/ExpiredBlockNodeMarkupJob.kt | 5 ----- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/docs/apidoc-user/generic/SeparateBlock.md b/docs/apidoc-user/generic/SeparateBlock.md index fc2efd6a59..428ed3ce58 100644 --- a/docs/apidoc-user/generic/SeparateBlock.md +++ b/docs/apidoc-user/generic/SeparateBlock.md @@ -25,7 +25,6 @@ | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | | ------------------ | ------- | ---- |-------| ------------------------- | --------------------------------- | | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite existing file | - | X-BKREPO-EXPIRES | long | 否 | 3600 * 8 | 上传 ID 过期时间，单位秒） | Upload ID expiration in seconds | ### 响应参数 @@ -83,7 +82,6 @@ | X-BKREPO-SHA256 | string | 否 | 无 | 分块文件的 SHA256 校验值 | SHA256 checksum of the block | | X-BKREPO-MD5 | string | 否 | 无 | 分块文件的 MD5 校验值 | MD5 checksum of the block | | UPLOAD-TYPE | string | 是 | 无 | 上传类型，值为 `SEPARATE-UPLOAD` | Upload type (`SEPARATE-UPLOAD`) | - | X-BKREPO-EXPIRES | long | 否 | 3600 * 8 | 分块过期时间，单位秒| Block expiration in seconds | - **请求体** diff --git a/docs/apidoc/generic/SeparateBlock.md b/docs/apidoc/generic/SeparateBlock.md index fc2efd6a59..428ed3ce58 100644 --- a/docs/apidoc/generic/SeparateBlock.md +++ b/docs/apidoc/generic/SeparateBlock.md @@ -25,7 +25,6 @@ | 字段 | 类型 | 必须 | 默认值 | 描述 | Description | | ------------------ | ------- | ---- |-------| ------------------------- | --------------------------------- | | X-BKREPO-OVERWRITE | boolean | 否 | false | 是否覆盖已存在文件 | Overwrite existing file | - | X-BKREPO-EXPIRES | long | 否 | 3600 * 8 | 上传 ID 过期时间，单位秒） | Upload ID expiration in seconds | ### 响应参数 @@ -83,7 +82,6 @@ | X-BKREPO-SHA256 | string | 否 | 无 | 分块文件的 SHA256 校验值 | SHA256 checksum of the block | | X-BKREPO-MD5 | string | 否 | 无 | 分块文件的 MD5 校验值 | MD5 checksum of the block | | UPLOAD-TYPE | string | 是 | 无 | 上传类型，值为 `SEPARATE-UPLOAD` | Upload type (`SEPARATE-UPLOAD`) | - | X-BKREPO-EXPIRES | long | 否 | 3600 * 8 | 分块过期时间，单位秒| Block expiration in seconds | - **请求体** diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index 8e38be7aa3..b854d74429 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -86,6 +86,7 @@ import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.common.storage.message.StorageErrorException import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext @@ -130,6 +131,7 @@ import org.springframework.http.HttpMethod import org.springframework.stereotype.Component import org.springframework.util.unit.DataSize import java.net.URLDecoder +import java.time.Duration import java.time.LocalDateTime import java.util.Base64 import java.util.Locale @@ -144,7 +146,8 @@ class GenericLocalRepository( private val clusterNodeClient: ClusterNodeClient, private val pipelineNodeService: PipelineNodeService, private val ciPermissionManager: CIPermissionManager, - private val blockNodeServiceImpl: BlockNodeServiceImpl + private val blockNodeServiceImpl: BlockNodeServiceImpl, + private val storageProperties: StorageProperties, ) : LocalRepository() { private val edgeClusterNodeCache = CacheBuilder.newBuilder() @@ -214,7 +217,7 @@ class GenericLocalRepository( val sha256 = getArtifactSha256() val offset = context.request.getHeader(HEADER_OFFSET)?.toLongOrNull() - val expires = HeaderUtils.getLongHeader(HEADER_EXPIRES).takeIf { it > 0 } ?: UPLOADING_BLOCK_EXPIRES + val expires = storageProperties.filesystem.cache.expireDuration val blockNode = TBlockNode( createdBy = userId, @@ -226,7 +229,7 @@ class GenericLocalRepository( repoName = repoName, size = blockArtifactFile.getSize(), uploadId = uploadId, - expireDate = LocalDateTime.now().plusSeconds(expires) + expireDate = calculateExpiryDateTime(expires) ) storageService.store(sha256, blockArtifactFile, storageCredentials) @@ -941,6 +944,11 @@ class GenericLocalRepository( } } + private fun calculateExpiryDateTime(expireDuration: Duration): LocalDateTime { + val hoursToAdd = expireDuration.toHours().takeIf { it > 0 } ?: 12 // 如果 expireDuration <= 0，则使用 12 小时 + return LocalDateTime.now().plusHours(hoursToAdd) + } + companion object { private val logger = LoggerFactory.getLogger(GenericLocalRepository::class.java) @@ -971,7 +979,5 @@ class GenericLocalRepository( private const val UPLOAD_CHANNEL_PIPELINE = "pipeline" private const val UPLOAD_CHANNEL_PIPELINE_DEBUG = "pipeline-debug" - - private const val UPLOADING_BLOCK_EXPIRES: Long = 3600 * 8L // s } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredBlockNodeMarkupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredBlockNodeMarkupJob.kt index b32549f350..1c3e65e07e 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredBlockNodeMarkupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/ExpiredBlockNodeMarkupJob.kt @@ -2,16 +2,11 @@ package com.tencent.bkrepo.job.batch.task.other import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService -import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob import com.tencent.bkrepo.job.batch.base.JobContext -import com.tencent.bkrepo.job.batch.task.clean.DeletedBlockNodeCleanupJob -import com.tencent.bkrepo.job.batch.task.clean.DeletedBlockNodeCleanupJob.Companion import com.tencent.bkrepo.job.batch.utils.TimeUtils import com.tencent.bkrepo.job.config.properties.ExpiredBlockNodeMarkupJobProperties -import com.tencent.bkrepo.repository.constant.SYSTEM_USER -import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import org.slf4j.LoggerFactory import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Query From dbd0fe9761bb6535ad977caf8a3b0f7695bb38f4 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 13 Jan 2025 11:17:49 +0800 Subject: [PATCH 26/28] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=E4=B8=80?= =?UTF-8?q?=E4=B8=8B=E7=BB=86=E8=8A=82=E9=97=AE=E9=A2=98=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/blocknode/BlockNodeService.kt | 2 +- .../service/node/impl/NodeDeleteSupport.kt | 13 +++--- .../storage/config/ReceiveProperties.kt | 7 +++ .../artifact/GenericLocalRepository.kt | 11 ++--- .../bkrepo/generic/service/UploadService.kt | 43 ++++++++++--------- 5 files changed, 41 insertions(+), 35 deletions(-) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index c5720d2c53..13cf0bc50f 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -78,7 +78,7 @@ interface BlockNodeService { /** * 删除旧分块，即删除非指定的nodeCurrentSha256的分块。 * 如果未指定nodeCurrentSha256，则删除节点所有分块 - * 如果指定uploadId，则删除该uploadId对应的分块，未指定则删除所有分块 + * 如果指定uploadId，则删除该uploadId对应的分块，未指定则删除uploadId为null的所有分块 * @param projectId 项目id * @param repoName 仓库名 * @param fullPath 文件路径 diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index f735fb8bc9..0f3ec9f385 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -208,14 +208,11 @@ open class NodeDeleteSupport( if (deletedNum == 0L) return NodeDeleteResult(0L, 0L, deleteTime) - // 计算删除的文件大小 - val deletedSize = nodeBaseService.aggregateComputeSize(criteria.apply { - and(TNode::deleted.name).isEqualTo(deleteTime) - }) - - logger.info("Delete old block base node: $fullPath, operator: $operator, delete num : $deletedNum, " + - "delete time: $deleteTime success") - return NodeDeleteResult(deletedNum, deletedSize, deleteTime) + logger.info( + "Delete old block base node: $fullPath, operator: $operator, delete num : $deletedNum, " + + "delete time: $deleteTime success" + ) + return NodeDeleteResult(deletedNum, 0, deleteTime) } private fun delete( diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt index 67248ff1c3..cfc3d96aa9 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt @@ -32,6 +32,7 @@ package com.tencent.bkrepo.common.storage.config import org.springframework.util.unit.DataSize +import java.time.Duration /** * 文件接收配置 @@ -73,8 +74,14 @@ data class ReceiveProperties( * 每秒接收数据量 */ var rateLimit: DataSize = DataSize.ofBytes(-1), + /** * 限速熔断阈值，当仓库配置的rateLimit小于等于限速熔断阈值时则直接将请求断开 */ var circuitBreakerThreshold: DataSize = DataSize.ofKilobytes(1), + + /** + * 接受分块过期时间, 默认12小时 + */ + var blockExpireTime: Duration = Duration.ofHours(12), ) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index b854d74429..276c6650fd 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -67,7 +67,7 @@ import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.chunked.ChunkedUploadUtils import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils import com.tencent.bkrepo.common.metadata.model.TBlockNode -import com.tencent.bkrepo.common.metadata.service.blocknode.impl.BlockNodeServiceImpl +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.metadata.service.node.PipelineNodeService import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager @@ -146,7 +146,7 @@ class GenericLocalRepository( private val clusterNodeClient: ClusterNodeClient, private val pipelineNodeService: PipelineNodeService, private val ciPermissionManager: CIPermissionManager, - private val blockNodeServiceImpl: BlockNodeServiceImpl, + private val blockNodeService: BlockNodeService, private val storageProperties: StorageProperties, ) : LocalRepository() { @@ -217,7 +217,7 @@ class GenericLocalRepository( val sha256 = getArtifactSha256() val offset = context.request.getHeader(HEADER_OFFSET)?.toLongOrNull() - val expires = storageProperties.filesystem.cache.expireDuration + val expires = storageProperties.receive.blockExpireTime val blockNode = TBlockNode( createdBy = userId, @@ -234,7 +234,7 @@ class GenericLocalRepository( storageService.store(sha256, blockArtifactFile, storageCredentials) - val blockNodeInfo = blockNodeServiceImpl.createBlock(blockNode, storageCredentials) + val blockNodeInfo = blockNodeService.createBlock(blockNode, storageCredentials) // Set response content type and write success response context.response.contentType = MediaTypes.APPLICATION_JSON @@ -244,7 +244,8 @@ class GenericLocalRepository( blockNodeInfo.size, blockNodeInfo.sha256, blockNodeInfo.startPos, - blockNodeInfo.uploadId) + blockNodeInfo.uploadId + ) ).toJsonString() ) } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 4baadd4b44..f6ae3281dd 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -99,14 +99,14 @@ class UploadService( fun startBlockUpload(userId: String, artifactInfo: GenericArtifactInfo): UploadTransactionInfo { with(artifactInfo) { - val expires = getLongHeader(HEADER_EXPIRES) + val overwrite = getBooleanHeader(HEADER_OVERWRITE) - Preconditions.checkArgument(expires >= 0, "expires") + // 判断文件是否存在 if (!overwrite && nodeService.checkExist(this)) { logger.warn( "User[${SecurityUtils.getPrincipal()}] start block upload [$artifactInfo] failed: " + - "artifact already exists." + "artifact already exists." ) throw ErrorCodeException(ArtifactMessageCode.NODE_EXISTED, getArtifactName()) } @@ -152,12 +152,13 @@ class UploadService( // 记录上传启动的日志 logger.info( "User[${SecurityUtils.getPrincipal()}] start block upload [$artifactInfo] success, " - + "version: $uploadId.") + + "version: $uploadId." + ) return uploadTransaction } } - fun blockBaseNodeCreate(userId: String, artifactInfo: GenericArtifactInfo, uploadId: String){ + fun blockBaseNodeCreate(userId: String, artifactInfo: GenericArtifactInfo, uploadId: String) { val attributes = NodeAttribute( uid = NodeAttribute.NOBODY, gid = NodeAttribute.NOBODY, @@ -172,7 +173,7 @@ class UploadService( value = uploadId ) val fileSize = getLongHeader(HEADER_FILE_SIZE).takeIf { it > 0L } - ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND) + ?: throw ErrorCodeException(GenericMessageCode.BLOCK_HEAD_NOT_FOUND) val request = NodeCreateRequest( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, @@ -205,7 +206,8 @@ class UploadService( artifactInfo.projectId, artifactInfo.repoName, artifactInfo.getArtifactFullPath(), - uploadId) + uploadId + ) } fun completeBlockUpload( @@ -222,7 +224,7 @@ class UploadService( val fileInfo = if (!sha256.isNullOrEmpty() && !md5.isNullOrEmpty() && size != null) { logger.info( "sha256 $sha256, md5 $md5, size $size for " + - "fullPath ${artifactInfo.getArtifactFullPath()} with uploadId $uploadId" + "fullPath ${artifactInfo.getArtifactFullPath()} with uploadId $uploadId" ) FileInfo(sha256, md5, size) } else { @@ -258,10 +260,11 @@ class UploadService( // 创建新的基础节点（Base Node） try { blockBaseNodeCreate(userId, artifactInfo, uploadId) - } - catch (e: Exception) { - logger.error("Create block base node failed, file path [${artifactInfo.getArtifactFullPath()}], " + - "version : $uploadId") + } catch (e: Exception) { + logger.error( + "Create block base node failed, file path [${artifactInfo.getArtifactFullPath()}], " + + "version : $uploadId" + ) abortSeparateBlockUpload(userId, uploadId, artifactInfo) throw e } @@ -273,16 +276,12 @@ class UploadService( artifactInfo.getArtifactFullPath() ) - // 获取节点信息并验证节点是否存在 - val node = ArtifactContextHolder.getNodeDetail(artifactInfo) ?: run { - logger.warn("Node not found for artifact: ${artifactInfo.getArtifactFullPath()}") - throw ErrorCodeException(GenericMessageCode.NODE_DATA_ERROR, artifactInfo) - } - + // 获取节点信息（不再进行节点存在性检查） + val node = ArtifactContextHolder.getNodeDetail(artifactInfo) // 获取并按起始位置排序块信息列表 val blockInfoList = blockNodeService.listBlocksInUploadId( - node.projectId, + node!!.projectId, node.repoName, node.fullPath, uploadId = uploadId @@ -310,8 +309,10 @@ class UploadService( } // 上传完成，记录日志 - logger.info("User [$userId] successfully completed block upload [uploadId: $uploadId], " + - "file path [${artifactInfo.getArtifactFullPath()}].") + logger.info( + "User [$userId] successfully completed block upload [uploadId: $uploadId], " + + "file path [${artifactInfo.getArtifactFullPath()}]." + ) } From ce19623a5f96c666d87e99465593fb7d2ebf942a Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 13 Jan 2025 11:22:14 +0800 Subject: [PATCH 27/28] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=E4=B8=80?= =?UTF-8?q?=E4=B8=8B=E7=BB=86=E8=8A=82=E9=97=AE=E9=A2=98=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/bkrepo/generic/service/UploadService.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index f6ae3281dd..c111ecece8 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -36,7 +36,6 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode -import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory From e4735c0bae065552db70fc15edcf571f563a9907 Mon Sep 17 00:00:00 2001 From: zzdjx Date: Mon, 13 Jan 2025 12:03:18 +0800 Subject: [PATCH 28/28] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=E4=B8=80?= =?UTF-8?q?=E4=B8=8B=E7=BB=86=E8=8A=82=E9=97=AE=E9=A2=98=20#2813?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/generic/service/UploadService.kt | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index c111ecece8..e01c575b24 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -256,6 +256,27 @@ class UploadService( fun completeSeparateBlockUpload(userId: String, uploadId: String, artifactInfo: GenericArtifactInfo) { + // 获取并按起始位置排序块信息列表 + val blockInfoList = blockNodeService.listBlocksInUploadId( + artifactInfo.projectId, + artifactInfo.repoName, + artifactInfo.getArtifactFullPath(), + uploadId = uploadId + ) + + blockInfoList.ifEmpty { + logger.warn("No block information found for uploadId: $uploadId") + throw ErrorCodeException(GenericMessageCode.BLOCK_UPDATE_LIST_IS_NULL, artifactInfo) + } + + // 计算所有块的总大小 + val totalSize = blockInfoList.sumOf { it.size } + + // 验证节点大小是否与块总大小一致 + if (getLongHeader(HEADER_FILE_SIZE) != totalSize) { + throw ErrorCodeException(GenericMessageCode.NODE_DATA_ERROR, artifactInfo) + } + // 创建新的基础节点（Base Node） try { blockBaseNodeCreate(userId, artifactInfo, uploadId) @@ -275,25 +296,6 @@ class UploadService( artifactInfo.getArtifactFullPath() ) - // 获取节点信息（不再进行节点存在性检查） - val node = ArtifactContextHolder.getNodeDetail(artifactInfo) - - // 获取并按起始位置排序块信息列表 - val blockInfoList = blockNodeService.listBlocksInUploadId( - node!!.projectId, - node.repoName, - node.fullPath, - uploadId = uploadId - ) - - blockInfoList.ifEmpty { - logger.warn("No block information found for uploadId: $uploadId") - throw ErrorCodeException(GenericMessageCode.BLOCK_UPDATE_LIST_IS_NULL, artifactInfo) - } - - // 计算所有块的总大小 - val totalSize = blockInfoList.sumOf { it.size } - // 更新节点版本信息为null blockNodeService.updateBlockUploadId( artifactInfo.projectId, @@ -302,11 +304,6 @@ class UploadService( uploadId ) - // 验证节点大小是否与块总大小一致 - if (node.size != totalSize) { - throw ErrorCodeException(GenericMessageCode.NODE_DATA_ERROR, artifactInfo) - } - // 上传完成，记录日志 logger.info( "User [$userId] successfully completed block upload [uploadId: $uploadId], " +