From 4fbe7b1e3f2ecc7b9395ff718388a967594e620d Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 13 Nov 2024 11:12:08 +0800 Subject: [PATCH 1/8] =?UTF-8?q?feat:=20conan=20=E5=85=A8=E5=B1=80search?= =?UTF-8?q?=E4=BC=98=E5=8C=96#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/conan/api-conan/build.gradle.kts | 1 + .../bkrepo/conan/api/ConanMetadataClient.kt | 54 ++++++++++++++ .../pojo/metadata/ConanMetadataRequest.kt | 38 ++++++++++ .../repository/ConanLocalRepository.kt | 17 +++++ .../service/ConanMetadataController.kt | 48 +++++++++++++ .../bkrepo/conan/dao/ConanMetadataDao.kt | 37 ++++++++++ .../conan/model/TConanMetadataRecord.kt | 25 +++++++ .../conan/service/ConanMetadataService.kt | 49 +++++++++++++ .../service/impl/ConanDeleteServiceImpl.kt | 9 +++ .../service/impl/ConanSearchServiceImpl.kt | 36 +++------- ...ommitEdgeCenterConanMetadataServiceImpl.kt | 40 +++++++++++ .../impl/edge/EdgeConanMetadataService.kt | 71 +++++++++++++++++++ 12 files changed, 398 insertions(+), 27 deletions(-) create mode 100644 src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt create mode 100644 src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanMetadataService.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/center/CommitEdgeCenterConanMetadataServiceImpl.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/edge/EdgeConanMetadataService.kt diff --git a/src/backend/conan/api-conan/build.gradle.kts b/src/backend/conan/api-conan/build.gradle.kts index d440265e5a..eadf05b548 100644 --- a/src/backend/conan/api-conan/build.gradle.kts +++ b/src/backend/conan/api-conan/build.gradle.kts @@ -31,6 +31,7 @@ dependencies { api(project(":repository:api-repository")) + api(project(":common:common-artifact:artifact-service")) implementation(project(":common:common-api")) implementation(project(":common:common-artifact:artifact-api")) compileOnly("org.springframework:spring-web") diff --git a/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt new file mode 100644 index 0000000000..0918f67f42 --- /dev/null +++ b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.api + +import com.tencent.bkrepo.common.api.constant.MAVEN_SERVICE_NAME +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import io.swagger.annotations.ApiOperation +import org.springframework.cloud.openfeign.FeignClient +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam + +@FeignClient(MAVEN_SERVICE_NAME, contextId = "ConanMetadataClient") +@RequestMapping("/service/metadata") +interface ConanMetadataClient { + @ApiOperation("存储conan版本元数据") + @PostMapping("/update") + fun storeMetadata(@RequestBody request: ConanMetadataRequest): Response + + @ApiOperation("删除conan版本元数据") + @PostMapping("/delete") + fun delete( + @RequestParam projectId: String, + @RequestParam repoName: String, + @RequestParam recipe: String + ): Response +} diff --git a/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt new file mode 100644 index 0000000000..24e376e653 --- /dev/null +++ b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.pojo.metadata + +data class ConanMetadataRequest( + val projectId: String, + val repoName: String, + val user: String, + val name: String, + val version: String, + val channel: String, + val recipe: String +) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt index 7e7aaabc95..235a351e4a 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt @@ -41,8 +41,11 @@ import com.tencent.bkrepo.conan.constant.X_CHECKSUM_SHA1 import com.tencent.bkrepo.conan.listener.event.ConanPackageUploadEvent import com.tencent.bkrepo.conan.listener.event.ConanRecipeUploadEvent import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import com.tencent.bkrepo.conan.service.ConanMetadataService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanPathUtils +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.generateFullPath import com.tencent.bkrepo.conan.utils.ObjectBuildUtil import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.buildDownloadResponse @@ -55,11 +58,15 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @Component class ConanLocalRepository : LocalRepository() { + @Autowired + lateinit var conanMetadataService: ConanMetadataService + override fun buildNodeCreateRequest(context: ArtifactUploadContext): NodeCreateRequest { with(context) { val tempArtifactInfo = context.artifactInfo as ConanArtifactInfo @@ -134,6 +141,16 @@ class ConanLocalRepository : LocalRepository() { packageService.createPackageVersion(packageVersionCreateRequest).apply { logger.info("user: [$userId] create package version [$packageVersionCreateRequest] success!") } + val request = ConanMetadataRequest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + name = artifactInfo.name, + version = artifactInfo.version, + user = artifactInfo.userName, + channel = artifactInfo.channel, + recipe = buildConanFileName(convertToConanFileReference(artifactInfo)) + ) + conanMetadataService.storeMetadata(request) } override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt new file mode 100644 index 0000000000..43aa36e03a --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.controller.service + +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.conan.api.ConanMetadataClient +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import com.tencent.bkrepo.conan.service.ConanMetadataService +import org.springframework.web.bind.annotation.RestController + +@RestController +class ConanMetadataController(private val conanMetadataService: ConanMetadataService) : ConanMetadataClient { + override fun storeMetadata(request: ConanMetadataRequest): Response { + conanMetadataService.storeMetadata(request) + return ResponseBuilder.success() + } + + override fun delete(projectId: String, repoName: String, recipe: String): Response { + conanMetadataService.delete(projectId, repoName, recipe) + return ResponseBuilder.success() + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt new file mode 100644 index 0000000000..ed69e41493 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt @@ -0,0 +1,37 @@ +package com.tencent.bkrepo.conan.dao + +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.conan.model.TConanMetadataRecord +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import org.springframework.data.mongodb.core.FindAndModifyOptions +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.stereotype.Repository + +@Repository +class ConanMetadataDao : SimpleMongoDao() { + + fun findAndModify(request: ConanMetadataRequest) { + with(request) { + val query = Query(buildCriteria(projectId, repoName, recipe)) + val options = FindAndModifyOptions().upsert(true) + val update = Update().set(TConanMetadataRecord::name.name, name) + .set(TConanMetadataRecord::user.name, user) + .set(TConanMetadataRecord::version.name, version) + .set(TConanMetadataRecord::channel.name, channel) + determineMongoTemplate().findAndModify(query, update, options, TConanMetadataRecord::class.java) + } + } + + fun delete(projectId: String, repoName: String, recipe: String) { + remove(Query(buildCriteria(projectId, repoName, recipe))) + } + + private fun buildCriteria(projectId: String, repoName: String, recipe: String): Criteria { + return Criteria + .where(TConanMetadataRecord::projectId.name).`is`(projectId) + .and(TConanMetadataRecord::repoName.name).`is`(repoName) + .and(TConanMetadataRecord::recipe.name).`is`(recipe) + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt new file mode 100644 index 0000000000..2792bec1cf --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt @@ -0,0 +1,25 @@ +package com.tencent.bkrepo.conan.model + +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document + +@Document("conan_metadata") +@CompoundIndexes( + CompoundIndex( + name = "unique_index", + def = "{'projectId':1, 'repoName':1, 'recipe':1", + background = true, + unique = true + ) +) +data class TConanMetadataRecord( + val id: String?, + val projectId: String, + val repoName: String, + val user: String, + val name: String, + val version: String, + val channel: String, + val recipe: String +) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanMetadataService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanMetadataService.kt new file mode 100644 index 0000000000..c5c27cb803 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanMetadataService.kt @@ -0,0 +1,49 @@ +package com.tencent.bkrepo.conan.service + +import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition +import com.tencent.bkrepo.conan.dao.ConanMetadataDao +import com.tencent.bkrepo.conan.model.TConanMetadataRecord +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.stereotype.Service + +@Service +@Conditional(DefaultCondition::class) +class ConanMetadataService( + private val conanMetadataDao: ConanMetadataDao +) { + + fun search(projectId: String, repoName: String, pattern: String?, ignoreCase: Boolean): List { + val criteria = Criteria.where(TConanMetadataRecord::projectId.name).`is`(projectId) + .and(TConanMetadataRecord::repoName.name).`is`(repoName) + pattern?.let { + if (ignoreCase) { + criteria.and(TConanMetadataRecord::recipe.name).regex("$pattern", "i") + } else { + criteria.and(TConanMetadataRecord::recipe.name).regex("$pattern") + } + } + val query = Query(criteria) + query.fields().include(TConanMetadataRecord::recipe.name) + val records = conanMetadataDao.find(query, MutableMap::class.java) + return records.map { it[TConanMetadataRecord::recipe.name] as String } + } + + fun storeMetadata(request: ConanMetadataRequest) { + logger.info("store conan metadata: [$request]") + conanMetadataDao.findAndModify(request) + } + + fun delete(projectId: String, repoName: String, recipe: String) { + logger.info("delete conan metadata for $recipe in repo $projectId|$repoName") + conanMetadataDao.delete(projectId, repoName, recipe) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(ConanMetadataService::class.java) + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt index eff2d380e3..8c907c676a 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt @@ -45,9 +45,11 @@ import com.tencent.bkrepo.conan.pojo.ConanDomainInfo import com.tencent.bkrepo.conan.pojo.PackageVersionInfo import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.service.ConanDeleteService +import com.tencent.bkrepo.conan.service.ConanMetadataService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference import com.tencent.bkrepo.conan.utils.ConanPathUtils +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildExportFolderPath import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageFolderPath import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageIdFolderPath @@ -70,10 +72,15 @@ import org.springframework.stereotype.Service @Service class ConanDeleteServiceImpl : ConanDeleteService { + @Autowired + lateinit var conanMetadataService: ConanMetadataService + @Autowired lateinit var nodeService: NodeService + @Autowired lateinit var packageService: PackageService + @Autowired lateinit var commonService: CommonService @@ -88,6 +95,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { val refStr = ConanPathUtils.buildReferenceWithoutVersion(conanFileReference) val packageKey = PackageKeys.ofConan(refStr) packageService.deleteVersion(projectId, repoName, packageKey, version) + conanMetadataService.delete(projectId, repoName, buildConanFileName(conanFileReference)) } else { val conanFileReference = convertToConanFileReference(conanArtifactInfo, revision) val rootPath = PathUtils.normalizeFullPath(buildRevisionPath(conanFileReference)) @@ -221,6 +229,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { logger.info("Will delete folder $rootPath in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, rootPath, SecurityUtils.getUserId()) nodeService.deleteNode(request) + conanMetadataService.delete(projectId, repoName, buildConanFileName(conanFileReference)) if (deleteVersion) { packageService.deleteVersion(projectId, repoName, packageKey, versionDetail.name) } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt index 4d2ecdf04b..8eba162cdc 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt @@ -34,12 +34,10 @@ import com.tencent.bkrepo.conan.exception.ConanSearchNotFoundException import com.tencent.bkrepo.conan.pojo.ConanInfo import com.tencent.bkrepo.conan.pojo.ConanSearchResult import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo +import com.tencent.bkrepo.conan.service.ConanMetadataService import com.tencent.bkrepo.conan.service.ConanSearchService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference -import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackagePath -import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.toConanFileReference -import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -52,6 +50,9 @@ class ConanSearchServiceImpl : ConanSearchService { @Autowired lateinit var commonService: CommonService + @Autowired + lateinit var conanMetadataService: ConanMetadataService + override fun search( projectId: String, repoName: String, @@ -59,25 +60,14 @@ class ConanSearchServiceImpl : ConanSearchService { ignoreCase: Boolean ): ConanSearchResult { // TODO 需要对pattern进行校验， -q parameter only allowed with a valid recipe reference, not with a pattern - val recipes = searchRecipes(projectId, repoName) - val list = if (pattern.isNullOrEmpty()) { - recipes - } else { - val realPattern = pattern.replace("*", ".*") - val regex = if (ignoreCase) { - Regex(realPattern, RegexOption.IGNORE_CASE) - } else { - Regex(realPattern) - } - recipes.filter { regex.containsMatchIn(it) } - } - if (list.isEmpty()) { + val recipes = searchRecipes(projectId, repoName, pattern, ignoreCase) + if (recipes.isEmpty()) { throw ConanSearchNotFoundException( ConanMessageCode.CONAN_SEARCH_NOT_FOUND, pattern ?: StringPool.EMPTY, "$projectId/$repoName" ) } - return ConanSearchResult(list) + return ConanSearchResult(recipes) } override fun searchPackages(pattern: String?, conanArtifactInfo: ConanArtifactInfo): Map { @@ -94,15 +84,7 @@ class ConanSearchServiceImpl : ConanSearchService { } } - fun searchRecipes(projectId: String, repoName: String): List { - val result = mutableListOf() - packageService.listAllPackageName(projectId, repoName).forEach { - packageService.listAllVersion(projectId, repoName, it, VersionListOption()).forEach { pv -> - pv.packageMetadata.toConanFileReference()?.apply { - result.add(buildConanFileName(this)) - } - } - } - return result.sorted() + fun searchRecipes(projectId: String, repoName: String, pattern: String? = null, ignoreCase: Boolean): List { + return conanMetadataService.search(projectId, repoName, pattern, ignoreCase) } } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/center/CommitEdgeCenterConanMetadataServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/center/CommitEdgeCenterConanMetadataServiceImpl.kt new file mode 100644 index 0000000000..1ea8355b15 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/center/CommitEdgeCenterConanMetadataServiceImpl.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.service.impl.center + +import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition +import com.tencent.bkrepo.conan.dao.ConanMetadataDao +import com.tencent.bkrepo.conan.service.ConanMetadataService +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Service + +@Service +@Conditional(CommitEdgeCenterCondition::class) +class CommitEdgeCenterConanMetadataServiceImpl( + conanMetadataDao: ConanMetadataDao +) : ConanMetadataService(conanMetadataDao) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/edge/EdgeConanMetadataService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/edge/EdgeConanMetadataService.kt new file mode 100644 index 0000000000..186da5799a --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/edge/EdgeConanMetadataService.kt @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.service.impl.edge + +import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition +import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties +import com.tencent.bkrepo.common.service.feign.FeignClientFactory +import com.tencent.bkrepo.conan.api.ConanMetadataClient +import com.tencent.bkrepo.conan.dao.ConanMetadataDao +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import com.tencent.bkrepo.conan.service.ConanMetadataService +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Service + +@Service +@Conditional(CommitEdgeEdgeCondition::class) +class EdgeConanMetadataService( + clusterProperties: ClusterProperties, + private val conanMetadataDao: ConanMetadataDao +) : ConanMetadataService(conanMetadataDao) { + private val centerConanMetadataClient: ConanMetadataClient by lazy { + FeignClientFactory.create(clusterProperties.center, "conan", clusterProperties.self.name) + } + + override fun storeMetadata(request: ConanMetadataRequest) { + with(request) { + // 更新center节点metadata信息 + val centerRequest = ConanMetadataRequest( + projectId = projectId, + repoName = repoName, + name = name, + user = user, + channel = channel, + version = version, + recipe = recipe, + ) + centerConanMetadataClient.storeMetadata(centerRequest) + super.storeMetadata(request) + } + } + + override fun delete(projectId: String, repoName: String, recipe: String) { + centerConanMetadataClient.delete(projectId, repoName, recipe) + super.delete(projectId, repoName, recipe) + } +} From 59a848d8848930e50c5af13015af2c6f4491c867 Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 13 Nov 2024 18:09:54 +0800 Subject: [PATCH 2/8] =?UTF-8?q?feat:=20=E7=B4=A2=E5=BC=95=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=B0=83=E6=95=B4=EF=BC=8C=E5=B9=B6=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E6=96=87=E4=BB=B6=E5=88=B7=E6=96=B0=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../conan/controller/UserConanController.kt | 38 +++ .../conan/pojo/request/IndexRefreshRequest.kt | 37 +++ .../bkrepo/conan/service/ConanExtService.kt | 49 ++++ .../conan/service/impl/CommonService.kt | 12 +- .../conan/service/impl/ConanExtServiceImpl.kt | 231 ++++++++++++++++++ .../service/impl/ConanSearchServiceImpl.kt | 1 - .../conan/service/impl/ConanServiceImpl.kt | 8 +- .../bkrepo/conan/utils/ConanPathUtils.kt | 15 +- .../bkrepo/conan/utils/ObjectBuildUtil.kt | 12 +- 9 files changed, 374 insertions(+), 29 deletions(-) create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt index aac41e8dc8..29f1277608 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt @@ -46,13 +46,17 @@ import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_PACKAGE_DELETE_URL import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_VERSION_DELETE_URL import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_VERSION_DETAIL +import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest import com.tencent.bkrepo.conan.service.ConanDeleteService +import com.tencent.bkrepo.conan.service.ConanExtService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestAttribute +import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController @@ -63,6 +67,7 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping("/ext") class UserConanController( private val conanDeleteService: ConanDeleteService, + private val conanExtService: ConanExtService ) { @ApiOperation("查询包的版本详情") @@ -149,4 +154,37 @@ class UserConanController( fun getRegistryDomain(): Response { return ResponseBuilder.success(conanDeleteService.getDomain()) } + + @ApiOperation("仓库索引修正") + @PostMapping("/repo/index/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun repoIndexRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + ): Response { + conanExtService.indexRefreshForRepo(artifactInfo.projectId, artifactInfo.repoName) + return ResponseBuilder.success() + } + + @ApiOperation("recipe索引修正") + @PostMapping("/recipe/index/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun repoIndexRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + @RequestBody request: IndexRefreshRequest + ): Response { + conanExtService.indexRefreshForRecipe(artifactInfo.projectId, artifactInfo.repoName, request) + return ResponseBuilder.success() + } + + + @ApiOperation("packagekey下索引修正") + @PostMapping("/packagekey/index/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun recipeIndexRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + @RequestParam packageKey: String, + ): Response { + conanExtService.indexRefreshByPackageKey(artifactInfo.projectId, artifactInfo.repoName, packageKey) + return ResponseBuilder.success() + } } \ No newline at end of file diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt new file mode 100644 index 0000000000..e62f4fd450 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.pojo.request + +data class IndexRefreshRequest( + val name: String, + val version: String, + val userName: String, + val channel: String, + val revision: String? = null, + val packageId: String? = null, +) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt new file mode 100644 index 0000000000..4e18f45768 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.service + +import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest + +interface ConanExtService { + + /** + * 重新生成整个仓库下所有的index.json文件 + */ + fun indexRefreshForRepo(projectId: String, repoName: String) + + /** + * 重新生成整个指定key下所有的index.json文件 + */ + fun indexRefreshByPackageKey(projectId: String, repoName: String, key: String) + + /** + * 重新生成recipe所有的index.json文件 + */ + fun indexRefreshForRecipe(projectId: String, repoName: String, request: IndexRefreshRequest) + +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt index 1d88307792..cb7798b972 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt @@ -58,10 +58,10 @@ import com.tencent.bkrepo.conan.pojo.PackageReference import com.tencent.bkrepo.conan.pojo.RevisionInfo import com.tencent.bkrepo.conan.utils.ConanInfoLoadUtil import com.tencent.bkrepo.conan.utils.ConanPathUtils +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildExportFolderPath import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageReference import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageRevisionFolderPath -import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildReference import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildRevisionPath import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageConanInfoFile import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageRevisionsFile @@ -71,11 +71,11 @@ import com.tencent.bkrepo.conan.utils.TimeFormatUtil.convertToUtcTime import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest +import java.time.LocalDateTime import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component -import java.time.LocalDateTime @Component class CommonService( @@ -83,10 +83,13 @@ class CommonService( ) { @Autowired lateinit var nodeService: NodeService + @Autowired lateinit var storageManager: StorageManager + @Autowired lateinit var repositoryService: RepositoryService + @Autowired lateinit var lockOperation: LockOperation @@ -341,7 +344,7 @@ class CommonService( conanFileReference: ConanFileReference ): RevisionInfo? { val revPath = getRecipeRevisionsFile(conanFileReference) - val refStr = buildReference(conanFileReference) + val refStr = buildConanFileName(conanFileReference) return lockAction(projectId, repoName, revPath) { getLatestRevision( projectId = projectId, @@ -503,7 +506,7 @@ class CommonService( conanFileReference: ConanFileReference ): IndexInfo { val revPath = getRecipeRevisionsFile(conanFileReference) - val refStr = buildReference(conanFileReference) + val refStr = buildConanFileName(conanFileReference) return lockAction(projectId, repoName, revPath) { getRevisionsList( projectId = projectId, @@ -571,6 +574,7 @@ class CommonService( action() } finally { lockOperation.close(lockKey, lock) + logger.info("Lock for key $lockKey has been released.") } } else { action() diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt new file mode 100644 index 0000000000..3c4b34bb3d --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt @@ -0,0 +1,231 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.service.impl + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.packages.PackageService +import com.tencent.bkrepo.conan.constant.INDEX_JSON +import com.tencent.bkrepo.conan.pojo.IndexInfo +import com.tencent.bkrepo.conan.pojo.RevisionInfo +import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo +import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest +import com.tencent.bkrepo.conan.service.ConanExtService +import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference +import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName +import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageRevisionsFile +import com.tencent.bkrepo.conan.utils.ConanPathUtils.getRecipeRevisionsFile +import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.toConanFileReference +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.packages.PackageListOption +import com.tencent.bkrepo.repository.pojo.packages.PackageVersion +import com.tencent.bkrepo.repository.pojo.packages.VersionListOption +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +@Service +class ConanExtServiceImpl : ConanExtService { + + @Autowired + lateinit var nodeService: NodeService + + @Autowired + lateinit var packageService: PackageService + + @Autowired + lateinit var commonService: CommonService + + override fun indexRefreshForRepo(projectId: String, repoName: String) { + // 查询包 + var pageNumber = 1 + var packageOption = PackageListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) + var packagePage = packageService.listPackagePage(projectId, repoName, option = packageOption) + while (packagePage.records.isNotEmpty()) { + packagePage.records.forEach { pkg -> indexRefreshByPackageKey(projectId, repoName, pkg.key) } + pageNumber += 1 + packageOption = PackageListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) + packagePage = packageService.listPackagePage(projectId, repoName, option = packageOption) + } + } + + override fun indexRefreshByPackageKey(projectId: String, repoName: String, key: String) { + // 查询包 + var pageNumber = 1 + var versionOption = VersionListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) + var versionPage = packageService.listVersionPage(projectId, repoName, key, option = versionOption) + while (versionPage.records.isNotEmpty()) { + versionPage.records.forEach { ver -> indexRefreshByVersion(projectId, repoName, ver) } + pageNumber += 1 + versionOption = VersionListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) + versionPage = packageService.listVersionPage(projectId, repoName, key, option = versionOption) + } + } + + override fun indexRefreshForRecipe(projectId: String, repoName: String, request: IndexRefreshRequest) { + with(request) { + val artifactInfo = buildConanArtifactInfo(projectId, repoName, request) + if (revision.isNullOrEmpty()) { + refreshIndexForRecipe(artifactInfo) + } else { + if (packageId.isNullOrEmpty()) { + refreshIndexForRecipeRevision(artifactInfo) + } else { + refreshIndexForRecipePackage(artifactInfo) + } + } + } + } + + private fun indexRefreshByVersion(projectId: String, repoName: String, version: PackageVersion) { + val conanFileReference = version.packageMetadata.toConanFileReference() ?: return + val artifactInfo = ConanArtifactInfo( + projectId = projectId, + repoName = repoName, + artifactUri = StringPool.SLASH, + name = conanFileReference.name, + userName = conanFileReference.userName, + version = conanFileReference.version, + channel = conanFileReference.channel, + revision = conanFileReference.revision, + packageId = null + ) + refreshIndexForRecipe(artifactInfo) + } + + /** + * 更新recipe下所有index文件 + */ + private fun refreshIndexForRecipe(artifactInfo: ConanArtifactInfo) { + val conanFileReference = convertToConanFileReference(artifactInfo) + val revPath = PathUtils.normalizeFullPath(getRecipeRevisionsFile(conanFileReference)) + val revPathPrefix = revPath.removeSuffix(INDEX_JSON) + val revisionsList = mutableListOf>() + revisionsList.addAll(listSubFolder(artifactInfo.projectId, artifactInfo.repoName, revPathPrefix)) + revisionsList.forEach { + artifactInfo.revision = it.first + refreshIndexForRecipeRevision(artifactInfo) + } + } + + + /** + * 更新recipe下指定revision下的所有index文件 + */ + private fun refreshIndexForRecipeRevision(artifactInfo: ConanArtifactInfo) { + val conanFileReference = convertToConanFileReference(artifactInfo) + val revPath = PathUtils.normalizeFullPath(getPackageRevisionsFile(conanFileReference)) + val revisionsList = mutableListOf>() + revisionsList.addAll(listSubFolder(artifactInfo.projectId, artifactInfo.repoName, revPath)) + revisionsList.forEach { + artifactInfo.packageId = it.first + refreshIndexForRecipePackage(artifactInfo) + } + } + + + /** + * 更新recipe下指定revision下的指定packageId下所有index文件 + */ + private fun refreshIndexForRecipePackage(artifactInfo: ConanArtifactInfo) { + try { + val packageReference = convertToPackageReference(artifactInfo) + val pRevPath = PathUtils.normalizeFullPath(getPackageRevisionsFile(packageReference)) + val pRevPathPrefix = pRevPath.removeSuffix(INDEX_JSON) + val revisionsList = mutableListOf>() + revisionsList.addAll(listSubFolder(artifactInfo.projectId, artifactInfo.repoName, pRevPathPrefix)) + val conanFileReference = convertToConanFileReference(artifactInfo) + val reference = buildConanFileName(conanFileReference) + val indexInfo = IndexInfo(reference = reference) + revisionsList.forEach { indexInfo.addRevision(RevisionInfo(it.first, it.second)) } + commonService.uploadIndexJson( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + fullPath = pRevPath, + indexInfo = indexInfo + ) + } catch (e: Exception) { + logger.warn("refresh index for $artifactInfo error: ${e.message}") + } + + } + + private fun listSubFolder(projectId: String, repoName: String, path: String): List> { + var pageNumber = 1 + val result = mutableListOf>() + do { + val option = NodeListOption( + pageNumber = pageNumber, + pageSize = DEFAULT_PAGE_SIZE, + includeFolder = true, + includeMetadata = false, + deep = false + ) + val records = + nodeService.listNodePage( + ArtifactInfo(projectId, repoName, path), option + ).records + if (records.isEmpty()) { + break + } + result.addAll(records.filter { it.folder }.map { Pair(it.name, it.lastModifiedDate) }) + pageNumber++ + } while (records.size == DEFAULT_PAGE_SIZE) + return result + } + + private fun buildConanArtifactInfo( + projectId: String, + repoName: String, + request: IndexRefreshRequest + ): ConanArtifactInfo { + with(request) { + return ConanArtifactInfo( + projectId = projectId, + repoName = repoName, + artifactUri = StringPool.SLASH, + name = name, + userName = userName, + version = version, + channel = channel, + revision = revision, + packageId = packageId + ) + } + } + + + companion object { + val logger: Logger = LoggerFactory.getLogger(ConanExtServiceImpl::class.java) + private const val DEFAULT_PAGE_SIZE = 1000 + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt index 8eba162cdc..e6a8103adb 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt @@ -59,7 +59,6 @@ class ConanSearchServiceImpl : ConanSearchService { pattern: String?, ignoreCase: Boolean ): ConanSearchResult { - // TODO 需要对pattern进行校验， -q parameter only allowed with a valid recipe reference, not with a pattern val recipes = searchRecipes(projectId, repoName, pattern, ignoreCase) if (recipes.isEmpty()) { throw ConanSearchNotFoundException( diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanServiceImpl.kt index 1ac9f9874d..79068e69e0 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanServiceImpl.kt @@ -39,9 +39,9 @@ import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.service.ConanService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackagePath import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageReference -import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildReference import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -66,7 +66,7 @@ class ConanServiceImpl : ConanService { } if (urls.isEmpty()) throw ConanRecipeNotFoundException( - ConanMessageCode.CONAN_RECIPE_NOT_FOUND, buildReference(conanFileReference) + ConanMessageCode.CONAN_RECIPE_NOT_FOUND, buildConanFileName(conanFileReference) ) return urls } @@ -83,7 +83,7 @@ class ConanServiceImpl : ConanService { ) if (urls.isEmpty()) throw ConanRecipeNotFoundException( - ConanMessageCode.CONAN_RECIPE_NOT_FOUND, "${buildReference(packageReference.conRef)}/$packageId" + ConanMessageCode.CONAN_RECIPE_NOT_FOUND, "${buildConanFileName(packageReference.conRef)}/$packageId" ) return urls } @@ -153,7 +153,7 @@ class ConanServiceImpl : ConanService { commonService.getNodeDetail(projectId, repoName, buildPackagePath(conanFileReference)) return commonService.getLastRevision(projectId, repoName, conanFileReference) ?: throw ConanFileNotFoundException( - ConanMessageCode.CONAN_FILE_NOT_FOUND, buildReference(conanFileReference), getRepoIdentify() + ConanMessageCode.CONAN_FILE_NOT_FOUND, buildConanFileName(conanFileReference), getRepoIdentify() ) } } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ConanPathUtils.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ConanPathUtils.kt index ebe6ccba4f..6f5e1a0441 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ConanPathUtils.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ConanPathUtils.kt @@ -96,19 +96,6 @@ object ConanPathUtils { } } - fun buildReference(fileReference: ConanFileReference): String { - with(fileReference) { - return StringBuilder(userName) - .append(CharPool.SLASH) - .append(version) - .append(CharPool.AT) - .append(name) - .append(CharPool.SLASH) - .append(channel) - .toString() - } - } - fun buildReferenceWithoutVersion(fileReference: ConanFileReference): String { with(fileReference) { return StringBuilder(userName) @@ -122,7 +109,7 @@ object ConanPathUtils { fun buildPackageReference(packageReference: PackageReference): String { with(packageReference) { - return StringBuilder(buildReference(conRef)) + return StringBuilder(buildConanFileName(conRef)) .append(CharPool.HASH_TAG) .append(conRef.revision) .append(CharPool.COLON) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ObjectBuildUtil.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ObjectBuildUtil.kt index 997ac0d395..2231406e15 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ObjectBuildUtil.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ObjectBuildUtil.kt @@ -51,8 +51,8 @@ import com.tencent.bkrepo.conan.pojo.PackageReference import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageReference -import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildReference import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildReferenceWithoutVersion import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageRevisionsFile import com.tencent.bkrepo.conan.utils.ConanPathUtils.getRecipeRevisionsFile @@ -147,7 +147,7 @@ object ObjectBuildUtil { with(artifactInfo) { val conanFileReference = convertToConanFileReference(this) val revPath = getRecipeRevisionsFile(conanFileReference) - val refStr = buildReference(conanFileReference) + val refStr = buildConanFileName(conanFileReference) return ConanRecipeUploadRequest( projectId = projectId, repoName = repoName, @@ -167,7 +167,7 @@ object ObjectBuildUtil { with(artifactInfo) { val packageReference = convertToPackageReference(this) val revPath = getRecipeRevisionsFile(packageReference.conRef) - val refStr = buildReference(packageReference.conRef) + val refStr = buildConanFileName(packageReference.conRef) val pRevPath = getPackageRevisionsFile(packageReference) val pRefStr = buildPackageReference(packageReference) return ConanPackageUploadRequest( @@ -192,7 +192,7 @@ object ObjectBuildUtil { with(artifactInfo) { val packageReference = convertToConanFileReference(this) val revPath = getRecipeRevisionsFile(packageReference) - val refStr = buildReference(packageReference) + val refStr = buildConanFileName(packageReference) return ConanRecipeDeleteRequest( projectId = projectId, repoName = repoName, @@ -213,7 +213,7 @@ object ObjectBuildUtil { with(artifactInfo) { val packageReference = PackageReference(convertToConanFileReference(artifactInfo), packageId, pRevision) val revPath = getRecipeRevisionsFile(packageReference.conRef) - val refStr = buildReference(packageReference.conRef) + val refStr = buildConanFileName(packageReference.conRef) val pRevPath = getPackageRevisionsFile(packageReference) val pRefStr = buildPackageReference(packageReference) return ConanPackageDeleteRequest( @@ -236,7 +236,7 @@ object ObjectBuildUtil { with(artifactInfo) { val packageReference = convertToPackageReference(this) val revPath = getRecipeRevisionsFile(packageReference.conRef) - val refStr = buildReference(packageReference.conRef) + val refStr = buildConanFileName(packageReference.conRef) val pRevPath = getPackageRevisionsFile(packageReference) val pRefStr = buildPackageReference(packageReference) return ConanPackageDeleteRequest( From f9256ca1df86ce87d088159ded2e8a0385d358cd Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 14 Nov 2024 16:16:50 +0800 Subject: [PATCH 3/8] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/conan/dao/ConanMetadataDao.kt | 3 +- .../conan/service/impl/ConanExtServiceImpl.kt | 45 ++++++++++++++----- .../service/impl/ConanSearchServiceImpl.kt | 3 +- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt index ed69e41493..42eed859bc 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt @@ -15,12 +15,11 @@ class ConanMetadataDao : SimpleMongoDao() { fun findAndModify(request: ConanMetadataRequest) { with(request) { val query = Query(buildCriteria(projectId, repoName, recipe)) - val options = FindAndModifyOptions().upsert(true) val update = Update().set(TConanMetadataRecord::name.name, name) .set(TConanMetadataRecord::user.name, user) .set(TConanMetadataRecord::version.name, version) .set(TConanMetadataRecord::channel.name, channel) - determineMongoTemplate().findAndModify(query, update, options, TConanMetadataRecord::class.java) + upsert(query, update) } } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt index 3c4b34bb3d..08be5d55f6 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt @@ -40,10 +40,12 @@ import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest import com.tencent.bkrepo.conan.service.ConanExtService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference +import com.tencent.bkrepo.conan.utils.ConanPathUtils import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageRevisionsFile import com.tencent.bkrepo.conan.utils.ConanPathUtils.getRecipeRevisionsFile import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.toConanFileReference +import com.tencent.bkrepo.conan.utils.TimeFormatUtil import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.packages.PackageListOption import com.tencent.bkrepo.repository.pojo.packages.PackageVersion @@ -52,6 +54,8 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter @Service class ConanExtServiceImpl : ConanExtService { @@ -135,6 +139,33 @@ class ConanExtServiceImpl : ConanExtService { artifactInfo.revision = it.first refreshIndexForRecipeRevision(artifactInfo) } + val reference = buildConanFileName(conanFileReference) + storeIndex(reference, artifactInfo.projectId, artifactInfo.repoName, revPath, revisionsList) + } + + private fun storeIndex( + reference: String, + projectId: String, + repoName: String, + fullPath: String, + revisionsList: List> + ) { + try { + val indexInfo = IndexInfo(reference = reference) + revisionsList.forEach { + val date = LocalDateTime.parse(it.second, DateTimeFormatter.ISO_DATE_TIME) + indexInfo.addRevision(RevisionInfo(it.first, TimeFormatUtil.convertToUtcTime(date))) + } + commonService.uploadIndexJson( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + indexInfo = indexInfo + ) + } catch (e: Exception) { + logger.warn("store index $fullPath in $projectId|$repoName error: ${e.message}") + } + } @@ -142,7 +173,7 @@ class ConanExtServiceImpl : ConanExtService { * 更新recipe下指定revision下的所有index文件 */ private fun refreshIndexForRecipeRevision(artifactInfo: ConanArtifactInfo) { - val conanFileReference = convertToConanFileReference(artifactInfo) + val conanFileReference = convertToConanFileReference(artifactInfo, artifactInfo.revision) val revPath = PathUtils.normalizeFullPath(getPackageRevisionsFile(conanFileReference)) val revisionsList = mutableListOf>() revisionsList.addAll(listSubFolder(artifactInfo.projectId, artifactInfo.repoName, revPath)) @@ -163,16 +194,8 @@ class ConanExtServiceImpl : ConanExtService { val pRevPathPrefix = pRevPath.removeSuffix(INDEX_JSON) val revisionsList = mutableListOf>() revisionsList.addAll(listSubFolder(artifactInfo.projectId, artifactInfo.repoName, pRevPathPrefix)) - val conanFileReference = convertToConanFileReference(artifactInfo) - val reference = buildConanFileName(conanFileReference) - val indexInfo = IndexInfo(reference = reference) - revisionsList.forEach { indexInfo.addRevision(RevisionInfo(it.first, it.second)) } - commonService.uploadIndexJson( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - fullPath = pRevPath, - indexInfo = indexInfo - ) + val reference = ConanPathUtils.buildPackageReference(packageReference) + storeIndex(reference, artifactInfo.projectId, artifactInfo.repoName, pRevPath, revisionsList) } catch (e: Exception) { logger.warn("refresh index for $artifactInfo error: ${e.message}") } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt index e6a8103adb..4dc9ab4299 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt @@ -59,7 +59,8 @@ class ConanSearchServiceImpl : ConanSearchService { pattern: String?, ignoreCase: Boolean ): ConanSearchResult { - val recipes = searchRecipes(projectId, repoName, pattern, ignoreCase) + val realPattern = pattern?.replace("*", ".*") + val recipes = searchRecipes(projectId, repoName, realPattern, ignoreCase) if (recipes.isEmpty()) { throw ConanSearchNotFoundException( ConanMessageCode.CONAN_SEARCH_NOT_FOUND, From 3bfcde3830ac557480e47aa064bbdfb5520559b3 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 14 Nov 2024 16:47:41 +0800 Subject: [PATCH 4/8] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E5=85=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=88=B7=E6=96=B0=E6=8E=A5=E5=8F=A3#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../conan/controller/UserConanController.kt | 18 +++-- .../bkrepo/conan/dao/ConanMetadataDao.kt | 1 - .../bkrepo/conan/service/ConanExtService.kt | 4 ++ .../conan/service/impl/ConanExtServiceImpl.kt | 71 +++++++++++++++---- 4 files changed, 74 insertions(+), 20 deletions(-) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt index 29f1277608..5f56c59d09 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt @@ -168,10 +168,10 @@ class UserConanController( @ApiOperation("recipe索引修正") @PostMapping("/recipe/index/refresh/{projectId}/{repoName}") @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) - fun repoIndexRefresh( + fun recipeIndexRefresh( @ArtifactPathVariable artifactInfo: ConanArtifactInfo, @RequestBody request: IndexRefreshRequest - ): Response { + ): Response { conanExtService.indexRefreshForRecipe(artifactInfo.projectId, artifactInfo.repoName, request) return ResponseBuilder.success() } @@ -180,11 +180,21 @@ class UserConanController( @ApiOperation("packagekey下索引修正") @PostMapping("/packagekey/index/refresh/{projectId}/{repoName}") @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) - fun recipeIndexRefresh( + fun packageKeyIndexRefresh( @ArtifactPathVariable artifactInfo: ConanArtifactInfo, @RequestParam packageKey: String, - ): Response { + ): Response { conanExtService.indexRefreshByPackageKey(artifactInfo.projectId, artifactInfo.repoName, packageKey) return ResponseBuilder.success() } + + @ApiOperation("重新生成仓库元数据信息") + @PostMapping("/metadata/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun metadataRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + ): Response { + conanExtService.metadataRefresh(artifactInfo.projectId, artifactInfo.repoName) + return ResponseBuilder.success() + } } \ No newline at end of file diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt index 42eed859bc..3d508bb1d4 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt @@ -3,7 +3,6 @@ package com.tencent.bkrepo.conan.dao import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao import com.tencent.bkrepo.conan.model.TConanMetadataRecord import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest -import org.springframework.data.mongodb.core.FindAndModifyOptions import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt index 4e18f45768..2734522a2d 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt @@ -46,4 +46,8 @@ interface ConanExtService { */ fun indexRefreshForRecipe(projectId: String, repoName: String, request: IndexRefreshRequest) + /** + * 刷新整个仓库元数据信息 + */ + fun metadataRefresh(projectId: String, repoName: String) } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt index 08be5d55f6..7c55d9a618 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt @@ -36,8 +36,10 @@ import com.tencent.bkrepo.conan.constant.INDEX_JSON import com.tencent.bkrepo.conan.pojo.IndexInfo import com.tencent.bkrepo.conan.pojo.RevisionInfo import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest import com.tencent.bkrepo.conan.service.ConanExtService +import com.tencent.bkrepo.conan.service.ConanMetadataService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference import com.tencent.bkrepo.conan.utils.ConanPathUtils @@ -69,44 +71,83 @@ class ConanExtServiceImpl : ConanExtService { @Autowired lateinit var commonService: CommonService + @Autowired + lateinit var conanMetadataService: ConanMetadataService + override fun indexRefreshForRepo(projectId: String, repoName: String) { + packageSearch(projectId, repoName, true) + } + + override fun indexRefreshByPackageKey(projectId: String, repoName: String, key: String) { + versionSearch(projectId, repoName, key, true) + } + + override fun indexRefreshForRecipe(projectId: String, repoName: String, request: IndexRefreshRequest) { + with(request) { + val artifactInfo = buildConanArtifactInfo(projectId, repoName, request) + if (revision.isNullOrEmpty()) { + refreshIndexForRecipe(artifactInfo) + } else { + if (packageId.isNullOrEmpty()) { + refreshIndexForRecipeRevision(artifactInfo) + } else { + refreshIndexForRecipePackage(artifactInfo) + } + } + } + } + + override fun metadataRefresh(projectId: String, repoName: String) { + packageSearch(projectId, repoName, false) + } + + private fun packageSearch(projectId: String, repoName: String, indexRefresh: Boolean) { // 查询包 var pageNumber = 1 var packageOption = PackageListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) var packagePage = packageService.listPackagePage(projectId, repoName, option = packageOption) while (packagePage.records.isNotEmpty()) { - packagePage.records.forEach { pkg -> indexRefreshByPackageKey(projectId, repoName, pkg.key) } + packagePage.records.forEach { pkg -> versionSearch(projectId, repoName, pkg.key, indexRefresh) } pageNumber += 1 packageOption = PackageListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) packagePage = packageService.listPackagePage(projectId, repoName, option = packageOption) } } - override fun indexRefreshByPackageKey(projectId: String, repoName: String, key: String) { + private fun versionSearch(projectId: String, repoName: String, key: String, indexRefresh: Boolean) { // 查询包 var pageNumber = 1 var versionOption = VersionListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) var versionPage = packageService.listVersionPage(projectId, repoName, key, option = versionOption) while (versionPage.records.isNotEmpty()) { - versionPage.records.forEach { ver -> indexRefreshByVersion(projectId, repoName, ver) } + versionPage.records.forEach { + if (indexRefresh) { + indexRefreshByVersion(projectId, repoName, it) + } else { + refreshMetadataForVersion(projectId, repoName, it) + } + } pageNumber += 1 versionOption = VersionListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) versionPage = packageService.listVersionPage(projectId, repoName, key, option = versionOption) } } - override fun indexRefreshForRecipe(projectId: String, repoName: String, request: IndexRefreshRequest) { - with(request) { - val artifactInfo = buildConanArtifactInfo(projectId, repoName, request) - if (revision.isNullOrEmpty()) { - refreshIndexForRecipe(artifactInfo) - } else { - if (packageId.isNullOrEmpty()) { - refreshIndexForRecipeRevision(artifactInfo) - } else { - refreshIndexForRecipePackage(artifactInfo) - } - } + private fun refreshMetadataForVersion(projectId: String, repoName: String, version: PackageVersion) { + val conanFileReference = version.packageMetadata.toConanFileReference() ?: return + val request = ConanMetadataRequest( + projectId = projectId, + repoName = repoName, + name = conanFileReference.name, + version = conanFileReference.version, + user = conanFileReference.userName, + channel = conanFileReference.channel, + recipe = buildConanFileName(conanFileReference) + ) + try { + conanMetadataService.storeMetadata(request) + } catch (e: Exception) { + logger.warn("metadata refresh for $conanFileReference in $projectId|$repoName error: ${e.message}") } } From c7ade608c68e484c9f6c6b0a799ae3e03e0a0673 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 14 Nov 2024 18:09:20 +0800 Subject: [PATCH 5/8] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E5=85=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=88=B7=E6=96=B0=E6=8E=A5=E5=8F=A3#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../conan/controller/UserConanController.kt | 23 +++++++++++++++++++ .../bkrepo/conan/service/ConanExtService.kt | 10 ++++++++ .../conan/service/impl/ConanExtServiceImpl.kt | 9 ++++++++ 3 files changed, 42 insertions(+) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt index 5f56c59d09..569587a8cc 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt @@ -197,4 +197,27 @@ class UserConanController( conanExtService.metadataRefresh(artifactInfo.projectId, artifactInfo.repoName) return ResponseBuilder.success() } + + @ApiOperation("重新生成包元数据信息") + @PostMapping("/metadata/package/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun packageMetadataRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + @RequestParam packageKey: String, + ): Response { + conanExtService.packageMetadataRefresh(artifactInfo.projectId, artifactInfo.repoName, packageKey) + return ResponseBuilder.success() + } + + @ApiOperation("重新生成版本元数据信息") + @PostMapping("/metadata/version/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun versionMetadataRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + @RequestParam packageKey: String, + @RequestParam version: String, + ): Response { + conanExtService.versionMetadataRefresh(artifactInfo.projectId, artifactInfo.repoName, packageKey, version) + return ResponseBuilder.success() + } } \ No newline at end of file diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt index 2734522a2d..21d7b49c78 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt @@ -50,4 +50,14 @@ interface ConanExtService { * 刷新整个仓库元数据信息 */ fun metadataRefresh(projectId: String, repoName: String) + + /** + * 刷新整个仓库元数据信息 + */ + fun packageMetadataRefresh(projectId: String, repoName: String, packageKey: String) + + /** + * 刷新整个仓库元数据信息 + */ + fun versionMetadataRefresh(projectId: String, repoName: String, packageKey: String, version: String) } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt index 7c55d9a618..a4ae63bdfb 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt @@ -101,6 +101,15 @@ class ConanExtServiceImpl : ConanExtService { packageSearch(projectId, repoName, false) } + override fun packageMetadataRefresh(projectId: String, repoName: String, packageKey: String) { + versionSearch(projectId, repoName, packageKey, false) + } + + override fun versionMetadataRefresh(projectId: String, repoName: String, packageKey: String, version: String) { + val versionInfo = packageService.findVersionByName(projectId, repoName, packageKey, version) ?: return + refreshMetadataForVersion(projectId, repoName, versionInfo) + } + private fun packageSearch(projectId: String, repoName: String, indexRefresh: Boolean) { // 查询包 var pageNumber = 1 From 6df766c9f0520a579f06724f871aea1dcfffd523 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 14 Nov 2024 18:22:52 +0800 Subject: [PATCH 6/8] =?UTF-8?q?feat:=20=E4=BE=9D=E8=B5=96=E8=B0=83?= =?UTF-8?q?=E6=95=B4#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/conan/api-conan/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/conan/api-conan/build.gradle.kts b/src/backend/conan/api-conan/build.gradle.kts index eadf05b548..d963a13e3d 100644 --- a/src/backend/conan/api-conan/build.gradle.kts +++ b/src/backend/conan/api-conan/build.gradle.kts @@ -31,8 +31,8 @@ dependencies { api(project(":repository:api-repository")) - api(project(":common:common-artifact:artifact-service")) implementation(project(":common:common-api")) implementation(project(":common:common-artifact:artifact-api")) compileOnly("org.springframework:spring-web") + compileOnly("org.springframework.cloud:spring-cloud-openfeign-core") } From f657a3d7b151413cba081db73481eb1bc4679ebe Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 14 Nov 2024 19:50:47 +0800 Subject: [PATCH 7/8] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt index 2792bec1cf..9a697e03bb 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt @@ -8,7 +8,7 @@ import org.springframework.data.mongodb.core.mapping.Document @CompoundIndexes( CompoundIndex( name = "unique_index", - def = "{'projectId':1, 'repoName':1, 'recipe':1", + def = "{'projectId':1, 'repoName':1, 'recipe':1}", background = true, unique = true ) From e3e241983c8652a6b00ef9c245bf4ee0866dc8b2 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 14 Nov 2024 20:22:57 +0800 Subject: [PATCH 8/8] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt index 0918f67f42..a1c8da395c 100644 --- a/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt +++ b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt @@ -38,7 +38,7 @@ import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam @FeignClient(MAVEN_SERVICE_NAME, contextId = "ConanMetadataClient") -@RequestMapping("/service/metadata") +@RequestMapping("/cluster/metadata") interface ConanMetadataClient { @ApiOperation("存储conan版本元数据") @PostMapping("/update")