From ce0aff2c1a8bf2a562228899f0bbd08104ed8379 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Thu, 14 Nov 2024 20:43:51 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20conan=E5=85=A8=E5=B1=80search=E4=BB=A5?= =?UTF-8?q?=E5=8F=8Aindex=E4=BC=98=E5=8C=96#2735?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: conan 全局search优化#2735 * feat: 索引文件调整，并增加索引文件刷新接口#2735 * feat: 代码调整#2735 * feat: 增加元数据刷新接口#2735 * feat: 增加元数据刷新接口#2735 * feat: 依赖调整#2735 * feat: 代码调整#2735 * feat: 代码调整#2735 --- src/backend/conan/api-conan/build.gradle.kts | 1 + .../bkrepo/conan/api/ConanMetadataClient.kt | 54 ++++ .../pojo/metadata/ConanMetadataRequest.kt | 38 +++ .../repository/ConanLocalRepository.kt | 17 + .../conan/controller/UserConanController.kt | 71 ++++ .../service/ConanMetadataController.kt | 48 +++ .../bkrepo/conan/dao/ConanMetadataDao.kt | 35 ++ .../conan/model/TConanMetadataRecord.kt | 25 ++ .../conan/pojo/request/IndexRefreshRequest.kt | 37 +++ .../bkrepo/conan/service/ConanExtService.kt | 63 ++++ .../conan/service/ConanMetadataService.kt | 49 +++ .../conan/service/impl/CommonService.kt | 12 +- .../service/impl/ConanDeleteServiceImpl.kt | 9 + .../conan/service/impl/ConanExtServiceImpl.kt | 304 ++++++++++++++++++ .../service/impl/ConanSearchServiceImpl.kt | 38 +-- .../conan/service/impl/ConanServiceImpl.kt | 8 +- ...ommitEdgeCenterConanMetadataServiceImpl.kt | 40 +++ .../impl/edge/EdgeConanMetadataService.kt | 71 ++++ .../bkrepo/conan/utils/ConanPathUtils.kt | 15 +- .../bkrepo/conan/utils/ObjectBuildUtil.kt | 12 +- 20 files changed, 891 insertions(+), 56 deletions(-) create mode 100644 src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt create mode 100644 src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanMetadataService.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/center/CommitEdgeCenterConanMetadataServiceImpl.kt create mode 100644 src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/edge/EdgeConanMetadataService.kt diff --git a/src/backend/conan/api-conan/build.gradle.kts b/src/backend/conan/api-conan/build.gradle.kts index d440265e5a..d963a13e3d 100644 --- a/src/backend/conan/api-conan/build.gradle.kts +++ b/src/backend/conan/api-conan/build.gradle.kts @@ -34,4 +34,5 @@ dependencies { implementation(project(":common:common-api")) implementation(project(":common:common-artifact:artifact-api")) compileOnly("org.springframework:spring-web") + compileOnly("org.springframework.cloud:spring-cloud-openfeign-core") } diff --git a/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt new file mode 100644 index 0000000000..a1c8da395c --- /dev/null +++ b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.api + +import com.tencent.bkrepo.common.api.constant.MAVEN_SERVICE_NAME +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import io.swagger.annotations.ApiOperation +import org.springframework.cloud.openfeign.FeignClient +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam + +@FeignClient(MAVEN_SERVICE_NAME, contextId = "ConanMetadataClient") +@RequestMapping("/cluster/metadata") +interface ConanMetadataClient { + @ApiOperation("存储conan版本元数据") + @PostMapping("/update") + fun storeMetadata(@RequestBody request: ConanMetadataRequest): Response + + @ApiOperation("删除conan版本元数据") + @PostMapping("/delete") + fun delete( + @RequestParam projectId: String, + @RequestParam repoName: String, + @RequestParam recipe: String + ): Response +} diff --git a/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt new file mode 100644 index 0000000000..24e376e653 --- /dev/null +++ b/src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.pojo.metadata + +data class ConanMetadataRequest( + val projectId: String, + val repoName: String, + val user: String, + val name: String, + val version: String, + val channel: String, + val recipe: String +) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt index 7e7aaabc95..235a351e4a 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt @@ -41,8 +41,11 @@ import com.tencent.bkrepo.conan.constant.X_CHECKSUM_SHA1 import com.tencent.bkrepo.conan.listener.event.ConanPackageUploadEvent import com.tencent.bkrepo.conan.listener.event.ConanRecipeUploadEvent import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import com.tencent.bkrepo.conan.service.ConanMetadataService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanPathUtils +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.generateFullPath import com.tencent.bkrepo.conan.utils.ObjectBuildUtil import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.buildDownloadResponse @@ -55,11 +58,15 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @Component class ConanLocalRepository : LocalRepository() { + @Autowired + lateinit var conanMetadataService: ConanMetadataService + override fun buildNodeCreateRequest(context: ArtifactUploadContext): NodeCreateRequest { with(context) { val tempArtifactInfo = context.artifactInfo as ConanArtifactInfo @@ -134,6 +141,16 @@ class ConanLocalRepository : LocalRepository() { packageService.createPackageVersion(packageVersionCreateRequest).apply { logger.info("user: [$userId] create package version [$packageVersionCreateRequest] success!") } + val request = ConanMetadataRequest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + name = artifactInfo.name, + version = artifactInfo.version, + user = artifactInfo.userName, + channel = artifactInfo.channel, + recipe = buildConanFileName(convertToConanFileReference(artifactInfo)) + ) + conanMetadataService.storeMetadata(request) } override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt index aac41e8dc8..569587a8cc 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt @@ -46,13 +46,17 @@ import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_PACKAGE_DELETE_URL import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_VERSION_DELETE_URL import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_VERSION_DETAIL +import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest import com.tencent.bkrepo.conan.service.ConanDeleteService +import com.tencent.bkrepo.conan.service.ConanExtService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestAttribute +import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController @@ -63,6 +67,7 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping("/ext") class UserConanController( private val conanDeleteService: ConanDeleteService, + private val conanExtService: ConanExtService ) { @ApiOperation("查询包的版本详情") @@ -149,4 +154,70 @@ class UserConanController( fun getRegistryDomain(): Response { return ResponseBuilder.success(conanDeleteService.getDomain()) } + + @ApiOperation("仓库索引修正") + @PostMapping("/repo/index/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun repoIndexRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + ): Response { + conanExtService.indexRefreshForRepo(artifactInfo.projectId, artifactInfo.repoName) + return ResponseBuilder.success() + } + + @ApiOperation("recipe索引修正") + @PostMapping("/recipe/index/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun recipeIndexRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + @RequestBody request: IndexRefreshRequest + ): Response { + conanExtService.indexRefreshForRecipe(artifactInfo.projectId, artifactInfo.repoName, request) + return ResponseBuilder.success() + } + + + @ApiOperation("packagekey下索引修正") + @PostMapping("/packagekey/index/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun packageKeyIndexRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + @RequestParam packageKey: String, + ): Response { + conanExtService.indexRefreshByPackageKey(artifactInfo.projectId, artifactInfo.repoName, packageKey) + return ResponseBuilder.success() + } + + @ApiOperation("重新生成仓库元数据信息") + @PostMapping("/metadata/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun metadataRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + ): Response { + conanExtService.metadataRefresh(artifactInfo.projectId, artifactInfo.repoName) + return ResponseBuilder.success() + } + + @ApiOperation("重新生成包元数据信息") + @PostMapping("/metadata/package/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun packageMetadataRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + @RequestParam packageKey: String, + ): Response { + conanExtService.packageMetadataRefresh(artifactInfo.projectId, artifactInfo.repoName, packageKey) + return ResponseBuilder.success() + } + + @ApiOperation("重新生成版本元数据信息") + @PostMapping("/metadata/version/refresh/{projectId}/{repoName}") + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun versionMetadataRefresh( + @ArtifactPathVariable artifactInfo: ConanArtifactInfo, + @RequestParam packageKey: String, + @RequestParam version: String, + ): Response { + conanExtService.versionMetadataRefresh(artifactInfo.projectId, artifactInfo.repoName, packageKey, version) + return ResponseBuilder.success() + } } \ No newline at end of file diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt new file mode 100644 index 0000000000..43aa36e03a --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.controller.service + +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.conan.api.ConanMetadataClient +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import com.tencent.bkrepo.conan.service.ConanMetadataService +import org.springframework.web.bind.annotation.RestController + +@RestController +class ConanMetadataController(private val conanMetadataService: ConanMetadataService) : ConanMetadataClient { + override fun storeMetadata(request: ConanMetadataRequest): Response { + conanMetadataService.storeMetadata(request) + return ResponseBuilder.success() + } + + override fun delete(projectId: String, repoName: String, recipe: String): Response { + conanMetadataService.delete(projectId, repoName, recipe) + return ResponseBuilder.success() + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt new file mode 100644 index 0000000000..3d508bb1d4 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt @@ -0,0 +1,35 @@ +package com.tencent.bkrepo.conan.dao + +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.conan.model.TConanMetadataRecord +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.stereotype.Repository + +@Repository +class ConanMetadataDao : SimpleMongoDao() { + + fun findAndModify(request: ConanMetadataRequest) { + with(request) { + val query = Query(buildCriteria(projectId, repoName, recipe)) + val update = Update().set(TConanMetadataRecord::name.name, name) + .set(TConanMetadataRecord::user.name, user) + .set(TConanMetadataRecord::version.name, version) + .set(TConanMetadataRecord::channel.name, channel) + upsert(query, update) + } + } + + fun delete(projectId: String, repoName: String, recipe: String) { + remove(Query(buildCriteria(projectId, repoName, recipe))) + } + + private fun buildCriteria(projectId: String, repoName: String, recipe: String): Criteria { + return Criteria + .where(TConanMetadataRecord::projectId.name).`is`(projectId) + .and(TConanMetadataRecord::repoName.name).`is`(repoName) + .and(TConanMetadataRecord::recipe.name).`is`(recipe) + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt new file mode 100644 index 0000000000..9a697e03bb --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt @@ -0,0 +1,25 @@ +package com.tencent.bkrepo.conan.model + +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document + +@Document("conan_metadata") +@CompoundIndexes( + CompoundIndex( + name = "unique_index", + def = "{'projectId':1, 'repoName':1, 'recipe':1}", + background = true, + unique = true + ) +) +data class TConanMetadataRecord( + val id: String?, + val projectId: String, + val repoName: String, + val user: String, + val name: String, + val version: String, + val channel: String, + val recipe: String +) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt new file mode 100644 index 0000000000..e62f4fd450 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.pojo.request + +data class IndexRefreshRequest( + val name: String, + val version: String, + val userName: String, + val channel: String, + val revision: String? = null, + val packageId: String? = null, +) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt new file mode 100644 index 0000000000..21d7b49c78 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanExtService.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.service + +import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest + +interface ConanExtService { + + /** + * 重新生成整个仓库下所有的index.json文件 + */ + fun indexRefreshForRepo(projectId: String, repoName: String) + + /** + * 重新生成整个指定key下所有的index.json文件 + */ + fun indexRefreshByPackageKey(projectId: String, repoName: String, key: String) + + /** + * 重新生成recipe所有的index.json文件 + */ + fun indexRefreshForRecipe(projectId: String, repoName: String, request: IndexRefreshRequest) + + /** + * 刷新整个仓库元数据信息 + */ + fun metadataRefresh(projectId: String, repoName: String) + + /** + * 刷新整个仓库元数据信息 + */ + fun packageMetadataRefresh(projectId: String, repoName: String, packageKey: String) + + /** + * 刷新整个仓库元数据信息 + */ + fun versionMetadataRefresh(projectId: String, repoName: String, packageKey: String, version: String) +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanMetadataService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanMetadataService.kt new file mode 100644 index 0000000000..c5c27cb803 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/ConanMetadataService.kt @@ -0,0 +1,49 @@ +package com.tencent.bkrepo.conan.service + +import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition +import com.tencent.bkrepo.conan.dao.ConanMetadataDao +import com.tencent.bkrepo.conan.model.TConanMetadataRecord +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.stereotype.Service + +@Service +@Conditional(DefaultCondition::class) +class ConanMetadataService( + private val conanMetadataDao: ConanMetadataDao +) { + + fun search(projectId: String, repoName: String, pattern: String?, ignoreCase: Boolean): List { + val criteria = Criteria.where(TConanMetadataRecord::projectId.name).`is`(projectId) + .and(TConanMetadataRecord::repoName.name).`is`(repoName) + pattern?.let { + if (ignoreCase) { + criteria.and(TConanMetadataRecord::recipe.name).regex("$pattern", "i") + } else { + criteria.and(TConanMetadataRecord::recipe.name).regex("$pattern") + } + } + val query = Query(criteria) + query.fields().include(TConanMetadataRecord::recipe.name) + val records = conanMetadataDao.find(query, MutableMap::class.java) + return records.map { it[TConanMetadataRecord::recipe.name] as String } + } + + fun storeMetadata(request: ConanMetadataRequest) { + logger.info("store conan metadata: [$request]") + conanMetadataDao.findAndModify(request) + } + + fun delete(projectId: String, repoName: String, recipe: String) { + logger.info("delete conan metadata for $recipe in repo $projectId|$repoName") + conanMetadataDao.delete(projectId, repoName, recipe) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(ConanMetadataService::class.java) + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt index 1d88307792..cb7798b972 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/CommonService.kt @@ -58,10 +58,10 @@ import com.tencent.bkrepo.conan.pojo.PackageReference import com.tencent.bkrepo.conan.pojo.RevisionInfo import com.tencent.bkrepo.conan.utils.ConanInfoLoadUtil import com.tencent.bkrepo.conan.utils.ConanPathUtils +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildExportFolderPath import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageReference import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageRevisionFolderPath -import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildReference import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildRevisionPath import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageConanInfoFile import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageRevisionsFile @@ -71,11 +71,11 @@ import com.tencent.bkrepo.conan.utils.TimeFormatUtil.convertToUtcTime import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest +import java.time.LocalDateTime import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component -import java.time.LocalDateTime @Component class CommonService( @@ -83,10 +83,13 @@ class CommonService( ) { @Autowired lateinit var nodeService: NodeService + @Autowired lateinit var storageManager: StorageManager + @Autowired lateinit var repositoryService: RepositoryService + @Autowired lateinit var lockOperation: LockOperation @@ -341,7 +344,7 @@ class CommonService( conanFileReference: ConanFileReference ): RevisionInfo? { val revPath = getRecipeRevisionsFile(conanFileReference) - val refStr = buildReference(conanFileReference) + val refStr = buildConanFileName(conanFileReference) return lockAction(projectId, repoName, revPath) { getLatestRevision( projectId = projectId, @@ -503,7 +506,7 @@ class CommonService( conanFileReference: ConanFileReference ): IndexInfo { val revPath = getRecipeRevisionsFile(conanFileReference) - val refStr = buildReference(conanFileReference) + val refStr = buildConanFileName(conanFileReference) return lockAction(projectId, repoName, revPath) { getRevisionsList( projectId = projectId, @@ -571,6 +574,7 @@ class CommonService( action() } finally { lockOperation.close(lockKey, lock) + logger.info("Lock for key $lockKey has been released.") } } else { action() diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt index eff2d380e3..8c907c676a 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanDeleteServiceImpl.kt @@ -45,9 +45,11 @@ import com.tencent.bkrepo.conan.pojo.ConanDomainInfo import com.tencent.bkrepo.conan.pojo.PackageVersionInfo import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.service.ConanDeleteService +import com.tencent.bkrepo.conan.service.ConanMetadataService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference import com.tencent.bkrepo.conan.utils.ConanPathUtils +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildExportFolderPath import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageFolderPath import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageIdFolderPath @@ -70,10 +72,15 @@ import org.springframework.stereotype.Service @Service class ConanDeleteServiceImpl : ConanDeleteService { + @Autowired + lateinit var conanMetadataService: ConanMetadataService + @Autowired lateinit var nodeService: NodeService + @Autowired lateinit var packageService: PackageService + @Autowired lateinit var commonService: CommonService @@ -88,6 +95,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { val refStr = ConanPathUtils.buildReferenceWithoutVersion(conanFileReference) val packageKey = PackageKeys.ofConan(refStr) packageService.deleteVersion(projectId, repoName, packageKey, version) + conanMetadataService.delete(projectId, repoName, buildConanFileName(conanFileReference)) } else { val conanFileReference = convertToConanFileReference(conanArtifactInfo, revision) val rootPath = PathUtils.normalizeFullPath(buildRevisionPath(conanFileReference)) @@ -221,6 +229,7 @@ class ConanDeleteServiceImpl : ConanDeleteService { logger.info("Will delete folder $rootPath in repo $projectId|$repoName") val request = NodeDeleteRequest(projectId, repoName, rootPath, SecurityUtils.getUserId()) nodeService.deleteNode(request) + conanMetadataService.delete(projectId, repoName, buildConanFileName(conanFileReference)) if (deleteVersion) { packageService.deleteVersion(projectId, repoName, packageKey, versionDetail.name) } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt new file mode 100644 index 0000000000..a4ae63bdfb --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanExtServiceImpl.kt @@ -0,0 +1,304 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.service.impl + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.packages.PackageService +import com.tencent.bkrepo.conan.constant.INDEX_JSON +import com.tencent.bkrepo.conan.pojo.IndexInfo +import com.tencent.bkrepo.conan.pojo.RevisionInfo +import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest +import com.tencent.bkrepo.conan.service.ConanExtService +import com.tencent.bkrepo.conan.service.ConanMetadataService +import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference +import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference +import com.tencent.bkrepo.conan.utils.ConanPathUtils +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName +import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageRevisionsFile +import com.tencent.bkrepo.conan.utils.ConanPathUtils.getRecipeRevisionsFile +import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.toConanFileReference +import com.tencent.bkrepo.conan.utils.TimeFormatUtil +import com.tencent.bkrepo.repository.pojo.node.NodeListOption +import com.tencent.bkrepo.repository.pojo.packages.PackageListOption +import com.tencent.bkrepo.repository.pojo.packages.PackageVersion +import com.tencent.bkrepo.repository.pojo.packages.VersionListOption +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +@Service +class ConanExtServiceImpl : ConanExtService { + + @Autowired + lateinit var nodeService: NodeService + + @Autowired + lateinit var packageService: PackageService + + @Autowired + lateinit var commonService: CommonService + + @Autowired + lateinit var conanMetadataService: ConanMetadataService + + override fun indexRefreshForRepo(projectId: String, repoName: String) { + packageSearch(projectId, repoName, true) + } + + override fun indexRefreshByPackageKey(projectId: String, repoName: String, key: String) { + versionSearch(projectId, repoName, key, true) + } + + override fun indexRefreshForRecipe(projectId: String, repoName: String, request: IndexRefreshRequest) { + with(request) { + val artifactInfo = buildConanArtifactInfo(projectId, repoName, request) + if (revision.isNullOrEmpty()) { + refreshIndexForRecipe(artifactInfo) + } else { + if (packageId.isNullOrEmpty()) { + refreshIndexForRecipeRevision(artifactInfo) + } else { + refreshIndexForRecipePackage(artifactInfo) + } + } + } + } + + override fun metadataRefresh(projectId: String, repoName: String) { + packageSearch(projectId, repoName, false) + } + + override fun packageMetadataRefresh(projectId: String, repoName: String, packageKey: String) { + versionSearch(projectId, repoName, packageKey, false) + } + + override fun versionMetadataRefresh(projectId: String, repoName: String, packageKey: String, version: String) { + val versionInfo = packageService.findVersionByName(projectId, repoName, packageKey, version) ?: return + refreshMetadataForVersion(projectId, repoName, versionInfo) + } + + private fun packageSearch(projectId: String, repoName: String, indexRefresh: Boolean) { + // 查询包 + var pageNumber = 1 + var packageOption = PackageListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) + var packagePage = packageService.listPackagePage(projectId, repoName, option = packageOption) + while (packagePage.records.isNotEmpty()) { + packagePage.records.forEach { pkg -> versionSearch(projectId, repoName, pkg.key, indexRefresh) } + pageNumber += 1 + packageOption = PackageListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) + packagePage = packageService.listPackagePage(projectId, repoName, option = packageOption) + } + } + + private fun versionSearch(projectId: String, repoName: String, key: String, indexRefresh: Boolean) { + // 查询包 + var pageNumber = 1 + var versionOption = VersionListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) + var versionPage = packageService.listVersionPage(projectId, repoName, key, option = versionOption) + while (versionPage.records.isNotEmpty()) { + versionPage.records.forEach { + if (indexRefresh) { + indexRefreshByVersion(projectId, repoName, it) + } else { + refreshMetadataForVersion(projectId, repoName, it) + } + } + pageNumber += 1 + versionOption = VersionListOption(pageNumber = pageNumber, pageSize = DEFAULT_PAGE_SIZE) + versionPage = packageService.listVersionPage(projectId, repoName, key, option = versionOption) + } + } + + private fun refreshMetadataForVersion(projectId: String, repoName: String, version: PackageVersion) { + val conanFileReference = version.packageMetadata.toConanFileReference() ?: return + val request = ConanMetadataRequest( + projectId = projectId, + repoName = repoName, + name = conanFileReference.name, + version = conanFileReference.version, + user = conanFileReference.userName, + channel = conanFileReference.channel, + recipe = buildConanFileName(conanFileReference) + ) + try { + conanMetadataService.storeMetadata(request) + } catch (e: Exception) { + logger.warn("metadata refresh for $conanFileReference in $projectId|$repoName error: ${e.message}") + } + } + + private fun indexRefreshByVersion(projectId: String, repoName: String, version: PackageVersion) { + val conanFileReference = version.packageMetadata.toConanFileReference() ?: return + val artifactInfo = ConanArtifactInfo( + projectId = projectId, + repoName = repoName, + artifactUri = StringPool.SLASH, + name = conanFileReference.name, + userName = conanFileReference.userName, + version = conanFileReference.version, + channel = conanFileReference.channel, + revision = conanFileReference.revision, + packageId = null + ) + refreshIndexForRecipe(artifactInfo) + } + + /** + * 更新recipe下所有index文件 + */ + private fun refreshIndexForRecipe(artifactInfo: ConanArtifactInfo) { + val conanFileReference = convertToConanFileReference(artifactInfo) + val revPath = PathUtils.normalizeFullPath(getRecipeRevisionsFile(conanFileReference)) + val revPathPrefix = revPath.removeSuffix(INDEX_JSON) + val revisionsList = mutableListOf>() + revisionsList.addAll(listSubFolder(artifactInfo.projectId, artifactInfo.repoName, revPathPrefix)) + revisionsList.forEach { + artifactInfo.revision = it.first + refreshIndexForRecipeRevision(artifactInfo) + } + val reference = buildConanFileName(conanFileReference) + storeIndex(reference, artifactInfo.projectId, artifactInfo.repoName, revPath, revisionsList) + } + + private fun storeIndex( + reference: String, + projectId: String, + repoName: String, + fullPath: String, + revisionsList: List> + ) { + try { + val indexInfo = IndexInfo(reference = reference) + revisionsList.forEach { + val date = LocalDateTime.parse(it.second, DateTimeFormatter.ISO_DATE_TIME) + indexInfo.addRevision(RevisionInfo(it.first, TimeFormatUtil.convertToUtcTime(date))) + } + commonService.uploadIndexJson( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + indexInfo = indexInfo + ) + } catch (e: Exception) { + logger.warn("store index $fullPath in $projectId|$repoName error: ${e.message}") + } + + } + + + /** + * 更新recipe下指定revision下的所有index文件 + */ + private fun refreshIndexForRecipeRevision(artifactInfo: ConanArtifactInfo) { + val conanFileReference = convertToConanFileReference(artifactInfo, artifactInfo.revision) + val revPath = PathUtils.normalizeFullPath(getPackageRevisionsFile(conanFileReference)) + val revisionsList = mutableListOf>() + revisionsList.addAll(listSubFolder(artifactInfo.projectId, artifactInfo.repoName, revPath)) + revisionsList.forEach { + artifactInfo.packageId = it.first + refreshIndexForRecipePackage(artifactInfo) + } + } + + + /** + * 更新recipe下指定revision下的指定packageId下所有index文件 + */ + private fun refreshIndexForRecipePackage(artifactInfo: ConanArtifactInfo) { + try { + val packageReference = convertToPackageReference(artifactInfo) + val pRevPath = PathUtils.normalizeFullPath(getPackageRevisionsFile(packageReference)) + val pRevPathPrefix = pRevPath.removeSuffix(INDEX_JSON) + val revisionsList = mutableListOf>() + revisionsList.addAll(listSubFolder(artifactInfo.projectId, artifactInfo.repoName, pRevPathPrefix)) + val reference = ConanPathUtils.buildPackageReference(packageReference) + storeIndex(reference, artifactInfo.projectId, artifactInfo.repoName, pRevPath, revisionsList) + } catch (e: Exception) { + logger.warn("refresh index for $artifactInfo error: ${e.message}") + } + + } + + private fun listSubFolder(projectId: String, repoName: String, path: String): List> { + var pageNumber = 1 + val result = mutableListOf>() + do { + val option = NodeListOption( + pageNumber = pageNumber, + pageSize = DEFAULT_PAGE_SIZE, + includeFolder = true, + includeMetadata = false, + deep = false + ) + val records = + nodeService.listNodePage( + ArtifactInfo(projectId, repoName, path), option + ).records + if (records.isEmpty()) { + break + } + result.addAll(records.filter { it.folder }.map { Pair(it.name, it.lastModifiedDate) }) + pageNumber++ + } while (records.size == DEFAULT_PAGE_SIZE) + return result + } + + private fun buildConanArtifactInfo( + projectId: String, + repoName: String, + request: IndexRefreshRequest + ): ConanArtifactInfo { + with(request) { + return ConanArtifactInfo( + projectId = projectId, + repoName = repoName, + artifactUri = StringPool.SLASH, + name = name, + userName = userName, + version = version, + channel = channel, + revision = revision, + packageId = packageId + ) + } + } + + + companion object { + val logger: Logger = LoggerFactory.getLogger(ConanExtServiceImpl::class.java) + private const val DEFAULT_PAGE_SIZE = 1000 + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt index 4d2ecdf04b..4dc9ab4299 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanSearchServiceImpl.kt @@ -34,12 +34,10 @@ import com.tencent.bkrepo.conan.exception.ConanSearchNotFoundException import com.tencent.bkrepo.conan.pojo.ConanInfo import com.tencent.bkrepo.conan.pojo.ConanSearchResult import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo +import com.tencent.bkrepo.conan.service.ConanMetadataService import com.tencent.bkrepo.conan.service.ConanSearchService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference -import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackagePath -import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.toConanFileReference -import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -52,32 +50,24 @@ class ConanSearchServiceImpl : ConanSearchService { @Autowired lateinit var commonService: CommonService + @Autowired + lateinit var conanMetadataService: ConanMetadataService + override fun search( projectId: String, repoName: String, pattern: String?, ignoreCase: Boolean ): ConanSearchResult { - // TODO 需要对pattern进行校验， -q parameter only allowed with a valid recipe reference, not with a pattern - val recipes = searchRecipes(projectId, repoName) - val list = if (pattern.isNullOrEmpty()) { - recipes - } else { - val realPattern = pattern.replace("*", ".*") - val regex = if (ignoreCase) { - Regex(realPattern, RegexOption.IGNORE_CASE) - } else { - Regex(realPattern) - } - recipes.filter { regex.containsMatchIn(it) } - } - if (list.isEmpty()) { + val realPattern = pattern?.replace("*", ".*") + val recipes = searchRecipes(projectId, repoName, realPattern, ignoreCase) + if (recipes.isEmpty()) { throw ConanSearchNotFoundException( ConanMessageCode.CONAN_SEARCH_NOT_FOUND, pattern ?: StringPool.EMPTY, "$projectId/$repoName" ) } - return ConanSearchResult(list) + return ConanSearchResult(recipes) } override fun searchPackages(pattern: String?, conanArtifactInfo: ConanArtifactInfo): Map { @@ -94,15 +84,7 @@ class ConanSearchServiceImpl : ConanSearchService { } } - fun searchRecipes(projectId: String, repoName: String): List { - val result = mutableListOf() - packageService.listAllPackageName(projectId, repoName).forEach { - packageService.listAllVersion(projectId, repoName, it, VersionListOption()).forEach { pv -> - pv.packageMetadata.toConanFileReference()?.apply { - result.add(buildConanFileName(this)) - } - } - } - return result.sorted() + fun searchRecipes(projectId: String, repoName: String, pattern: String? = null, ignoreCase: Boolean): List { + return conanMetadataService.search(projectId, repoName, pattern, ignoreCase) } } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanServiceImpl.kt index 1ac9f9874d..79068e69e0 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanServiceImpl.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/ConanServiceImpl.kt @@ -39,9 +39,9 @@ import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.service.ConanService import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackagePath import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageReference -import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildReference import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -66,7 +66,7 @@ class ConanServiceImpl : ConanService { } if (urls.isEmpty()) throw ConanRecipeNotFoundException( - ConanMessageCode.CONAN_RECIPE_NOT_FOUND, buildReference(conanFileReference) + ConanMessageCode.CONAN_RECIPE_NOT_FOUND, buildConanFileName(conanFileReference) ) return urls } @@ -83,7 +83,7 @@ class ConanServiceImpl : ConanService { ) if (urls.isEmpty()) throw ConanRecipeNotFoundException( - ConanMessageCode.CONAN_RECIPE_NOT_FOUND, "${buildReference(packageReference.conRef)}/$packageId" + ConanMessageCode.CONAN_RECIPE_NOT_FOUND, "${buildConanFileName(packageReference.conRef)}/$packageId" ) return urls } @@ -153,7 +153,7 @@ class ConanServiceImpl : ConanService { commonService.getNodeDetail(projectId, repoName, buildPackagePath(conanFileReference)) return commonService.getLastRevision(projectId, repoName, conanFileReference) ?: throw ConanFileNotFoundException( - ConanMessageCode.CONAN_FILE_NOT_FOUND, buildReference(conanFileReference), getRepoIdentify() + ConanMessageCode.CONAN_FILE_NOT_FOUND, buildConanFileName(conanFileReference), getRepoIdentify() ) } } diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/center/CommitEdgeCenterConanMetadataServiceImpl.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/center/CommitEdgeCenterConanMetadataServiceImpl.kt new file mode 100644 index 0000000000..1ea8355b15 --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/center/CommitEdgeCenterConanMetadataServiceImpl.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.service.impl.center + +import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition +import com.tencent.bkrepo.conan.dao.ConanMetadataDao +import com.tencent.bkrepo.conan.service.ConanMetadataService +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Service + +@Service +@Conditional(CommitEdgeCenterCondition::class) +class CommitEdgeCenterConanMetadataServiceImpl( + conanMetadataDao: ConanMetadataDao +) : ConanMetadataService(conanMetadataDao) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/edge/EdgeConanMetadataService.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/edge/EdgeConanMetadataService.kt new file mode 100644 index 0000000000..186da5799a --- /dev/null +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/service/impl/edge/EdgeConanMetadataService.kt @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.conan.service.impl.edge + +import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition +import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties +import com.tencent.bkrepo.common.service.feign.FeignClientFactory +import com.tencent.bkrepo.conan.api.ConanMetadataClient +import com.tencent.bkrepo.conan.dao.ConanMetadataDao +import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest +import com.tencent.bkrepo.conan.service.ConanMetadataService +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Service + +@Service +@Conditional(CommitEdgeEdgeCondition::class) +class EdgeConanMetadataService( + clusterProperties: ClusterProperties, + private val conanMetadataDao: ConanMetadataDao +) : ConanMetadataService(conanMetadataDao) { + private val centerConanMetadataClient: ConanMetadataClient by lazy { + FeignClientFactory.create(clusterProperties.center, "conan", clusterProperties.self.name) + } + + override fun storeMetadata(request: ConanMetadataRequest) { + with(request) { + // 更新center节点metadata信息 + val centerRequest = ConanMetadataRequest( + projectId = projectId, + repoName = repoName, + name = name, + user = user, + channel = channel, + version = version, + recipe = recipe, + ) + centerConanMetadataClient.storeMetadata(centerRequest) + super.storeMetadata(request) + } + } + + override fun delete(projectId: String, repoName: String, recipe: String) { + centerConanMetadataClient.delete(projectId, repoName, recipe) + super.delete(projectId, repoName, recipe) + } +} diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ConanPathUtils.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ConanPathUtils.kt index ebe6ccba4f..6f5e1a0441 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ConanPathUtils.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ConanPathUtils.kt @@ -96,19 +96,6 @@ object ConanPathUtils { } } - fun buildReference(fileReference: ConanFileReference): String { - with(fileReference) { - return StringBuilder(userName) - .append(CharPool.SLASH) - .append(version) - .append(CharPool.AT) - .append(name) - .append(CharPool.SLASH) - .append(channel) - .toString() - } - } - fun buildReferenceWithoutVersion(fileReference: ConanFileReference): String { with(fileReference) { return StringBuilder(userName) @@ -122,7 +109,7 @@ object ConanPathUtils { fun buildPackageReference(packageReference: PackageReference): String { with(packageReference) { - return StringBuilder(buildReference(conRef)) + return StringBuilder(buildConanFileName(conRef)) .append(CharPool.HASH_TAG) .append(conRef.revision) .append(CharPool.COLON) diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ObjectBuildUtil.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ObjectBuildUtil.kt index 997ac0d395..2231406e15 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ObjectBuildUtil.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/utils/ObjectBuildUtil.kt @@ -51,8 +51,8 @@ import com.tencent.bkrepo.conan.pojo.PackageReference import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToPackageReference +import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildPackageReference -import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildReference import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildReferenceWithoutVersion import com.tencent.bkrepo.conan.utils.ConanPathUtils.getPackageRevisionsFile import com.tencent.bkrepo.conan.utils.ConanPathUtils.getRecipeRevisionsFile @@ -147,7 +147,7 @@ object ObjectBuildUtil { with(artifactInfo) { val conanFileReference = convertToConanFileReference(this) val revPath = getRecipeRevisionsFile(conanFileReference) - val refStr = buildReference(conanFileReference) + val refStr = buildConanFileName(conanFileReference) return ConanRecipeUploadRequest( projectId = projectId, repoName = repoName, @@ -167,7 +167,7 @@ object ObjectBuildUtil { with(artifactInfo) { val packageReference = convertToPackageReference(this) val revPath = getRecipeRevisionsFile(packageReference.conRef) - val refStr = buildReference(packageReference.conRef) + val refStr = buildConanFileName(packageReference.conRef) val pRevPath = getPackageRevisionsFile(packageReference) val pRefStr = buildPackageReference(packageReference) return ConanPackageUploadRequest( @@ -192,7 +192,7 @@ object ObjectBuildUtil { with(artifactInfo) { val packageReference = convertToConanFileReference(this) val revPath = getRecipeRevisionsFile(packageReference) - val refStr = buildReference(packageReference) + val refStr = buildConanFileName(packageReference) return ConanRecipeDeleteRequest( projectId = projectId, repoName = repoName, @@ -213,7 +213,7 @@ object ObjectBuildUtil { with(artifactInfo) { val packageReference = PackageReference(convertToConanFileReference(artifactInfo), packageId, pRevision) val revPath = getRecipeRevisionsFile(packageReference.conRef) - val refStr = buildReference(packageReference.conRef) + val refStr = buildConanFileName(packageReference.conRef) val pRevPath = getPackageRevisionsFile(packageReference) val pRefStr = buildPackageReference(packageReference) return ConanPackageDeleteRequest( @@ -236,7 +236,7 @@ object ObjectBuildUtil { with(artifactInfo) { val packageReference = convertToPackageReference(this) val revPath = getRecipeRevisionsFile(packageReference.conRef) - val refStr = buildReference(packageReference.conRef) + val refStr = buildConanFileName(packageReference.conRef) val pRevPath = getPackageRevisionsFile(packageReference) val pRefStr = buildPackageReference(packageReference) return ConanPackageDeleteRequest(