feat: conan全局search以及index优化#2735

src/backend/conan/api-conan/build.gradle.kts

@@ -34,4 +34,5 @@ dependencies {
     implementation(project(":common:common-api"))
     implementation(project(":common:common-artifact:artifact-api"))
     compileOnly("org.springframework:spring-web")
+    compileOnly("org.springframework.cloud:spring-cloud-openfeign-core")
 }

src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/api/ConanMetadataClient.kt

@@ -0,0 +1,54 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.conan.api
+
+import com.tencent.bkrepo.common.api.constant.MAVEN_SERVICE_NAME
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest
+import io.swagger.annotations.ApiOperation
+import org.springframework.cloud.openfeign.FeignClient
+import org.springframework.web.bind.annotation.PostMapping
+import org.springframework.web.bind.annotation.RequestBody
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
+
+@FeignClient(MAVEN_SERVICE_NAME, contextId = "ConanMetadataClient")
+@RequestMapping("/service/metadata")
+interface ConanMetadataClient {
+    @ApiOperation("存储conan版本元数据")
+    @PostMapping("/update")
+    fun storeMetadata(@RequestBody request: ConanMetadataRequest): Response<Void>
+
+    @ApiOperation("删除conan版本元数据")
+    @PostMapping("/delete")
+    fun delete(
+        @RequestParam projectId: String,
+        @RequestParam repoName: String,
+        @RequestParam recipe: String
+    ): Response<Void>
+}

src/backend/conan/api-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/metadata/ConanMetadataRequest.kt

@@ -0,0 +1,38 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.conan.pojo.metadata
+
+data class ConanMetadataRequest(
+    val projectId: String,
+    val repoName: String,
+    val user: String,
+    val name: String,
+    val version: String,
+    val channel: String,
+    val recipe: String
+)

src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/artifact/repository/ConanLocalRepository.kt

@@ -41,8 +41,11 @@ import com.tencent.bkrepo.conan.constant.X_CHECKSUM_SHA1
 import com.tencent.bkrepo.conan.listener.event.ConanPackageUploadEvent
 import com.tencent.bkrepo.conan.listener.event.ConanRecipeUploadEvent
 import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo
+import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest
+import com.tencent.bkrepo.conan.service.ConanMetadataService
 import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference
 import com.tencent.bkrepo.conan.utils.ConanPathUtils
+import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName
 import com.tencent.bkrepo.conan.utils.ConanPathUtils.generateFullPath
 import com.tencent.bkrepo.conan.utils.ObjectBuildUtil
 import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.buildDownloadResponse
@@ -55,11 +58,15 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
 import com.tencent.bkrepo.repository.pojo.packages.PackageVersion
 import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
 class ConanLocalRepository : LocalRepository() {
 
+    @Autowired
+    lateinit var conanMetadataService: ConanMetadataService
+
     override fun buildNodeCreateRequest(context: ArtifactUploadContext): NodeCreateRequest {
         with(context) {
             val tempArtifactInfo = context.artifactInfo as ConanArtifactInfo
@@ -134,6 +141,16 @@ class ConanLocalRepository : LocalRepository() {
         packageService.createPackageVersion(packageVersionCreateRequest).apply {
             logger.info("user: [$userId] create package version [$packageVersionCreateRequest] success!")
         }
+        val request = ConanMetadataRequest(
+            projectId = artifactInfo.projectId,
+            repoName = artifactInfo.repoName,
+            name = artifactInfo.name,
+            version = artifactInfo.version,
+            user = artifactInfo.userName,
+            channel = artifactInfo.channel,
+            recipe = buildConanFileName(convertToConanFileReference(artifactInfo))
+        )
+        conanMetadataService.storeMetadata(request)
     }
 
     override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? {

src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt

@@ -46,13 +46,17 @@ import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo
 import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_PACKAGE_DELETE_URL
 import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_VERSION_DELETE_URL
 import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_VERSION_DETAIL
+import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest
 import com.tencent.bkrepo.conan.service.ConanDeleteService
+import com.tencent.bkrepo.conan.service.ConanExtService
 import io.swagger.annotations.Api
 import io.swagger.annotations.ApiOperation
 import io.swagger.annotations.ApiParam
 import org.springframework.web.bind.annotation.DeleteMapping
 import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestAttribute
+import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.bind.annotation.RequestMapping
 import org.springframework.web.bind.annotation.RequestParam
 import org.springframework.web.bind.annotation.RestController
@@ -63,6 +67,7 @@ import org.springframework.web.bind.annotation.RestController
 @RequestMapping("/ext")
 class UserConanController(
     private val conanDeleteService: ConanDeleteService,
+    private val conanExtService: ConanExtService
 ) {
 
     @ApiOperation("查询包的版本详情")
@@ -149,4 +154,70 @@ class UserConanController(
     fun getRegistryDomain(): Response<ConanDomainInfo> {
         return ResponseBuilder.success(conanDeleteService.getDomain())
     }
+
+    @ApiOperation("仓库索引修正")
+    @PostMapping("/repo/index/refresh/{projectId}/{repoName}")
+    @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
+    fun repoIndexRefresh(
+        @ArtifactPathVariable artifactInfo: ConanArtifactInfo,
+    ): Response<PackageVersionInfo> {
+        conanExtService.indexRefreshForRepo(artifactInfo.projectId, artifactInfo.repoName)
+        return ResponseBuilder.success()
+    }
+
+    @ApiOperation("recipe索引修正")
+    @PostMapping("/recipe/index/refresh/{projectId}/{repoName}")
+    @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
+    fun recipeIndexRefresh(
+        @ArtifactPathVariable artifactInfo: ConanArtifactInfo,
+        @RequestBody request: IndexRefreshRequest
+    ): Response<Void> {
+        conanExtService.indexRefreshForRecipe(artifactInfo.projectId, artifactInfo.repoName, request)
+        return ResponseBuilder.success()
+    }
+
+
+    @ApiOperation("packagekey下索引修正")
+    @PostMapping("/packagekey/index/refresh/{projectId}/{repoName}")
+    @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
+    fun packageKeyIndexRefresh(
+        @ArtifactPathVariable artifactInfo: ConanArtifactInfo,
+        @RequestParam packageKey: String,
+    ): Response<Void> {
+        conanExtService.indexRefreshByPackageKey(artifactInfo.projectId, artifactInfo.repoName, packageKey)
+        return ResponseBuilder.success()
+    }
+
+    @ApiOperation("重新生成仓库元数据信息")
+    @PostMapping("/metadata/refresh/{projectId}/{repoName}")
+    @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
+    fun metadataRefresh(
+        @ArtifactPathVariable artifactInfo: ConanArtifactInfo,
+    ): Response<Void> {
+        conanExtService.metadataRefresh(artifactInfo.projectId, artifactInfo.repoName)
+        return ResponseBuilder.success()
+    }
+
+    @ApiOperation("重新生成包元数据信息")
+    @PostMapping("/metadata/package/refresh/{projectId}/{repoName}")
+    @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
+    fun packageMetadataRefresh(
+        @ArtifactPathVariable artifactInfo: ConanArtifactInfo,
+        @RequestParam packageKey: String,
+    ): Response<Void> {
+        conanExtService.packageMetadataRefresh(artifactInfo.projectId, artifactInfo.repoName, packageKey)
+        return ResponseBuilder.success()
+    }
+
+    @ApiOperation("重新生成版本元数据信息")
+    @PostMapping("/metadata/version/refresh/{projectId}/{repoName}")
+    @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
+    fun versionMetadataRefresh(
+        @ArtifactPathVariable artifactInfo: ConanArtifactInfo,
+        @RequestParam packageKey: String,
+        @RequestParam version: String,
+    ): Response<Void> {
+        conanExtService.versionMetadataRefresh(artifactInfo.projectId, artifactInfo.repoName, packageKey, version)
+        return ResponseBuilder.success()
+    }
 }

src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/service/ConanMetadataController.kt

@@ -0,0 +1,48 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.conan.controller.service
+
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.service.util.ResponseBuilder
+import com.tencent.bkrepo.conan.api.ConanMetadataClient
+import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest
+import com.tencent.bkrepo.conan.service.ConanMetadataService
+import org.springframework.web.bind.annotation.RestController
+
+@RestController
+class ConanMetadataController(private val conanMetadataService: ConanMetadataService) : ConanMetadataClient {
+    override fun storeMetadata(request: ConanMetadataRequest): Response<Void> {
+        conanMetadataService.storeMetadata(request)
+        return ResponseBuilder.success()
+    }
+
+    override fun delete(projectId: String, repoName: String, recipe: String): Response<Void> {
+        conanMetadataService.delete(projectId, repoName, recipe)
+        return ResponseBuilder.success()
+    }
+}

src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/dao/ConanMetadataDao.kt

@@ -0,0 +1,35 @@
+package com.tencent.bkrepo.conan.dao
+
+import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao
+import com.tencent.bkrepo.conan.model.TConanMetadataRecord
+import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest
+import org.springframework.data.mongodb.core.query.Criteria
+import org.springframework.data.mongodb.core.query.Query
+import org.springframework.data.mongodb.core.query.Update
+import org.springframework.stereotype.Repository
+
+@Repository
+class ConanMetadataDao : SimpleMongoDao<TConanMetadataRecord>() {
+
+    fun findAndModify(request: ConanMetadataRequest) {
+        with(request) {
+            val query = Query(buildCriteria(projectId, repoName, recipe))
+            val update = Update().set(TConanMetadataRecord::name.name, name)
+                .set(TConanMetadataRecord::user.name, user)
+                .set(TConanMetadataRecord::version.name, version)
+                .set(TConanMetadataRecord::channel.name, channel)
+            upsert(query, update)
+        }
+    }
+
+    fun delete(projectId: String, repoName: String, recipe: String) {
+        remove(Query(buildCriteria(projectId, repoName, recipe)))
+    }
+
+    private fun buildCriteria(projectId: String, repoName: String, recipe: String): Criteria {
+        return Criteria
+            .where(TConanMetadataRecord::projectId.name).`is`(projectId)
+            .and(TConanMetadataRecord::repoName.name).`is`(repoName)
+            .and(TConanMetadataRecord::recipe.name).`is`(recipe)
+    }
+}

src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/model/TConanMetadataRecord.kt

@@ -0,0 +1,25 @@
+package com.tencent.bkrepo.conan.model
+
+import org.springframework.data.mongodb.core.index.CompoundIndex
+import org.springframework.data.mongodb.core.index.CompoundIndexes
+import org.springframework.data.mongodb.core.mapping.Document
+
+@Document("conan_metadata")
+@CompoundIndexes(
+    CompoundIndex(
+        name = "unique_index",
+        def = "{'projectId':1, 'repoName':1, 'recipe':1}",
+        background = true,
+        unique = true
+    )
+)
+data class TConanMetadataRecord(
+    val id: String?,
+    val projectId: String,
+    val repoName: String,
+    val user: String,
+    val name: String,
+    val version: String,
+    val channel: String,
+    val recipe: String
+)

src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/pojo/request/IndexRefreshRequest.kt

@@ -0,0 +1,37 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.conan.pojo.request
+
+data class IndexRefreshRequest(
+    val name: String,
+    val version: String,
+    val userName: String,
+    val channel: String,
+    val revision: String? = null,
+    val packageId: String? = null,
+)