Skip to content

Commit

Permalink
feat: conan全局search以及index优化#2735
Browse files Browse the repository at this point in the history
* feat: conan 全局search优化#2735

* feat: 索引文件调整,并增加索引文件刷新接口#2735

* feat: 代码调整#2735

* feat: 增加元数据刷新接口#2735

* feat: 增加元数据刷新接口#2735

* feat: 依赖调整#2735

* feat: 代码调整#2735

* feat: 代码调整#2735
  • Loading branch information
zacYL authored Nov 14, 2024
1 parent a7f9af1 commit ce0aff2
Show file tree
Hide file tree
Showing 20 changed files with 891 additions and 56 deletions.
1 change: 1 addition & 0 deletions src/backend/conan/api-conan/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,5 @@ dependencies {
implementation(project(":common:common-api"))
implementation(project(":common:common-artifact:artifact-api"))
compileOnly("org.springframework:spring-web")
compileOnly("org.springframework.cloud:spring-cloud-openfeign-core")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.conan.api

import com.tencent.bkrepo.common.api.constant.MAVEN_SERVICE_NAME
import com.tencent.bkrepo.common.api.pojo.Response
import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest
import io.swagger.annotations.ApiOperation
import org.springframework.cloud.openfeign.FeignClient
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RequestParam

@FeignClient(MAVEN_SERVICE_NAME, contextId = "ConanMetadataClient")
@RequestMapping("/cluster/metadata")
interface ConanMetadataClient {
@ApiOperation("存储conan版本元数据")
@PostMapping("/update")
fun storeMetadata(@RequestBody request: ConanMetadataRequest): Response<Void>

@ApiOperation("删除conan版本元数据")
@PostMapping("/delete")
fun delete(
@RequestParam projectId: String,
@RequestParam repoName: String,
@RequestParam recipe: String
): Response<Void>
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.conan.pojo.metadata

data class ConanMetadataRequest(
val projectId: String,
val repoName: String,
val user: String,
val name: String,
val version: String,
val channel: String,
val recipe: String
)
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,11 @@ import com.tencent.bkrepo.conan.constant.X_CHECKSUM_SHA1
import com.tencent.bkrepo.conan.listener.event.ConanPackageUploadEvent
import com.tencent.bkrepo.conan.listener.event.ConanRecipeUploadEvent
import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo
import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest
import com.tencent.bkrepo.conan.service.ConanMetadataService
import com.tencent.bkrepo.conan.utils.ConanArtifactInfoUtil.convertToConanFileReference
import com.tencent.bkrepo.conan.utils.ConanPathUtils
import com.tencent.bkrepo.conan.utils.ConanPathUtils.buildConanFileName
import com.tencent.bkrepo.conan.utils.ConanPathUtils.generateFullPath
import com.tencent.bkrepo.conan.utils.ObjectBuildUtil
import com.tencent.bkrepo.conan.utils.ObjectBuildUtil.buildDownloadResponse
Expand All @@ -55,11 +58,15 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDetail
import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
import com.tencent.bkrepo.repository.pojo.packages.PackageVersion
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component

@Component
class ConanLocalRepository : LocalRepository() {

@Autowired
lateinit var conanMetadataService: ConanMetadataService

override fun buildNodeCreateRequest(context: ArtifactUploadContext): NodeCreateRequest {
with(context) {
val tempArtifactInfo = context.artifactInfo as ConanArtifactInfo
Expand Down Expand Up @@ -134,6 +141,16 @@ class ConanLocalRepository : LocalRepository() {
packageService.createPackageVersion(packageVersionCreateRequest).apply {
logger.info("user: [$userId] create package version [$packageVersionCreateRequest] success!")
}
val request = ConanMetadataRequest(
projectId = artifactInfo.projectId,
repoName = artifactInfo.repoName,
name = artifactInfo.name,
version = artifactInfo.version,
user = artifactInfo.userName,
channel = artifactInfo.channel,
recipe = buildConanFileName(convertToConanFileReference(artifactInfo))
)
conanMetadataService.storeMetadata(request)
}

override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,17 @@ import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo
import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_PACKAGE_DELETE_URL
import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_VERSION_DELETE_URL
import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.CONAN_VERSION_DETAIL
import com.tencent.bkrepo.conan.pojo.request.IndexRefreshRequest
import com.tencent.bkrepo.conan.service.ConanDeleteService
import com.tencent.bkrepo.conan.service.ConanExtService
import io.swagger.annotations.Api
import io.swagger.annotations.ApiOperation
import io.swagger.annotations.ApiParam
import org.springframework.web.bind.annotation.DeleteMapping
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestAttribute
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RequestParam
import org.springframework.web.bind.annotation.RestController
Expand All @@ -63,6 +67,7 @@ import org.springframework.web.bind.annotation.RestController
@RequestMapping("/ext")
class UserConanController(
private val conanDeleteService: ConanDeleteService,
private val conanExtService: ConanExtService
) {

@ApiOperation("查询包的版本详情")
Expand Down Expand Up @@ -149,4 +154,70 @@ class UserConanController(
fun getRegistryDomain(): Response<ConanDomainInfo> {
return ResponseBuilder.success(conanDeleteService.getDomain())
}

@ApiOperation("仓库索引修正")
@PostMapping("/repo/index/refresh/{projectId}/{repoName}")
@Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
fun repoIndexRefresh(
@ArtifactPathVariable artifactInfo: ConanArtifactInfo,
): Response<PackageVersionInfo> {
conanExtService.indexRefreshForRepo(artifactInfo.projectId, artifactInfo.repoName)
return ResponseBuilder.success()
}

@ApiOperation("recipe索引修正")
@PostMapping("/recipe/index/refresh/{projectId}/{repoName}")
@Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
fun recipeIndexRefresh(
@ArtifactPathVariable artifactInfo: ConanArtifactInfo,
@RequestBody request: IndexRefreshRequest
): Response<Void> {
conanExtService.indexRefreshForRecipe(artifactInfo.projectId, artifactInfo.repoName, request)
return ResponseBuilder.success()
}


@ApiOperation("packagekey下索引修正")
@PostMapping("/packagekey/index/refresh/{projectId}/{repoName}")
@Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
fun packageKeyIndexRefresh(
@ArtifactPathVariable artifactInfo: ConanArtifactInfo,
@RequestParam packageKey: String,
): Response<Void> {
conanExtService.indexRefreshByPackageKey(artifactInfo.projectId, artifactInfo.repoName, packageKey)
return ResponseBuilder.success()
}

@ApiOperation("重新生成仓库元数据信息")
@PostMapping("/metadata/refresh/{projectId}/{repoName}")
@Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
fun metadataRefresh(
@ArtifactPathVariable artifactInfo: ConanArtifactInfo,
): Response<Void> {
conanExtService.metadataRefresh(artifactInfo.projectId, artifactInfo.repoName)
return ResponseBuilder.success()
}

@ApiOperation("重新生成包元数据信息")
@PostMapping("/metadata/package/refresh/{projectId}/{repoName}")
@Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
fun packageMetadataRefresh(
@ArtifactPathVariable artifactInfo: ConanArtifactInfo,
@RequestParam packageKey: String,
): Response<Void> {
conanExtService.packageMetadataRefresh(artifactInfo.projectId, artifactInfo.repoName, packageKey)
return ResponseBuilder.success()
}

@ApiOperation("重新生成版本元数据信息")
@PostMapping("/metadata/version/refresh/{projectId}/{repoName}")
@Permission(type = ResourceType.REPO, action = PermissionAction.WRITE)
fun versionMetadataRefresh(
@ArtifactPathVariable artifactInfo: ConanArtifactInfo,
@RequestParam packageKey: String,
@RequestParam version: String,
): Response<Void> {
conanExtService.versionMetadataRefresh(artifactInfo.projectId, artifactInfo.repoName, packageKey, version)
return ResponseBuilder.success()
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.conan.controller.service

import com.tencent.bkrepo.common.api.pojo.Response
import com.tencent.bkrepo.common.service.util.ResponseBuilder
import com.tencent.bkrepo.conan.api.ConanMetadataClient
import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest
import com.tencent.bkrepo.conan.service.ConanMetadataService
import org.springframework.web.bind.annotation.RestController

@RestController
class ConanMetadataController(private val conanMetadataService: ConanMetadataService) : ConanMetadataClient {
override fun storeMetadata(request: ConanMetadataRequest): Response<Void> {
conanMetadataService.storeMetadata(request)
return ResponseBuilder.success()
}

override fun delete(projectId: String, repoName: String, recipe: String): Response<Void> {
conanMetadataService.delete(projectId, repoName, recipe)
return ResponseBuilder.success()
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package com.tencent.bkrepo.conan.dao

import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao
import com.tencent.bkrepo.conan.model.TConanMetadataRecord
import com.tencent.bkrepo.conan.pojo.metadata.ConanMetadataRequest
import org.springframework.data.mongodb.core.query.Criteria
import org.springframework.data.mongodb.core.query.Query
import org.springframework.data.mongodb.core.query.Update
import org.springframework.stereotype.Repository

@Repository
class ConanMetadataDao : SimpleMongoDao<TConanMetadataRecord>() {

fun findAndModify(request: ConanMetadataRequest) {
with(request) {
val query = Query(buildCriteria(projectId, repoName, recipe))
val update = Update().set(TConanMetadataRecord::name.name, name)
.set(TConanMetadataRecord::user.name, user)
.set(TConanMetadataRecord::version.name, version)
.set(TConanMetadataRecord::channel.name, channel)
upsert(query, update)
}
}

fun delete(projectId: String, repoName: String, recipe: String) {
remove(Query(buildCriteria(projectId, repoName, recipe)))
}

private fun buildCriteria(projectId: String, repoName: String, recipe: String): Criteria {
return Criteria
.where(TConanMetadataRecord::projectId.name).`is`(projectId)
.and(TConanMetadataRecord::repoName.name).`is`(repoName)
.and(TConanMetadataRecord::recipe.name).`is`(recipe)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package com.tencent.bkrepo.conan.model

import org.springframework.data.mongodb.core.index.CompoundIndex
import org.springframework.data.mongodb.core.index.CompoundIndexes
import org.springframework.data.mongodb.core.mapping.Document

@Document("conan_metadata")
@CompoundIndexes(
CompoundIndex(
name = "unique_index",
def = "{'projectId':1, 'repoName':1, 'recipe':1}",
background = true,
unique = true
)
)
data class TConanMetadataRecord(
val id: String?,
val projectId: String,
val repoName: String,
val user: String,
val name: String,
val version: String,
val channel: String,
val recipe: String
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.conan.pojo.request

data class IndexRefreshRequest(
val name: String,
val version: String,
val userName: String,
val channel: String,
val revision: String? = null,
val packageId: String? = null,
)
Loading

0 comments on commit ce0aff2

Please sign in to comment.