Skip to content

Commit

Permalink
refactor: pypi query结构调整 TencentBlueKing#1749
Browse files Browse the repository at this point in the history
  • Loading branch information
scplsy authored Feb 20, 2024
1 parent 54fb6ba commit 430d1a5
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 51 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.pypi.constants

const val REPO_TYPE = "PYPI"

const val METADATA = "metadata"
const val FIELD_NAME = "Name"
const val FIELD_VERSION = "Version"
const val NAME = "name"
const val VERSION = "version"

const val QUERY_TYPE = "queryType"
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package com.tencent.bkrepo.pypi.constants

enum class PypiQueryType {
PACKAGE_INDEX,
VERSION_INDEX,
VERSION_DETAIL
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,10 @@

package com.tencent.bkrepo.pypi.artifact.repository

import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.api.constant.ensureSuffix
import com.tencent.bkrepo.common.api.exception.BadRequestException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.artifact.api.ArtifactFile
import com.tencent.bkrepo.common.artifact.api.ArtifactInfo
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext
Expand All @@ -41,7 +40,6 @@ import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository
import com.tencent.bkrepo.common.artifact.resolve.file.multipart.MultipartArtifactFile
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.util.PackageKeys
import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter
import com.tencent.bkrepo.common.artifact.util.version.SemVersion
import com.tencent.bkrepo.common.artifact.util.version.SemVersionParser
import com.tencent.bkrepo.common.query.enums.OperationType
Expand All @@ -51,10 +49,11 @@ import com.tencent.bkrepo.common.query.model.Rule
import com.tencent.bkrepo.common.query.model.Sort
import com.tencent.bkrepo.common.service.util.HttpContextHolder
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.pypi.artifact.PypiProperties
import com.tencent.bkrepo.pypi.artifact.url.UrlPatternUtil.parameterMaps
import com.tencent.bkrepo.pypi.artifact.xml.Value
import com.tencent.bkrepo.pypi.artifact.xml.XmlUtil
import com.tencent.bkrepo.pypi.constants.PypiQueryType
import com.tencent.bkrepo.pypi.constants.QUERY_TYPE
import com.tencent.bkrepo.pypi.exception.PypiSimpleNotFoundException
import com.tencent.bkrepo.pypi.pojo.Basic
import com.tencent.bkrepo.pypi.pojo.PypiArtifactVersionData
Expand All @@ -76,8 +75,7 @@ import org.springframework.stereotype.Component

@Component
class PypiLocalRepository(
private val stageClient: StageClient,
private val pypiProperties: PypiProperties
private val stageClient: StageClient
) : LocalRepository() {

/**
Expand Down Expand Up @@ -254,16 +252,15 @@ class PypiLocalRepository(
* 2,pypi simple html页面
*/
override fun query(context: ArtifactQueryContext): Any? {
val servletPath = ArtifactContextHolder.getUrlPath(this.javaClass.name)!!
return if (servletPath.startsWith("/ext/version/detail")) {
// 请求版本详情
getVersionDetail(context)
} else {
getSimpleHtml(context.artifactInfo)
return when (val queryType = context.getAttribute<PypiQueryType>(QUERY_TYPE)) {
PypiQueryType.PACKAGE_INDEX,
PypiQueryType.VERSION_INDEX -> getSimpleHtml(context.artifactInfo, queryType)
PypiQueryType.VERSION_DETAIL -> getVersionDetail(context)
null -> throw BadRequestException(CommonMessageCode.REQUEST_CONTENT_INVALID)
}
}

fun getVersionDetail(context: ArtifactQueryContext): Any? {
fun getVersionDetail(context: ArtifactQueryContext): PypiArtifactVersionData? {
val packageKey = context.request.getParameter("packageKey")
val version = context.request.getParameter("version")
logger.info("Get version detail. packageKey[$packageKey], version[$version]")
Expand Down Expand Up @@ -300,33 +297,11 @@ class PypiLocalRepository(
}
}

/**
* 本地调试删除 pypi.domain 配置
*/
fun getRedirectUrl(path: String): String {
val domain = pypiProperties.domain
return UrlFormatter.format(domain, path).ensureSuffix(StringPool.SLASH)
}

/**
*
*/
fun getSimpleHtml(artifactInfo: ArtifactInfo): Any? {
fun getSimpleHtml(artifactInfo: ArtifactInfo, type: PypiQueryType): String? {
logger.info("Get simple html. artifactInfo[${artifactInfo.getArtifactFullPath()}]")
val path = ArtifactContextHolder.getUrlPath(this.javaClass.name)!!
if (!path.endsWith("/")) {
val response = HttpContextHolder.getResponse()
response.sendRedirect(getRedirectUrl(path))
return null
}
with(artifactInfo) {
val node = nodeClient.getNodeDetail(projectId, repoName, getArtifactFullPath()).data
?: throw PypiSimpleNotFoundException(getArtifactFullPath())
if (!node.folder) {
return null
}
// 请求不带包名,返回包名列表.
if (getArtifactFullPath() == "/") {
if (type == PypiQueryType.PACKAGE_INDEX) {
val nodeList = nodeClient.listNode(
projectId, repoName, getArtifactFullPath(), includeFolder = true, deep = true
).data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,22 @@

package com.tencent.bkrepo.pypi.artifact.repository

import com.tencent.bkrepo.common.api.exception.BadRequestException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.artifact.api.ArtifactFile
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
import com.tencent.bkrepo.common.artifact.repository.remote.RemoteRepository
import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.service.util.HttpContextHolder
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.pypi.FLUSH_CACHE_EXPIRE
import com.tencent.bkrepo.pypi.REMOTE_HTML_CACHE_FULL_PATH
import com.tencent.bkrepo.pypi.XML_RPC_URI
import com.tencent.bkrepo.pypi.artifact.xml.XmlConvertUtil
import com.tencent.bkrepo.pypi.constants.PypiQueryType
import com.tencent.bkrepo.pypi.constants.QUERY_TYPE
import com.tencent.bkrepo.pypi.exception.PypiRemoteSearchException
import com.tencent.bkrepo.pypi.util.XmlUtils.readXml
import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
Expand Down Expand Up @@ -83,18 +86,11 @@ class PypiRemoteRepository : RemoteRepository() {
}

override fun query(context: ArtifactQueryContext): Any? {
val response = HttpContextHolder.getResponse()
response.contentType = "text/html"
if (context.artifactInfo.getArtifactFullPath() == "/") {
val cacheHtml = getCacheHtml(context) ?: "Can not cache remote html"
response.setContentLength(cacheHtml.length)
response.writer.print(cacheHtml)
} else {
val responseStr = remoteRequest(context) ?: ""
response.setContentLength(responseStr.length)
response.writer.print(responseStr)
return when (context.getAttribute<PypiQueryType>(QUERY_TYPE)) {
PypiQueryType.PACKAGE_INDEX -> getCacheHtml(context) ?: "Can not cache remote html"
PypiQueryType.VERSION_INDEX -> remoteRequest(context) ?: ""
else -> throw BadRequestException(CommonMessageCode.REQUEST_CONTENT_INVALID)
}
return null
}

fun remoteRequest(context: ArtifactQueryContext): String? {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,30 @@ package com.tencent.bkrepo.pypi.service

import com.tencent.bkrepo.auth.pojo.enums.PermissionAction
import com.tencent.bkrepo.auth.pojo.enums.ResourceType
import com.tencent.bkrepo.common.api.constant.StringPool.SLASH
import com.tencent.bkrepo.common.artifact.api.ArtifactFileMap
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext
import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService
import com.tencent.bkrepo.common.security.permission.Permission
import com.tencent.bkrepo.common.service.util.HttpContextHolder
import com.tencent.bkrepo.pypi.artifact.PypiArtifactInfo
import com.tencent.bkrepo.pypi.artifact.PypiProperties
import com.tencent.bkrepo.pypi.artifact.xml.Value
import com.tencent.bkrepo.pypi.artifact.xml.XmlConvertUtil
import com.tencent.bkrepo.pypi.artifact.xml.XmlUtil
import com.tencent.bkrepo.pypi.constants.PypiQueryType
import com.tencent.bkrepo.pypi.constants.QUERY_TYPE
import com.tencent.bkrepo.pypi.util.UrlUtils
import org.springframework.stereotype.Service

@Service
class PypiService : ArtifactService() {
class PypiService(
private val pypiProperties: PypiProperties
) : ArtifactService() {

@Permission(ResourceType.REPO, PermissionAction.READ)
fun packages(pypiArtifactInfo: PypiArtifactInfo) {
Expand All @@ -57,7 +66,15 @@ class PypiService : ArtifactService() {

@Permission(ResourceType.REPO, PermissionAction.READ)
fun simple(artifactInfo: PypiArtifactInfo): Any? {
val urlPath = ArtifactContextHolder.getUrlPath(this.javaClass.name)!!
if (!urlPath.endsWith(SLASH)) {
HttpContextHolder.getResponse().sendRedirect(UrlUtils.getRedirectUrl(pypiProperties.domain, urlPath))
return null
}
val context = ArtifactQueryContext()
val artifactName = context.artifactInfo.getArtifactName()
val queryType = if (artifactName == SLASH) PypiQueryType.PACKAGE_INDEX else PypiQueryType.VERSION_INDEX
context.putAttribute(QUERY_TYPE, queryType)
return repository.query(context)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ import com.tencent.bkrepo.common.artifact.util.version.SemVersion
import com.tencent.bkrepo.common.artifact.util.version.SemVersionParser
import com.tencent.bkrepo.common.security.permission.Permission
import com.tencent.bkrepo.pypi.artifact.PypiArtifactInfo
import com.tencent.bkrepo.pypi.constants.PypiQueryType
import com.tencent.bkrepo.pypi.constants.QUERY_TYPE
import com.tencent.bkrepo.repository.api.NodeClient
import com.tencent.bkrepo.repository.api.PackageClient
import com.tencent.bkrepo.repository.pojo.node.NodeInfo
Expand Down Expand Up @@ -72,6 +74,7 @@ class PypiWebService(
@Permission(type = ResourceType.REPO, action = PermissionAction.READ)
fun artifactDetail(pypiArtifactInfo: PypiArtifactInfo, packageKey: String, version: String?): Any? {
val context = ArtifactQueryContext()
context.putAttribute(QUERY_TYPE, PypiQueryType.VERSION_DETAIL)
return repository.query(context)
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package com.tencent.bkrepo.pypi.util

import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.api.constant.ensureSuffix
import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter

object UrlUtils {
fun getRedirectUrl(domain: String, requestPath: String): String {
return UrlFormatter.format(domain, requestPath).ensureSuffix(StringPool.SLASH)
}
}

0 comments on commit 430d1a5

Please sign in to comment.