Skip to content

Commit

Permalink
feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587
Browse files Browse the repository at this point in the history
* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587

* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587

* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587

* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587

* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587

* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587

* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587

* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587

* feat:后台admin支持配置文件缓存时间,路径,后缀 TencentBlueKing#1587
  • Loading branch information
lannoy0523 authored Feb 18, 2024
1 parent 4be3a30 commit 5ba5ebb
Show file tree
Hide file tree
Showing 23 changed files with 1,223 additions and 23 deletions.
1 change: 1 addition & 0 deletions src/backend/common/common-api/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,5 @@ dependencies {
api("com.fasterxml.jackson.module:jackson-module-parameter-names")
api("org.apache.commons:commons-compress")
api("com.google.guava:guava")
api("org.springframework.boot:spring-boot-starter-test")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package com.tencent.bkrepo.common.api.serializer

import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.databind.SerializerProvider
import com.fasterxml.jackson.databind.ser.std.StdSerializer
import org.springframework.util.unit.DataSize

class DataSizeSerializer : StdSerializer<DataSize>(DataSize::class.java) {
override fun serialize(value: DataSize?, gen: JsonGenerator?, provider: SerializerProvider?) {
if (gen != null && value != null) {
gen.writeString(value.toMegabytes().toString() + "MB")
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ package com.tencent.bkrepo.common.api.util

import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.SerializationFeature
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer
Expand All @@ -44,6 +45,8 @@ import com.fasterxml.jackson.datatype.jsr310.ser.LocalTimeSerializer
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.jacksonTypeRef
import com.fasterxml.jackson.module.paramnames.ParameterNamesModule
import com.tencent.bkrepo.common.api.serializer.DataSizeSerializer
import org.springframework.util.unit.DataSize
import java.io.InputStream
import java.time.LocalDate
import java.time.LocalDateTime
Expand All @@ -68,7 +71,9 @@ object JsonUtils {
registerModule(javaTimeModule)
registerModule(ParameterNamesModule())
registerModule(Jdk8Module())

var dateSizeModule = SimpleModule()
dateSizeModule.addSerializer(DataSize::class.java, DataSizeSerializer())
registerModule(dateSizeModule)
enable(SerializationFeature.INDENT_OUTPUT)
disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
package com.tencent.bkrepo.job.batch.file

import com.tencent.bkrepo.common.api.pojo.Page
import com.tencent.bkrepo.common.api.pojo.Response
import com.tencent.bkrepo.common.query.enums.OperationType
import com.tencent.bkrepo.common.query.model.Rule
import com.tencent.bkrepo.common.storage.filesystem.cleanup.FileExpireResolver
import com.tencent.bkrepo.job.FULL_PATH
import com.tencent.bkrepo.job.LAST_ACCESS_DATE
import com.tencent.bkrepo.job.SHA256
import com.tencent.bkrepo.job.pojo.TFileCache
import com.tencent.bkrepo.job.service.FileCacheService
import com.tencent.bkrepo.repository.api.NodeClient
import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder
import java.io.File
import java.time.LocalDateTime
import org.slf4j.LoggerFactory
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler
import org.springframework.util.unit.DataSize
import java.io.File
import java.time.LocalDateTime

/**
* 基于仓库配置判断文件是否过期
Expand All @@ -20,6 +25,7 @@ class BasedRepositoryFileExpireResolver(
private val nodeClient: NodeClient,
private val expireConfig: RepositoryExpireConfig,
taskScheduler: ThreadPoolTaskScheduler,
private val fileCacheService: FileCacheService,
) : FileExpireResolver {

private var retainNodes = mutableSetOf<String>()
Expand All @@ -34,35 +40,71 @@ class BasedRepositoryFileExpireResolver(

private fun refreshRetainNode() {
logger.info("Refresh retain nodes.")
val newRetainNodes = mutableSetOf<String>()
expireConfig.repos.forEach {
// for each repo
retainNodes.clear()
getNodeFromConfig()
getNodeFromDataBase()
}

private fun getNodeFromConfig() {
var temp = mutableSetOf<String>()
expireConfig.repos.map{ convertRepoConfigToFileCache(it) }.forEach {
val projectId = it.projectId
val repoName = it.repoName

val queryModel = NodeQueryBuilder()
.projectId(it.projectId)
.repoName(it.repoName)
.excludeFolder()
.size(expireConfig.size.toBytes(), OperationType.GT)
.rule(LAST_ACCESS_DATE, LocalDateTime.now().minusDays(it.days.toLong()), OperationType.AFTER)
.page(1, expireConfig.max)
.select(SHA256, FULL_PATH).build()
val fullPathRuleList = it.pathPrefix.map { prefix ->
Rule.QueryRule(FULL_PATH, prefix, OperationType.PREFIX)
val pages = getNodes(it)
pages.data?.records?.forEach { ret ->
// 获取每个的sha256
val sha256 = ret[SHA256].toString()
val fullPath = ret[FULL_PATH].toString()
temp.add(sha256)
logger.info("Retain node $projectId/$repoName$fullPath, $sha256.")
}
val rule = Rule.NestedRule(fullPathRuleList.toMutableList(), Rule.NestedRule.RelationType.OR)
(queryModel.rule as Rule.NestedRule).rules.add(rule)
val pages = nodeClient.queryWithoutCount(queryModel)
}
retainNodes.addAll(temp)
}

private fun getNodeFromDataBase() {
var temp = mutableSetOf<String>()
fileCacheService.list().forEach {
val projectId = it.projectId
val repoName = it.repoName
val pages = getNodes(it)
pages.data?.records?.forEach { ret ->
// 获取每个的sha256
val sha256 = ret[SHA256].toString()
val fullPath = ret[FULL_PATH].toString()
newRetainNodes.add(sha256)
temp.add(sha256)
logger.info("Retain node $projectId/$repoName$fullPath, $sha256.")
}
}
retainNodes = newRetainNodes
retainNodes.addAll(temp)
}

private fun convertRepoConfigToFileCache(repoConfig: RepoConfig):TFileCache {
return TFileCache(
id = null,
projectId = repoConfig.projectId,
repoName = repoConfig.repoName,
pathPrefix = repoConfig.pathPrefix,
days = repoConfig.days,
size = expireConfig.size.toMegabytes()
)
}

private fun getNodes(tFileCache: TFileCache): Response<Page<Map<String, Any?>>> {
val queryModel = NodeQueryBuilder()
.projectId(tFileCache.projectId)
.repoName(tFileCache.repoName)
.excludeFolder()
.size(DataSize.ofMegabytes(tFileCache.size).toBytes(), OperationType.GT)
.rule(LAST_ACCESS_DATE, LocalDateTime.now().minusDays(tFileCache.days.toLong()), OperationType.AFTER)
.page(1, expireConfig.max)
.select(SHA256, FULL_PATH).build()
val fullPathRuleList = tFileCache.pathPrefix.map { prefix ->
Rule.QueryRule(FULL_PATH, prefix, OperationType.PREFIX)
}
val rule = Rule.NestedRule(fullPathRuleList.toMutableList(), Rule.NestedRule.RelationType.OR)
(queryModel.rule as Rule.NestedRule).rules.add(rule)
return nodeClient.queryWithoutCount(queryModel)
}

companion object {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package com.tencent.bkrepo.job.batch.file

import com.tencent.bkrepo.common.storage.filesystem.cleanup.FileExpireResolver
import com.tencent.bkrepo.job.config.properties.ExpiredCacheFileCleanupJobProperties
import com.tencent.bkrepo.job.service.FileCacheService
import com.tencent.bkrepo.repository.api.NodeClient
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.context.annotation.Bean
Expand All @@ -20,11 +21,13 @@ class ExpireFileResolverConfig {
fun fileExpireResolver(
expiredCacheFileCleanupJobProperties: ExpiredCacheFileCleanupJobProperties,
scheduler: ThreadPoolTaskScheduler,
fileCacheService: FileCacheService
): FileExpireResolver {
return BasedRepositoryFileExpireResolver(
nodeClient,
expiredCacheFileCleanupJobProperties.repoConfig,
scheduler,
fileCacheService
)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.job.controller.user

import com.tencent.bkrepo.common.api.constant.HttpStatus
import com.tencent.bkrepo.common.api.pojo.Response
import com.tencent.bkrepo.common.api.util.toJsonString
import com.tencent.bkrepo.common.security.permission.Principal
import com.tencent.bkrepo.common.security.permission.PrincipalType
import com.tencent.bkrepo.common.service.util.ResponseBuilder
import com.tencent.bkrepo.job.config.properties.ExpiredCacheFileCleanupJobProperties
import com.tencent.bkrepo.job.pojo.FileCacheCheckRequest
import com.tencent.bkrepo.job.pojo.FileCacheRequest
import com.tencent.bkrepo.job.pojo.TFileCache
import com.tencent.bkrepo.job.service.FileCacheService
import org.springframework.web.bind.annotation.DeleteMapping
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.PathVariable
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController

@RestController
@RequestMapping("/api/cache")
@Principal(PrincipalType.ADMIN)
class FileCacheController(
val fileCacheService: FileCacheService,
val properties: ExpiredCacheFileCleanupJobProperties,
) {

@GetMapping("/list")
fun list(): Response<List<TFileCache>> {
return ResponseBuilder.success(fileCacheService.list())
}

@PostMapping("/update")
fun update(@RequestBody request:FileCacheRequest):Response<Void> {
request.id?.let {
var checkStatus = updateCheck(request)
if (checkStatus.status) {
return ResponseBuilder.success()
} else {
return ResponseBuilder.fail(HttpStatus.BAD_REQUEST.value, checkStatus.msg)
}
} ?: let {
return ResponseBuilder.fail(HttpStatus.BAD_REQUEST.value, "id is null")
}
}

fun updateCheck(request: FileCacheRequest):CheckStatus {
fileCacheService.getById(request.id!!)?.let {
var fileCacheCheckRequest = FileCacheCheckRequest(
projectId = request.projectId,
repoName = request.repoName,
days = request.days,
size = request.size
)
fileCacheService.checkExist(fileCacheCheckRequest)?.let {
if( it.id != request.id) {
var checkStatus = CheckStatus(
status = false,
msg = "has same config"
)
return checkStatus
}
}
fileCacheService.update(request)
var checkStatus = CheckStatus(
status = true,
msg = ""
)
return checkStatus
}
var checkStatus = CheckStatus(
status = false,
msg = "id not existed"
)
return checkStatus
}

// 新增
@PostMapping("/create")
fun create(@RequestBody request:FileCacheRequest):Response<Void> {
var fileCacheCheckRequest = FileCacheCheckRequest(
repoName = request.repoName,
projectId = request.projectId,
days = request.days,
size = request.size
)
fileCacheService.checkExist(fileCacheCheckRequest)?.let {
return ResponseBuilder.fail(HttpStatus.BAD_REQUEST.value, "has same config")
}
fileCacheService.create(request)
return ResponseBuilder.success()
}

// 删除
@DeleteMapping("/delete/{id}")
fun delete(@PathVariable id:String): Response<Void> {
fileCacheService.getById(id)?.let {
fileCacheService.delete(id)
return ResponseBuilder.success()
}
return ResponseBuilder.fail(HttpStatus.BAD_REQUEST.value, "id not existed")
}

// 获取配置中的属性
@GetMapping("/config")
fun getConfig(): Response<String> {
var config = properties
return ResponseBuilder.success(config.toJsonString())
}

}

data class CheckStatus(
val status: Boolean,
val msg: String
)
Loading

0 comments on commit 5ba5ebb

Please sign in to comment.