diff --git a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/config/ArtifactPreloadProperties.kt b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/config/ArtifactPreloadProperties.kt index 814ca613a7..736b68876f 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/config/ArtifactPreloadProperties.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/config/ArtifactPreloadProperties.kt @@ -85,4 +85,14 @@ data class ArtifactPreloadProperties( * 是否仅模拟预加载，为true时不执行加载计划，仅输出一条日志 */ var mock: Boolean = false, + /** + * 是否在缓存保留时生成预加载计划 + * + * 系统中存在缓存保留策略时可能会由于保留缓存导致较大的空间占用，此时可以通过配置预加载策略，仅在需要时才将文件加载到缓存中减少空间占用， + * 但是由于直接移除缓存保留策略可能影响系统稳定性，不移除保留策略会导致缓存文件不会被清理无法触发预加载计划生成，无法验证预加载效果， + * 因此增加此开关，在缓存应该被清理，但是由于保留策略导致缓存未被清理时，也能触发预加载计划生成，验证预加载效果后再移除保留策略 + * + * 需要注意此开关开启的同时需要开启[mock]，仅模拟预加载验证效果而不执行实际的加载操作，需要等所有计划执行完毕才能关闭此开关与[mock] + */ + var generateOnRetained: Boolean = false, ) diff --git a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt index 9dba4d4537..d3f4d1c1a3 100644 --- a/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt +++ b/src/backend/common/common-artifact/artifact-cache/src/main/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt @@ -30,6 +30,8 @@ package com.tencent.bkrepo.common.artifact.cache.service.impl import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadProperties import com.tencent.bkrepo.common.artifact.cache.service.ArtifactPreloadPlanService import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileDeletedEvent +import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileEventData +import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileRetainedEvent import org.slf4j.LoggerFactory import org.springframework.context.event.EventListener import org.springframework.scheduling.annotation.Async @@ -49,10 +51,26 @@ open class CacheFileEventListener( @EventListener(CacheFileDeletedEvent::class) open fun onCacheFileDeleted(event: CacheFileDeletedEvent) { if (properties.enabled && event.data.size >= properties.minSize.toBytes()) { - with(event.data) { - logger.info("try generate preload plan for sha256[${sha256}], fullPath[$fullPath], size[$size") - preloadPlanService.generatePlan(credentials.key, sha256) - } + generatePreloadPlan(event.data) + } + } + + /** + * 缓存被保留时判断是否需要创建预加载执行计划 + */ + @Async + @EventListener(CacheFileRetainedEvent::class) + open fun onCacheFileDeleted(event: CacheFileRetainedEvent) { + if (properties.enabled && event.data.size >= properties.minSize.toBytes() && properties.generateOnRetained) { + generatePreloadPlan(event.data) + } + } + + + private fun generatePreloadPlan(data: CacheFileEventData) { + with(data) { + logger.info("try generate preload plan for sha256[${sha256}], fullPath[$fullPath], size[$size") + preloadPlanService.generatePlan(credentials.key, sha256) } } diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt index f45824ee78..dfc905f58d 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt @@ -229,19 +229,30 @@ class CacheStorageService( filename: String, credentials: StorageCredentials, ): Boolean { + val cacheFilePath = "${credentials.cache.path}$path$filename" return if (doExist(path, filename, credentials)) { - val cacheFilePath = "${credentials.cache.path}$path$filename" val size = File(cacheFilePath).length() getCacheClient(credentials).delete(path, filename) cacheFileEventPublisher.publishCacheFileDeletedEvent(path, filename, size, credentials) - logger.info("Cache [${credentials.cache.path}/$path/$filename] was deleted") + logger.info("Cache [$cacheFilePath] was deleted") true } else { - logger.info("Cache file[${credentials.cache.path}/$path/$filename] was not in storage") + logger.info("Cache file[$cacheFilePath] was not in storage") false } } + /** + * 判断缓存文件是否存在 + */ + fun cacheExists( + path: String, + filename: String, + credentials: StorageCredentials, + ): Boolean { + return getCacheClient(credentials).exist(path, filename) + } + /** * 获取存储的缓存目录健康状态 */ diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/event/CacheFileRetainedEvent.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/event/CacheFileRetainedEvent.kt new file mode 100644 index 0000000000..e1f2c7d64e --- /dev/null +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/event/CacheFileRetainedEvent.kt @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.storage.core.cache.event + +data class CacheFileRetainedEvent(val data: CacheFileEventData) diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/filesystem/cleanup/CleanupFileVisitor.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/filesystem/cleanup/CleanupFileVisitor.kt index 68b1004af0..bd1ac6cec8 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/filesystem/cleanup/CleanupFileVisitor.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/filesystem/cleanup/CleanupFileVisitor.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.artifact.constant.SHA256_STR_LENGTH import com.tencent.bkrepo.common.storage.core.FileStorage import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileDeletedEvent import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileEventData +import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileRetainedEvent import com.tencent.bkrepo.common.storage.core.locator.FileLocator import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.filesystem.ArtifactFileVisitor @@ -78,20 +79,29 @@ class CleanupFileVisitor( val file = filePath.toFile() val expired = fileExpireResolver.isExpired(file) val retain = fileRetainResolver?.retain(file) ?: false - if (expired && !retain && !isNFSTempFile(filePath)) { - if (isTempFile || existInStorage(filePath)) { - rateLimiter.acquire() - Files.delete(filePath) - result.cleanupFile += 1 - result.cleanupSize += size - deleted = true - onFileCleaned(filePath, size) - logger.info("Clean up file[$filePath], size[$size], summary: $result") + + var shouldDelete = expired && !isNFSTempFile(filePath) + if (shouldDelete && !isTempFile) { + val existInStorage = existInStorage(filePath) + if (!existInStorage) { + logger.info("cache file[${filePath}] not exists in storage[${credentials.key}]") } + shouldDelete = existInStorage + } + + if (shouldDelete && !retain) { + rateLimiter.acquire() + Files.delete(filePath) + result.cleanupFile += 1 + result.cleanupSize += size + deleted = true + onFileCleaned(filePath, size) + logger.info("Clean up file[$filePath], size[$size], summary: $result") } - if (expired && retain) { + if (shouldDelete && retain) { result.retainFile += 1 result.retainSize += size + onFileRetained(filePath, size) } } catch (ignored: Exception) { logger.error("Clean file[${filePath.fileName}] error.", ignored) @@ -192,20 +202,26 @@ class CleanupFileVisitor( } private fun onFileCleaned(filePath: Path, size: Long) { - val fileName = filePath.fileName.toString() val event = FileDeletedEvent( credentials = credentials, rootPath = rootPath.toString(), fullPath = filePath.toString(), ) - if (rootPath == credentials.cache.path.toPath() && filePath.fileName.toString().length == SHA256_STR_LENGTH) { - val data = CacheFileEventData(credentials, fileName, filePath.toString(), size) + if (isCacheFile(filePath)) { + val data = buildCacheFileEventData(filePath, size) publisher.publishEvent(CacheFileDeletedEvent(data)) } publisher.publishEvent(event) } + private fun onFileRetained(filePath: Path, size: Long) { + if (isCacheFile(filePath)) { + val data = buildCacheFileEventData(filePath, size) + publisher.publishEvent(CacheFileRetainedEvent(data)) + } + } + private fun onFileSurvived(filePath: Path) { val event = FileSurvivedEvent( credentials = credentials, @@ -215,6 +231,15 @@ class CleanupFileVisitor( publisher.publishEvent(event) } + private fun buildCacheFileEventData(filePath: Path, size: Long): CacheFileEventData { + val fileName = filePath.fileName.toString() + return CacheFileEventData(credentials, fileName, filePath.toString(), size) + } + + private fun isCacheFile(filePath: Path): Boolean { + return rootPath == credentials.cache.path.toPath() && filePath.fileName.toString().length == SHA256_STR_LENGTH + } + companion object { private val logger = LoggerFactory.getLogger(JOB_LOGGER_NAME) private const val permitsPerSecond = 30.0 diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/StorageCacheIndexerCustomizer.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/StorageCacheIndexerCustomizer.kt index c475d97908..4c99d49e7a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/StorageCacheIndexerCustomizer.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/StorageCacheIndexerCustomizer.kt @@ -30,6 +30,8 @@ package com.tencent.bkrepo.job.batch.task.cache import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.core.cache.CacheStorageService +import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileEventData +import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileRetainedEvent import com.tencent.bkrepo.common.storage.core.cache.indexer.IndexerCustomizer import com.tencent.bkrepo.common.storage.core.cache.indexer.StorageCacheIndexer import com.tencent.bkrepo.common.storage.core.cache.indexer.listener.StorageEldestRemovedListener @@ -37,6 +39,7 @@ import com.tencent.bkrepo.common.storage.core.cache.indexer.metrics.StorageCache import com.tencent.bkrepo.common.storage.core.locator.FileLocator import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.filesystem.cleanup.FileRetainResolver +import org.springframework.context.ApplicationEventPublisher import org.springframework.stereotype.Component @Component @@ -45,11 +48,14 @@ class StorageCacheIndexerCustomizer( private val fileLocator: FileLocator, private val storageService: StorageService, private val storageCacheIndexerMetrics: StorageCacheIndexerMetrics? = null, + private val publisher: ApplicationEventPublisher, ) : IndexerCustomizer { override fun customize(indexer: StorageCacheIndexer, credentials: StorageCredentials) { if (storageService is CacheStorageService) { indexer.addEldestRemovedListener( - EldestRemovedListener(credentials, fileLocator, storageService, storageCacheIndexerMetrics, resolver) + EldestRemovedListener( + credentials, fileLocator, storageService, storageCacheIndexerMetrics, resolver, publisher + ) ) } } @@ -63,11 +69,21 @@ class StorageCacheIndexerCustomizer( storageService: CacheStorageService, storageCacheIndexerMetrics: StorageCacheIndexerMetrics?, private val resolver: FileRetainResolver, + private val publisher: ApplicationEventPublisher, ) : StorageEldestRemovedListener(storageCredentials, fileLocator, storageService, storageCacheIndexerMetrics) { override fun onEldestRemoved(key: String, value: Long) { if (!resolver.retain(key)) { super.onEldestRemoved(key, value) } else { + // publish retained event + val path = fileLocator.locate(key) + if (storageService.cacheExists(path, key, storageCredentials)) { + val fullPath = "${storageCredentials.cache.path}$path$key" + val data = CacheFileEventData(storageCredentials, key, fullPath, value) + publisher.publishEvent(CacheFileRetainedEvent(data)) + } + + // metrics val storageKey = storageCredentials.key ?: DEFAULT_STORAGE_KEY storageCacheIndexerMetrics?.evicted(storageKey, value, false) }