diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/OverloadException.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/OverloadException.kt new file mode 100644 index 0000000000..35150db7d7 --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/OverloadException.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.exception + +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.message.CommonMessageCode + +/** + * 超过限流配置异常 + */ +class OverloadException( + val resource: String +) : ErrorCodeException(CommonMessageCode.RATE_LIMITER_OVERLOAD, resource, status = HttpStatus.TOO_MANY_REQUESTS) \ No newline at end of file diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt index 84a3d35d8d..280eb767d8 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt @@ -61,6 +61,9 @@ enum class CommonMessageCode(private val key: String) : MessageCode { MEDIA_TYPE_UNACCEPTABLE("system.media-type.unacceptable"), TOO_MANY_REQUESTS("too.many.requests"), PIPELINE_NOT_RUNNING("pipeline.not-running"), + INVALID_CONFIG("system.config.invalid"), + ACQUIRE_LOCK_FAILED("acquire.lock.failed"), + RATE_LIMITER_OVERLOAD("rate.limiter.overload") ; override fun getBusinessCode() = ordinal + 1 diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties index c5923db62d..1dbf17a2bd 100644 --- a/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties +++ b/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties @@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=Cross location operation is not allowed system.media-type.unacceptable=Unacceptable Media Type too.many.requests=Too Many Requests: {0} pipeline.not-running=Pipeline[{0}] is not running status +system.config.invalid=Config [{0}] is invalid +acquire.lock.failed=acquire lock failed:[{0}] +rate.limiter.overload=resource requests reached rate limit:[{0}] \ No newline at end of file diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties index a298ce086a..afda36c8c1 100644 --- a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties @@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=不允许跨地点操作 system.media-type.unacceptable=不接受的Media Type too.many.requests=请求过多: {0} pipeline.not-running=流水线[{0}]不是运行状态 +system.config.invalid=配置[{0}]无效 +acquire.lock.failed=获取锁失败: [{0}] +rate.limiter.overload=资源请求量超过限流值: [{0}] \ No newline at end of file diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties index 815c32bce1..bbac10f772 100644 --- a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties @@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=不允許跨地點操作 system.media-type.unacceptable=不接受的Media Type too.many.requests=請求過多: {0} pipeline.not-running=流水線[{0}]不是運行狀態 +system.config.invalid=配置[{0}]無效 +acquire.lock.failed=獲取鎖失敗: [{0}] +rate.limiter.overload=資源請求量超過限流值: [{0}] \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/build.gradle.kts b/src/backend/common/common-artifact/artifact-service/build.gradle.kts index 42c9e945d4..4750552336 100644 --- a/src/backend/common/common-artifact/artifact-service/build.gradle.kts +++ b/src/backend/common/common-artifact/artifact-service/build.gradle.kts @@ -40,6 +40,7 @@ dependencies { api(project(":common:common-security")) api(project(":common:common-artifact:artifact-api")) api(project(":common:common-storage:storage-service")) + api(project(":common:common-ratelimiter")) api(project(":common:common-stream")) api(project(":common:common-metrics-push")) api(project(":common:common-metadata:metadata-service")) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/ArtifactResolverConfiguration.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/ArtifactResolverConfiguration.kt index 22d3dbfb65..5bb8d5741b 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/ArtifactResolverConfiguration.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/ArtifactResolverConfiguration.kt @@ -39,6 +39,7 @@ import com.tencent.bkrepo.common.artifact.resolve.path.ResolverMap import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter import com.tencent.bkrepo.common.artifact.resolve.response.DefaultArtifactResourceWriter import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -84,8 +85,14 @@ class ArtifactResolverConfiguration { @Bean @ConditionalOnMissingBean(ArtifactResourceWriter::class) - fun artifactResourceWriter(storageProperties: StorageProperties): ArtifactResourceWriter { - return DefaultArtifactResourceWriter(storageProperties) + fun artifactResourceWriter( + storageProperties: StorageProperties, + requestLimitCheckService: RequestLimitCheckService + ): ArtifactResourceWriter { + return DefaultArtifactResourceWriter( + storageProperties, + requestLimitCheckService + ) } @Bean diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactDataReceiver.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactDataReceiver.kt index 1fa20ddcc6..bcf75261e3 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactDataReceiver.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactDataReceiver.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.common.artifact.resolve.file import com.tencent.bkrepo.common.api.constant.retry +import com.tencent.bkrepo.common.api.exception.OverloadException import com.tencent.bkrepo.common.artifact.exception.ArtifactReceiveException import com.tencent.bkrepo.common.artifact.hash.sha256 import com.tencent.bkrepo.common.artifact.metrics.ArtifactMetrics @@ -35,14 +36,15 @@ import com.tencent.bkrepo.common.artifact.metrics.TrafficHandler import com.tencent.bkrepo.common.artifact.stream.DigestCalculateListener import com.tencent.bkrepo.common.artifact.stream.rateLimit import com.tencent.bkrepo.common.artifact.util.http.IOExceptionUtils +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream +import com.tencent.bkrepo.common.storage.config.MonitorProperties import com.tencent.bkrepo.common.storage.config.ReceiveProperties import com.tencent.bkrepo.common.storage.core.locator.HashFileLocator -import com.tencent.bkrepo.common.storage.config.MonitorProperties import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor import com.tencent.bkrepo.common.storage.monitor.Throughput import com.tencent.bkrepo.common.storage.util.createFile import com.tencent.bkrepo.common.storage.util.delete -import org.slf4j.LoggerFactory import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.IOException @@ -57,6 +59,7 @@ import java.security.SecureRandom import java.time.Duration import kotlin.math.abs import kotlin.system.measureTimeMillis +import org.slf4j.LoggerFactory /** * artifact数据接收类，作用： @@ -75,6 +78,8 @@ class ArtifactDataReceiver( private val filename: String = generateRandomName(), private val randomPath: Boolean = false, private val originPath: Path = path, + private val requestLimitCheckService: RequestLimitCheckService? = null, + private val contentLength: Long? = null, ) : StorageHealthMonitor.Observer, AutoCloseable { /** @@ -187,9 +192,15 @@ class ArtifactDataReceiver( startTime = System.nanoTime() } try { + requestLimitCheckService?.uploadBandwidthCheck( + length.toLong(), + receiveProperties.circuitBreakerThreshold + ) writeData(chunk, offset, length) } catch (exception: IOException) { handleIOException(exception) + } catch (overloadEx: OverloadException) { + handleOverloadException(overloadEx) } } @@ -203,6 +214,9 @@ class ArtifactDataReceiver( startTime = System.nanoTime() } try { + requestLimitCheckService?.uploadBandwidthCheck( + 1, receiveProperties.circuitBreakerThreshold + ) checkFallback() outputStream.write(b) listener.data(b) @@ -210,6 +224,8 @@ class ArtifactDataReceiver( checkThreshold() } catch (exception: IOException) { handleIOException(exception) + } catch (overloadEx: OverloadException) { + handleOverloadException(overloadEx) } } @@ -222,8 +238,13 @@ class ArtifactDataReceiver( if (startTime == 0L) { startTime = System.nanoTime() } + var rateLimitFlag = false + var exp: Exception? = null try { - val input = source.rateLimit(receiveProperties.rateLimit.toBytes()) + val input = requestLimitCheckService?.bandwidthCheck( + source, receiveProperties.circuitBreakerThreshold, contentLength + ) ?: source.rateLimit(receiveProperties.rateLimit.toBytes()) + rateLimitFlag = input is CommonRateLimitInputStream val buffer = ByteArray(bufferSize) input.use { var bytes = input.read(buffer) @@ -233,7 +254,15 @@ class ArtifactDataReceiver( } } } catch (exception: IOException) { + exp = exception handleIOException(exception) + } catch (overloadEx: OverloadException) { + exp = overloadEx + handleOverloadException(overloadEx) + } finally { + if (rateLimitFlag) { + requestLimitCheckService?.bandwidthFinish(exp) + } } } @@ -322,9 +351,7 @@ class ArtifactDataReceiver( * 处理IO异常 */ private fun handleIOException(exception: IOException) { - finished = true - endTime = System.nanoTime() - close() + finishWithException() if (IOExceptionUtils.isClientBroken(exception)) { throw ArtifactReceiveException(exception.message.orEmpty()) } else { @@ -332,6 +359,20 @@ class ArtifactDataReceiver( } } + /** + * 处理限流请求 + */ + private fun handleOverloadException(exception: OverloadException) { + finishWithException() + throw exception + } + + private fun finishWithException() { + finished = true + endTime = System.nanoTime() + close() + } + /** * 检查是否需要fall back操作 */ diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactFileFactory.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactFileFactory.kt index d99b4b93c5..90de9a6769 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactFileFactory.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactFileFactory.kt @@ -37,6 +37,7 @@ import com.tencent.bkrepo.common.artifact.resolve.file.multipart.MultipartArtifa import com.tencent.bkrepo.common.artifact.resolve.file.stream.StreamArtifactFile import com.tencent.bkrepo.common.bksync.BlockChannel import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitorHelper @@ -54,17 +55,20 @@ import java.io.InputStream class ArtifactFileFactory( storageProperties: StorageProperties, storageHealthMonitorHelper: StorageHealthMonitorHelper, + private val limitCheckService: RequestLimitCheckService ) { init { monitorHelper = storageHealthMonitorHelper properties = storageProperties + requestLimitCheckService = limitCheckService } companion object { private lateinit var monitorHelper: StorageHealthMonitorHelper private lateinit var properties: StorageProperties + private lateinit var requestLimitCheckService: RequestLimitCheckService const val ARTIFACT_FILES = "artifact.files" @@ -89,34 +93,49 @@ class ArtifactFileFactory( * 构造分块接收数据的artifact file */ fun buildChunked(): ChunkedArtifactFile { - return ChunkedArtifactFile(getMonitor(), properties, getStorageCredentials()).apply { + return ChunkedArtifactFile( + getMonitor(), properties, getStorageCredentials(), + ).apply { track(this) } } fun buildChunked(storageCredentials: StorageCredentials): ChunkedArtifactFile { - return ChunkedArtifactFile(getMonitor(storageCredentials), properties, storageCredentials).apply { + return ChunkedArtifactFile( + getMonitor(storageCredentials), properties, storageCredentials, + ).apply { track(this) } } fun buildDfsArtifactFile(): RandomAccessArtifactFile { - return RandomAccessArtifactFile(getMonitor(), getStorageCredentials(), properties).apply { + return RandomAccessArtifactFile( + getMonitor(), getStorageCredentials(), properties, + ).apply { + track(this) + } + } + + /** + * 通过输入流构造artifact file, 主要针对上传请求对其做限流操作 + * @param inputStream 输入流 + */ + fun buildWithRateLimiter(inputStream: InputStream, contentLength: Long? = null): ArtifactFile { + return StreamArtifactFile( + inputStream, getMonitor(), properties, getStorageCredentials(), contentLength, + requestLimitCheckService = requestLimitCheckService + ).apply { track(this) } } /** - * 通过输入流构造artifact file + * 通过输入流构造artifact file，服务内部输入流转换成文件使用 * @param inputStream 输入流 */ fun build(inputStream: InputStream, contentLength: Long? = null): ArtifactFile { return StreamArtifactFile( - inputStream, - getMonitor(), - properties, - getStorageCredentials(), - contentLength, + inputStream, getMonitor(), properties, getStorageCredentials(), contentLength ).apply { track(this) } @@ -137,10 +156,8 @@ class ArtifactFileFactory( */ fun build(multipartFile: MultipartFile, storageCredentials: StorageCredentials): ArtifactFile { return MultipartArtifactFile( - multipartFile, - getMonitor(storageCredentials), - properties, - storageCredentials, + multipartFile, getMonitor(storageCredentials), properties, storageCredentials, + requestLimitCheckService = requestLimitCheckService ).apply { track(this) } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/chunk/RandomAccessArtifactFile.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/chunk/RandomAccessArtifactFile.kt index cbea40ee8b..1855681dcd 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/chunk/RandomAccessArtifactFile.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/chunk/RandomAccessArtifactFile.kt @@ -23,7 +23,7 @@ import java.nio.file.NoSuchFileException class RandomAccessArtifactFile( private val monitor: StorageHealthMonitor, private val storageCredentials: StorageCredentials, - storageProperties: StorageProperties + storageProperties: StorageProperties, ) : ArtifactFile { /** @@ -43,7 +43,9 @@ class RandomAccessArtifactFile( init { val path = storageCredentials.upload.location.toPath() - receiver = ArtifactDataReceiver(storageProperties.receive, storageProperties.monitor, path) + receiver = ArtifactDataReceiver( + storageProperties.receive, storageProperties.monitor, path, + ) monitor.add(receiver) if (!monitor.healthy.get()) { receiver.unhealthy(monitor.getFallbackPath(), monitor.fallBackReason) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/multipart/MultipartArtifactFile.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/multipart/MultipartArtifactFile.kt index 657a4db3ce..75b207f496 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/multipart/MultipartArtifactFile.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/multipart/MultipartArtifactFile.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.common.artifact.resolve.file.multipart import com.tencent.bkrepo.common.artifact.resolve.file.stream.StreamArtifactFile import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor import org.springframework.web.multipart.MultipartFile @@ -37,9 +38,11 @@ class MultipartArtifactFile( private val multipartFile: MultipartFile, monitor: StorageHealthMonitor, storageProperties: StorageProperties, - storageCredentials: StorageCredentials + storageCredentials: StorageCredentials, + requestLimitCheckService: RequestLimitCheckService ) : StreamArtifactFile( - multipartFile.inputStream, monitor, storageProperties, storageCredentials, multipartFile.size + multipartFile.inputStream, monitor, storageProperties, storageCredentials, multipartFile.size, + requestLimitCheckService ) { fun getOriginalFilename() = multipartFile.originalFilename.orEmpty() } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/ArtifactFileMethodArgumentResolver.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/ArtifactFileMethodArgumentResolver.kt index 524fe08496..ed7d4e06ff 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/ArtifactFileMethodArgumentResolver.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/ArtifactFileMethodArgumentResolver.kt @@ -60,6 +60,6 @@ class ArtifactFileMethodArgumentResolver : HandlerMethodArgumentResolver { } private fun resolveOctetStream(request: HttpServletRequest): ArtifactFile { - return ArtifactFileFactory.build(request.inputStream, request.contentLengthLong) + return ArtifactFileFactory.buildWithRateLimiter(request.inputStream, request.contentLengthLong) } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/StreamArtifactFile.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/StreamArtifactFile.kt index f08f0c7445..dad470cfbc 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/StreamArtifactFile.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/StreamArtifactFile.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.event.ArtifactReceivedEvent import com.tencent.bkrepo.common.artifact.hash.sha1 import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactDataReceiver +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.common.storage.credentials.StorageCredentials @@ -47,7 +48,8 @@ open class StreamArtifactFile( private val monitor: StorageHealthMonitor, private val storageProperties: StorageProperties, private val storageCredentials: StorageCredentials, - private val contentLength: Long? = null + private val contentLength: Long? = null, + private val requestLimitCheckService: RequestLimitCheckService? = null ) : ArtifactFile { /** @@ -83,7 +85,9 @@ open class StreamArtifactFile( storageProperties.receive, storageProperties.monitor, receivePath, - randomPath = !useLocalPath + randomPath = !useLocalPath, + requestLimitCheckService = requestLimitCheckService, + contentLength = contentLength ) if (!storageProperties.receive.resolveLazily) { init() diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt index 403c4d16c4..cf7078da6a 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt @@ -36,6 +36,9 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.stream.rateLimit import com.tencent.bkrepo.common.artifact.util.http.IOExceptionUtils import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream import com.tencent.bkrepo.common.storage.monitor.Throughput import com.tencent.bkrepo.common.storage.monitor.measureThroughput import org.slf4j.LoggerFactory @@ -47,7 +50,8 @@ import javax.servlet.http.HttpServletResponse abstract class AbstractArtifactResourceHandler( - private val storageProperties: StorageProperties + private val storageProperties: StorageProperties, + private val requestLimitCheckService: RequestLimitCheckService ) : ArtifactResourceWriter { /** * 获取动态buffer size @@ -88,6 +92,14 @@ abstract class AbstractArtifactResourceHandler( } } + /** + * 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接 + */ + protected fun downloadRateLimitCheck(resource: ArtifactResource) { + val applyPermits = resource.getSingleStream().range.length + requestLimitCheckService.postLimitCheck(applyPermits) + } + /** * 将数据流以Range方式写入响应 */ @@ -100,13 +112,23 @@ abstract class AbstractArtifactResourceHandler( if (request.method == HttpMethod.HEAD.name) { return Throughput.EMPTY } + val length = inputStream.range.length + var rateLimitFlag = false + var exp: Exception? = null val recordAbleInputStream = RecordAbleInputStream(inputStream) try { return measureThroughput { - recordAbleInputStream.rateLimit(responseRateLimitWrapper(storageProperties.response.rateLimit)).use { + val stream = requestLimitCheckService.bandwidthCheck( + recordAbleInputStream, storageProperties.response.circuitBreakerThreshold, + length + ) ?: recordAbleInputStream.rateLimit( + responseRateLimitWrapper(storageProperties.response.rateLimit) + ) + rateLimitFlag = stream is CommonRateLimitInputStream + stream.use { it.copyTo( out = response.outputStream, - bufferSize = getBufferSize(inputStream.range.length) + bufferSize = getBufferSize(length) ) } } @@ -116,13 +138,21 @@ abstract class AbstractArtifactResourceHandler( // org.springframework.http.converter.HttpMessageNotWritableException异常，会重定向到/error页面 // 又因为/error页面不存在，最终返回404，所以要对IOException进行包装，在上一层捕捉处理 val message = exception.message.orEmpty() - val status = if (IOExceptionUtils.isClientBroken(exception)){ + val status = if (IOExceptionUtils.isClientBroken(exception)) { HttpStatus.BAD_REQUEST } else { logger.warn("write range stream failed", exception) HttpStatus.INTERNAL_SERVER_ERROR } + exp = exception throw ArtifactResponseException(message, status) + } catch (overloadEx: OverloadException) { + exp = overloadEx + throw overloadEx + } finally { + if (rateLimitFlag) { + requestLimitCheckService.bandwidthFinish(exp) + } } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt index 12e9fefee9..df9b39d9c0 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.OverloadException import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_MD5 import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_SHA256 import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException @@ -43,6 +44,8 @@ import com.tencent.bkrepo.common.artifact.stream.rateLimit import com.tencent.bkrepo.common.artifact.util.http.HttpHeaderUtils.determineMediaType import com.tencent.bkrepo.common.artifact.util.http.HttpHeaderUtils.encodeDisposition import com.tencent.bkrepo.common.artifact.util.http.IOExceptionUtils.isClientBroken +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream import com.tencent.bkrepo.common.service.otel.util.TraceHeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.config.StorageProperties @@ -55,6 +58,7 @@ import java.io.IOException import java.time.LocalDateTime import java.time.ZoneOffset import java.time.format.DateTimeFormatter +import java.util.Locale import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream import javax.servlet.http.HttpServletRequest @@ -64,12 +68,16 @@ import javax.servlet.http.HttpServletResponse * ArtifactResourceWriter默认实现 */ open class DefaultArtifactResourceWriter( - private val storageProperties: StorageProperties -) : AbstractArtifactResourceHandler(storageProperties) { + private val storageProperties: StorageProperties, + private val requestLimitCheckService: RequestLimitCheckService +) : AbstractArtifactResourceHandler( + storageProperties, requestLimitCheckService +) { - @Throws(ArtifactResponseException::class) + @Throws(ArtifactResponseException::class, OverloadException::class) override fun write(resource: ArtifactResource): Throughput { responseRateLimitCheck() + downloadRateLimitCheck(resource) TraceHeaderUtils.setResponseHeader() return if (resource.containsMultiArtifact()) { writeMultiArtifact(resource) @@ -88,7 +96,7 @@ open class DefaultArtifactResourceWriter( val name = resource.getSingleName() val range = resource.getSingleStream().range val cacheControl = resource.node?.metadata?.get(HttpHeaders.CACHE_CONTROL)?.toString() - ?: resource.node?.metadata?.get(HttpHeaders.CACHE_CONTROL.toLowerCase())?.toString() + ?: resource.node?.metadata?.get(HttpHeaders.CACHE_CONTROL.lowercase(Locale.getDefault()))?.toString() ?: StringPool.NO_CACHE response.bufferSize = getBufferSize(range.length) @@ -176,6 +184,8 @@ open class DefaultArtifactResourceWriter( if (request.method == HttpMethod.HEAD.name) { return Throughput.EMPTY } + var rateLimitFlag = false + var exp: Exception? = null try { return measureThroughput { val zipOutput = ZipOutputStream(response.outputStream.buffered()) @@ -184,9 +194,14 @@ open class DefaultArtifactResourceWriter( resource.artifactMap.forEach { (name, inputStream) -> val recordAbleInputStream = RecordAbleInputStream(inputStream) zipOutput.putNextEntry(generateZipEntry(name, inputStream)) - recordAbleInputStream.rateLimit( + val stream = requestLimitCheckService.bandwidthCheck( + recordAbleInputStream, storageProperties.response.circuitBreakerThreshold, + inputStream.range.length + ) ?: recordAbleInputStream.rateLimit( responseRateLimitWrapper(storageProperties.response.rateLimit) - ).use { + ) + rateLimitFlag = stream is CommonRateLimitInputStream + stream.use { it.copyTo( out = zipOutput, bufferSize = getBufferSize(inputStream.range.length) @@ -205,8 +220,15 @@ open class DefaultArtifactResourceWriter( logger.warn("write zip stream failed", exception) HttpStatus.INTERNAL_SERVER_ERROR } + exp = exception throw ArtifactResponseException(message, status) + } catch (overloadEx: OverloadException) { + exp = overloadEx + throw overloadEx } finally { + if (rateLimitFlag) { + requestLimitCheckService.bandwidthFinish(exp) + } resource.artifactMap.values.forEach { it.closeQuietly() } } } diff --git a/src/backend/common/common-ratelimiter/build.gradle.kts b/src/backend/common/common-ratelimiter/build.gradle.kts new file mode 100644 index 0000000000..aca0432d66 --- /dev/null +++ b/src/backend/common/common-ratelimiter/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +dependencies { + api("io.micrometer:micrometer-registry-prometheus") + api(project(":common:common-artifact:artifact-api")) + api(project(":common:common-redis")) + api(project(":common:common-api")) + api(project(":common:common-security")) + api(project(":common:common-mongo")) + testImplementation("org.springframework.boot:spring-boot-starter-test") + testImplementation("de.flapdoodle.embed:de.flapdoodle.embed.mongo") + testImplementation("org.mockito.kotlin:mockito-kotlin") + testImplementation("io.mockk:mockk") + testImplementation(project(":common:common-redis")) + testImplementation("it.ozimov:embedded-redis:${Versions.EmbeddedRedis}") { + exclude("org.slf4j", "slf4j-simple") + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfiguration.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfiguration.kt new file mode 100644 index 0000000000..37731778cc --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfiguration.kt @@ -0,0 +1,265 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.interceptor.RateLimitHandlerInterceptor +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.repository.RateLimitRepository +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.service.bandwidth.DownloadBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.bandwidth.UploadBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.UrlRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.UrlRepoRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.user.UserUrlRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.user.UserUrlRepoRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.DownloadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.UploadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.user.UserDownloadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.user.UserUploadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Import +import org.springframework.core.Ordered +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.web.servlet.config.annotation.InterceptorRegistry +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer + +@Configuration +@EnableConfigurationProperties(RateLimiterProperties::class) +@ConditionalOnWebApplication +@Import(RateLimitRepository::class) +class RateLimiterAutoConfiguration { + + @Bean + fun rateLimitService(rateLimitRepository: RateLimitRepository): RateLimiterConfigService { + return RateLimiterConfigService(rateLimitRepository) + } + + @Bean + fun rateLimiterMetrics(registry: MeterRegistry): RateLimiterMetrics { + return RateLimiterMetrics(registry) + } + + @Bean(URL_REPO_RATELIMITER_SERVICE) + fun urlRepoRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UrlRepoRateLimiterService { + return UrlRepoRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(URL_RATELIMITER_SERVICE) + fun urlRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UrlRateLimiterService { + return UrlRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(USER_URL_REPO_RATELIMITER_SERVICE) + fun userUrlRepoRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UserUrlRepoRateLimiterService { + return UserUrlRepoRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(UPLOAD_USAGE_RATELIMITER_SERVICE) + fun uploadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UploadUsageRateLimiterService { + return UploadUsageRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(DOWNLOAD_USAGE_RATELIMITER_SERVICE) + fun downloadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): DownloadUsageRateLimiterService { + return DownloadUsageRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(USER_DOWNLOAD_USAGE_RATELIMITER_SERVICE) + fun userDownloadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UserDownloadUsageRateLimiterService { + return UserDownloadUsageRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean(USER_UPLOAD_USAGE_RATELIMITER_SERVICE) + fun userUploadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UserUploadUsageRateLimiterService { + return UserUploadUsageRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean(USER_URL_RATELIMITER_SERVICE) + fun userUrlRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UserUrlRateLimiterService { + return UserUrlRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean(UPLOAD_BANDWIDTH_RATELIMITER_ERVICE) + fun uploadBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UploadBandwidthRateLimiterService { + return UploadBandwidthRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean(DOWNLOAD_BANDWIDTH_RATELIMITER_SERVICE) + fun downloadBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): DownloadBandwidthRateLimiterService { + return DownloadBandwidthRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean + fun requestLimitCheckService( + rateLimiterProperties: RateLimiterProperties, + ): RequestLimitCheckService { + return RequestLimitCheckService(rateLimiterProperties) + } + + @Bean + @ConditionalOnWebApplication + fun rateLimitHandlerInterceptorRegister( + requestLimitCheckService: RequestLimitCheckService + ): WebMvcConfigurer { + return object : WebMvcConfigurer { + override fun addInterceptors(registry: InterceptorRegistry) { + registry.addInterceptor( + RateLimitHandlerInterceptor( + requestLimitCheckService = requestLimitCheckService + ) + ) + .excludePathPatterns("/service/**", "/replica/**") + .order(Ordered.LOWEST_PRECEDENCE) + super.addInterceptors(registry) + } + } + } + + + companion object { + const val DOWNLOAD_BANDWIDTH_RATELIMITER_SERVICE = "downloadBandwidthRateLimiterService" + const val UPLOAD_BANDWIDTH_RATELIMITER_ERVICE = "uploadBandwidthRateLimiterService" + const val USER_URL_RATELIMITER_SERVICE = "userUrlRateLimiterService" + const val USER_UPLOAD_USAGE_RATELIMITER_SERVICE = "userUploadUsageRateLimiterService" + const val USER_DOWNLOAD_USAGE_RATELIMITER_SERVICE = "userDownloadUsageRateLimiterService" + const val DOWNLOAD_USAGE_RATELIMITER_SERVICE = "downloadUsageRateLimiterService" + const val UPLOAD_USAGE_RATELIMITER_SERVICE = "uploadUsageRateLimiterService" + const val URL_RATELIMITER_SERVICE = "urlRateLimiterService" + const val URL_REPO_RATELIMITER_SERVICE = "urlRepoRateLimiterService" + const val USER_URL_REPO_RATELIMITER_SERVICE = "userUrlRepoRateLimiterService" + + } + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiter.kt new file mode 100644 index 0000000000..c6322b576e --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiter.kt @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.redis.LuaScript +import java.time.Duration +import kotlin.system.measureTimeMillis +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.data.redis.core.script.DefaultRedisScript + +/** + * 分布式固定时间窗口算法实现 + */ +class DistributedFixedWindowRateLimiter( + private val key: String, + private val limit: Long, + private val duration: Duration, + private val redisTemplate: RedisTemplate, +) : RateLimiter { + override fun tryAcquire(permits: Long): Boolean { + try { + var acquireResult = false + val elapsedTime = measureTimeMillis { + val redisScript = DefaultRedisScript(LuaScript.fixWindowRateLimiterScript, Long::class.java) + // 注意， 由于redis expire只支持秒为单位，所以周期最小单位为秒 + val result = redisTemplate.execute( + redisScript, listOf(key), limit.toString(), permits.toString(), duration.seconds.toString() + ) + acquireResult = result == 1L + } + if (logger.isDebugEnabled) { + logger.debug( + "acquire distributed fixed window rateLimiter " + + "elapsed time: $elapsedTime ms, acquireResult: $acquireResult" + ) + } + return acquireResult + } catch (e: Exception) { + logger.warn("${this.javaClass.simpleName} acquire error: ${e.message}") + throw AcquireLockFailedException("distributed lock acquire failed: $e") + } + } + + override fun removeCacheLimit(key: String) { + redisTemplate.delete(key) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DistributedFixedWindowRateLimiter::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiter.kt new file mode 100644 index 0000000000..ed03b9e806 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiter.kt @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.redis.LuaScript +import kotlin.system.measureTimeMillis +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.data.redis.core.script.DefaultRedisScript + +/** + * 分布式漏桶算法实现 + */ +class DistributedLeakyRateLimiter( + private val key: String, + private val permitsPerSecond: Double, + private val capacity: Long, + private val redisTemplate: RedisTemplate, +) : RateLimiter { + override fun tryAcquire(permits: Long): Boolean { + try { + var acquireResult = false + val elapsedTime = measureTimeMillis { + val redisScript = DefaultRedisScript(LuaScript.leakyRateLimiterScript, List::class.java) + // 时间统一从redis server获取 + // lua脚本中使用命令获取时间指令需要配合replicate_commands()使用，但是由于redis只有在某个特定版本上才支持该指令， + // 所以无法从lua脚本中去获取时间，只能分为多次调用。 + val currentTime = redisTemplate.execute { connection -> + connection.time() + } ?: System.currentTimeMillis() + val currentSeconds = (currentTime / 1000) + val results = redisTemplate.execute( + redisScript, getKeys(key), permitsPerSecond.toString(), + capacity.toString(), permits.toString(), currentSeconds.toString() + ) + acquireResult = results[0] == 1L + } + if (logger.isDebugEnabled) { + logger.debug( + "acquire distributed leaky rateLimiter" + + " elapsed time: $elapsedTime ms, acquireResult: $acquireResult" + ) + } + return acquireResult + } catch (e: Exception) { + logger.warn("${this.javaClass.simpleName} acquire error: ${e.message}") + throw AcquireLockFailedException("distributed lock acquire failed: $e") + } + } + + private fun getKeys(key: String): List { + return listOf(key, "$key.timestamp") + } + + override fun removeCacheLimit(key: String) { + getKeys(key).forEach { + redisTemplate.delete(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DistributedLeakyRateLimiter::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiter.kt new file mode 100644 index 0000000000..ef4d460988 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiter.kt @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.redis.LuaScript +import java.time.Duration +import kotlin.system.measureTimeMillis +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.data.redis.core.script.DefaultRedisScript + +/** + * 分布式滑动窗口算法实现 + */ +class DistributedSlidingWindowRateLimiter( + private val key: String, + private val limit: Long, + private val duration: Duration, + private val redisTemplate: RedisTemplate, +) : RateLimiter { + override fun tryAcquire(permits: Long): Boolean { + try { + var acquireResult = false + val elapsedTime = measureTimeMillis { + val redisScript = DefaultRedisScript(LuaScript.slidingWindowRateLimiterScript, List::class.java) + // 时间统一从redis server获取 + // lua脚本中使用命令获取时间指令需要配合replicate_commands()使用，但是由于redis只有在某个特定版本上才支持该指令， + // 所以无法从lua脚本中去获取时间，只能分为多次调用。 + val currentTime = redisTemplate.execute { connection -> + connection.time() + } ?: System.currentTimeMillis() + val currentSeconds = (currentTime / 1000) + val random = (currentTime % 1000) + // 注意， 由于redis expire只支持秒为单位，所以周期最小单位为秒 + val results = redisTemplate.execute( + redisScript, getKeys(key), limit.toString(), (duration.seconds).toString(), + permits.toString(), currentSeconds.toString(), random.toString() + ) + acquireResult = results[0] == 1L + } + if (logger.isDebugEnabled) { + logger.debug( + "acquire distributed sliding window rateLimiter" + + " elapsed time: $elapsedTime ms, acquireResult: $acquireResult" + ) + } + return acquireResult + } catch (e: Exception) { + logger.warn("${this.javaClass.simpleName} acquire error: ${e.message}") + throw AcquireLockFailedException("distributed lock acquire failed: $e") + } + } + + private fun getKeys(key: String): List { + return listOf(key) + } + + override fun removeCacheLimit(key: String) { + getKeys(key).forEach { + redisTemplate.delete(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DistributedSlidingWindowRateLimiter::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiter.kt new file mode 100644 index 0000000000..26dfe401ab --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiter.kt @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.redis.LuaScript +import kotlin.system.measureTimeMillis +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.data.redis.core.script.DefaultRedisScript + +/** + * 分布式令牌桶算法实现 + */ +class DistributedTokenBucketRateLimiter( + private val key: String, + private val permitsPerSecond: Double, + private val capacity: Long, + private val redisTemplate: RedisTemplate, +) : RateLimiter { + override fun tryAcquire(permits: Long): Boolean { + try { + var acquireResult: Boolean + val elapsedTime = measureTimeMillis { + val redisScript = DefaultRedisScript(LuaScript.tokenBucketRateLimiterScript, List::class.java) + // 时间统一从redis server获取 + // lua脚本中使用命令获取时间指令需要配合replicate_commands()使用，但是由于redis只有在某个特定版本上才支持该指令， + // 所以无法从lua脚本中去获取时间，只能分为多次调用。 + val currentTime = redisTemplate.execute { connection -> + connection.time() + } ?: System.currentTimeMillis() + val currentSeconds = (currentTime / 1000) + val results = redisTemplate.execute( + redisScript, getKeys(key), permitsPerSecond.toString(), + capacity.toString(), permits.toString(), currentSeconds.toString() + ) + acquireResult = results[0] == 1L + } + if (logger.isDebugEnabled) { + logger.debug( + "acquire distributed token bucket rateLimiter" + + " elapsed time: $elapsedTime ms, acquireResult: $acquireResult" + ) + } + return acquireResult + } catch (e: Exception) { + logger.warn("${this.javaClass.simpleName} acquire error: ${e.message}") + throw AcquireLockFailedException("distributed lock acquire failed: $e") + } + } + + private fun getKeys(key: String): List { + return listOf(key, "$key.timestamp") + } + + override fun removeCacheLimit(key: String) { + getKeys(key).forEach { + redisTemplate.delete(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DistributedTokenBucketRateLimiter::class.java) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiter.kt new file mode 100644 index 0000000000..af71989aea --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiter.kt @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.google.common.base.Stopwatch +import com.tencent.bkrepo.common.ratelimiter.constant.TRY_LOCK_TIMEOUT +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock + +/** + * 单机固定窗口算法实现 + */ +class FixedWindowRateLimiter( + private val limit: Long, + private val duration: Duration, + private val stopWatch: Stopwatch = Stopwatch.createStarted() +) : RateLimiter { + + private var currentValue: Long = 0 + private val lock: Lock = ReentrantLock() + + override fun tryAcquire(permits: Long): Boolean { + // TODO 当剩余容量少于permit时，会导致一直获取不到 + try { + if (!lock.tryLock(TRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS)) { + throw AcquireLockFailedException("fix window tryLock wait too long: $TRY_LOCK_TIMEOUT ms") + } + try { + if (stopWatch.elapsed(TimeUnit.MILLISECONDS) > duration.toMillis()) { + currentValue = 0 + stopWatch.reset() + } + if (currentValue + permits > limit) return false + currentValue += permits + return true + } finally { + lock.unlock() + } + } catch (e: InterruptedException) { + throw AcquireLockFailedException("fix window tryLock is interrupted by lock timeout: $e") + } + } + + override fun removeCacheLimit(key: String) { + // 非redis类实现不需要处理 + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiter.kt new file mode 100644 index 0000000000..902d2d1e30 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiter.kt @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.constant.TRY_LOCK_TIMEOUT +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock + + +/** + * 单机漏桶算法实现 + */ +class LeakyRateLimiter( + private val rate: Double, + private val capacity: Long, +) : RateLimiter { + + // 计算的起始时间 + private var lastLeakTime = System.currentTimeMillis() + private var water: Long = 0 + private val lock: Lock = ReentrantLock() + + + override fun tryAcquire(permits: Long): Boolean { + try { + if (!lock.tryLock(TRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS)) { + throw AcquireLockFailedException("leaky tryLock wait too long: $TRY_LOCK_TIMEOUT ms") + } + try { + return allow(permits) + } finally { + lock.unlock() + } + } catch (e: InterruptedException) { + throw AcquireLockFailedException("leaky tryLock is interrupted by lock timeout: $e") + } + } + + override fun removeCacheLimit(key: String) { + // 非redis类实现不需要处理 + } + + private fun allow(permits: Long): Boolean { + if (water == 0L) { + lastLeakTime = System.currentTimeMillis() + water += permits + return true + } + val waterLeaked: Double = ((System.currentTimeMillis() - lastLeakTime) / 1000) * rate + val waterLeft = (water - waterLeaked).toLong() + water = Math.max(0, waterLeft) // 漏水 + lastLeakTime = System.currentTimeMillis() + if (water + permits <= capacity) { + water += permits + return true + } + return false + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RateLimiter.kt new file mode 100644 index 0000000000..c8a654fa11 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RateLimiter.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +interface RateLimiter { + + fun tryAcquire(permits: Long): Boolean + + fun removeCacheLimit(key: String) + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiter.kt new file mode 100644 index 0000000000..ce17cc7f83 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiter.kt @@ -0,0 +1,114 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.constant.TRY_LOCK_TIMEOUT +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock + + +/** + * 单机滑动窗口算法实现 + * -- limit: 窗口时间单位内的阈值 + * -- interval: 窗口大小， + * -- limitUnit 窗口时间单位 + */ +class SlidingWindowRateLimiter( + private val limit: Long, + private val duration: Duration, +) : RateLimiter { + + private val lock: Lock = ReentrantLock() + + /** + * 子窗口个数 + */ + private val subWindowNum = 10 + + /** + * 窗口列表 + */ + private var windowArray = Array(subWindowNum) { WindowInfo() } + private val windowSize = duration.toMillis() * subWindowNum + + override fun tryAcquire(permits: Long): Boolean { + try { + if (!lock.tryLock(TRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS)) { + throw AcquireLockFailedException("sliding window tryLock wait too long: $TRY_LOCK_TIMEOUT ms") + } + try { + return allow(permits) + } finally { + lock.unlock() + } + } catch (e: InterruptedException) { + throw AcquireLockFailedException("sliding window tryLock is interrupted by lock timeout: $e") + } + } + + override fun removeCacheLimit(key: String) { + // 非redis类实现不需要处理 + } + + private fun allow(permits: Long): Boolean { + val currentTimeMillis = System.currentTimeMillis() + // 1. 计算当前时间窗口 + val currentIndex = (currentTimeMillis % windowSize / (windowSize / subWindowNum)).toInt() + // 2. 更新当前窗口计数 & 重置过期窗口计数 + var sum = 0L + val windowArrayCopy = windowArray.map { it.copy() }.toTypedArray() + for (i in windowArray.indices) { + val windowInfo = windowArray[i] + if (currentTimeMillis - windowInfo.time > windowSize) { + windowInfo.count = 0 + windowInfo.time = currentTimeMillis + } + if (currentIndex == i && windowInfo.count <= limit) { + windowInfo.count += permits + } + sum += windowInfo.count + } + // 3. 当前是否超过限制 + return if (sum <= limit) { + true + } else { + //如果最终sum>limit, windowInfo.count需要减掉permits + windowArray = windowArrayCopy + false + } + } + + data class WindowInfo( + var time: Long = System.currentTimeMillis(), + var count: Long = 0 + ) + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiter.kt new file mode 100644 index 0000000000..6c3d39156b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiter.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.constant.TRY_LOCK_TIMEOUT +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.util.concurrent.TimeUnit + +/** + * 单机令牌桶算法实现 + */ +class TokenBucketRateLimiter( + private val permitsPerSecond: Double, +) : RateLimiter { + + private val guavaRateLimiter = com.google.common.util.concurrent.RateLimiter.create(permitsPerSecond) + + override fun tryAcquire(permits: Long): Boolean { + try { + return guavaRateLimiter.tryAcquire(permits.toInt(), TRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS) + } catch (e: Exception) { + throw AcquireLockFailedException("lock acquire failed: $e") + } + } + + override fun removeCacheLimit(key: String) { + // 非redis类实现不需要处理 + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/config/RateLimiterProperties.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/config/RateLimiterProperties.kt new file mode 100644 index 0000000000..1136d45645 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/config/RateLimiterProperties.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.config + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "rate.limiter") +data class RateLimiterProperties( + var enabled: Boolean = false, + var dryRun: Boolean = false, + // 配置规则刷新频率 单位为秒 + var refreshDuration: Long = 10L, + // 本地缓存限流算法实现的最大个数 + var cacheCapacity: Long = 1024L, + // 限流配置 + var rules: List = mutableListOf(), + // 等待时间，单位毫秒 + var latency: Long = 70, + // 重试次数 + var waitRound: Int = 5, + // 针对读流的请求，避免频繁去请求，每次申请固定大小 + var permitsOnce: Long = 1024 * 1024, + // 只对指定url进行从request body解析项目仓库信息 + var specialUrls: List = emptyList() +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/constant/Constants.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/constant/Constants.kt new file mode 100644 index 0000000000..6de61b22ca --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/constant/Constants.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.constant + +const val TRY_LOCK_TIMEOUT = 10L +const val KEY_PREFIX = "rateLimiter:" +const val SLEEP_TIME = 10 + +const val TAG_STATUS = "status" +const val TAG_NAME = "name" + +const val RATE_LIMITER_TOTAL_COUNT = "rate.limiter.total.count" +const val RATE_LIMITER_TOTAL_COUNT_DESC = "总请求数" + +const val RATE_LIMITER_PASSED_COUNT = "rate.limiter.passed.count" +const val RATE_LIMITER_PASSED_COUNT_DESC = "通过请求数" + +const val RATE_LIMITER_LIMITED_COUNT = "rate.limiter.limited.count" +const val RATE_LIMITER_LIMITED_COUNT_DESC = "限流请求数" + +const val RATE_LIMITER_EXCEPTION_COUNT = "rate.limiter.exception.count" +const val RATE_LIMITER_EXCEPTION_COUNT_DESC = "异常请求数" diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/Algorithms.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/Algorithms.kt new file mode 100644 index 0000000000..6d4447f9bb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/Algorithms.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.enums + +/** + * 限流算法类型 + */ +enum class Algorithms { + FIXED_WINDOW, + SLIDING_WINDOW, + LEAKY_BUCKET, + TOKEN_BUCKET; +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/LimitDimension.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/LimitDimension.kt new file mode 100644 index 0000000000..8ffd9890de --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/LimitDimension.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.enums + +/** + * 限流维度： + */ +enum class LimitDimension { + URL, // 针对指定URL限流 + URL_REPO, // 针对访问指定项目/仓库的url进行限流 + UPLOAD_USAGE, // 针对仓库上传总大小进行限流 + DOWNLOAD_USAGE, // 针对仓库下载总大小进行限流 + USER_URL, // 针对指定用户指定请求进行限流 + USER_URL_REPO, // 针对指定用户访问指定项目/仓库的url进行限流 + USER_UPLOAD_USAGE, // 针对指定用户上传总大小进行限流 + USER_DOWNLOAD_USAGE, // 针对指定用户下载总大小进行限流 + UPLOAD_BANDWIDTH, // 针对项目维度上传带宽进行限流 + DOWNLOAD_BANDWIDTH, // 针对项目维度下载带宽进行限流 +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/WorkScope.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/WorkScope.kt new file mode 100644 index 0000000000..53bce5d00a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/WorkScope.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.enums + +/** + * 配置生效范围， 全局生效或者本地生效 + */ +enum class WorkScope { + LOCAL, + GLOBAL +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/AcquireLockFailedException.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/AcquireLockFailedException.kt new file mode 100644 index 0000000000..0490d90567 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/AcquireLockFailedException.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.exception + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode + +/** + * 获取对应执行计划失败 + */ +data class AcquireLockFailedException( + val reason: String +) : ErrorCodeException(CommonMessageCode.ACQUIRE_LOCK_FAILED, reason) + diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/InvalidResourceException.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/InvalidResourceException.kt new file mode 100644 index 0000000000..0255af6676 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/InvalidResourceException.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.exception + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode + +/** + * 资源无效异常 + */ +data class InvalidResourceException( + val reason: String +) : ErrorCodeException(CommonMessageCode.INVALID_CONFIG, reason) + diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/MonitorRateLimiterInterceptorAdaptor.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/MonitorRateLimiterInterceptorAdaptor.kt new file mode 100644 index 0000000000..f7681dd10e --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/MonitorRateLimiterInterceptorAdaptor.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流相关指标采集拦截器 + */ +class MonitorRateLimiterInterceptorAdaptor( + private val rateLimiterMetrics: RateLimiterMetrics +) : RateLimiterInterceptorAdapter() { + + override fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) = Unit + + override fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) { + if (resourceLimit == null) return + rateLimiterMetrics.collectMetrics(resource = resource, result = result, e = e) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimitHandlerInterceptor.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimitHandlerInterceptor.kt new file mode 100644 index 0000000000..c3bac2ce38 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimitHandlerInterceptor.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import org.springframework.web.servlet.HandlerInterceptor +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +/** + * 针对http请求添加限流拦截 + */ +class RateLimitHandlerInterceptor( + private val requestLimitCheckService: RequestLimitCheckService +) : HandlerInterceptor { + override fun preHandle(request: HttpServletRequest, response: HttpServletResponse, handler: Any): Boolean { + requestLimitCheckService.preLimitCheck(request) + return super.preHandle(request, response, handler) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptor.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptor.kt new file mode 100644 index 0000000000..a2e532231e --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptor.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流执行拦截器 + */ +interface RateLimiterInterceptor { + + /** + * 限流判断前处理 + */ + fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) + + /** + * 限流判断后处理 + */ + fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorAdapter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorAdapter.kt new file mode 100644 index 0000000000..a9f61df5df --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorAdapter.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流算法执行前后校验拦截器 + */ +abstract class RateLimiterInterceptorAdapter : RateLimiterInterceptor { + override fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) = Unit + + override fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) = Unit +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChain.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChain.kt new file mode 100644 index 0000000000..5165ce3912 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChain.kt @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流拦截器链 + */ +class RateLimiterInterceptorChain( + private val interceptors: MutableList = mutableListOf() +) { + + fun doBeforeLimitCheck(resource: String, resourceLimit: ResourceLimit) { + this.interceptors.forEach { + it.beforeLimitCheck(resource, resourceLimit) + } + } + + fun doAfterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) { + this.interceptors.forEach { + it.afterLimitCheck(resource, resourceLimit, result, e) + } + } + + fun addInterceptor(interceptor: RateLimiterInterceptor) { + this.interceptors.add(interceptor) + } + + fun addInterceptors(interceptors: Collection) { + this.interceptors.addAll(interceptors) + } + + fun clear() { + this.interceptors.clear() + } + + fun isEmpty(): Boolean { + return this.interceptors.isEmpty() + } + + fun size(): Int { + return this.interceptors.size + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/TargetRateLimiterInterceptorAdaptor.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/TargetRateLimiterInterceptorAdaptor.kt new file mode 100644 index 0000000000..8780b6b7c8 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/TargetRateLimiterInterceptorAdaptor.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService + +/** + * 执行限流机器判断 + */ +class TargetRateLimiterInterceptorAdaptor( + private val rateLimiterConfigService: RateLimiterConfigService, +) : RateLimiterInterceptorAdapter() { + + override fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) { + if (resourceLimit.targets.isNotEmpty() && !resourceLimit.targets.contains(rateLimiterConfigService.host)) { + throw InvalidResourceException("targets not contain ${rateLimiterConfigService.host}") + } + } + + override fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) = Unit +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/MetricType.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/MetricType.kt new file mode 100644 index 0000000000..7f0513980b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/MetricType.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.metrics + +/** + * 指标类型 + */ +enum class MetricType { + TOTAL, // 总请求数量 + PASSED, // 通过数量 + LIMITED, //限流数量 + EXCEPTION, //异常请求数量 +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/RateLimiterMetrics.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/RateLimiterMetrics.kt new file mode 100644 index 0000000000..05a56b27e2 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/RateLimiterMetrics.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.metrics + +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_EXCEPTION_COUNT +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_EXCEPTION_COUNT_DESC +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_LIMITED_COUNT +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_LIMITED_COUNT_DESC +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_PASSED_COUNT +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_PASSED_COUNT_DESC +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_TOTAL_COUNT +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_TOTAL_COUNT_DESC +import com.tencent.bkrepo.common.ratelimiter.constant.TAG_NAME +import com.tencent.bkrepo.common.ratelimiter.constant.TAG_STATUS +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.MeterRegistry + +/** + * 限流指标写入 + */ +class RateLimiterMetrics(private val registry: MeterRegistry) { + + fun collectMetrics( + resource: String, result: Boolean, e: Exception? + ) { + try { + getTotalCounter(resource).increment() + if (result) { + getPassedCounter(resource).increment() + } else { + getLimitedCounter(resource).increment() + } + if (e != null) { + getExceptionCounter(resource).increment() + } + } catch (ignore: Exception) { + } + + } + + private fun getTotalCounter(resource: String): Counter { + return getMetricsCount( + RATE_LIMITER_TOTAL_COUNT, RATE_LIMITER_TOTAL_COUNT_DESC, MetricType.TOTAL.name, resource + ) + } + + private fun getPassedCounter(resource: String): Counter { + return getMetricsCount( + RATE_LIMITER_PASSED_COUNT, RATE_LIMITER_PASSED_COUNT_DESC, MetricType.PASSED.name, resource + ) + } + + private fun getLimitedCounter(resource: String): Counter { + return getMetricsCount( + RATE_LIMITER_LIMITED_COUNT, RATE_LIMITER_LIMITED_COUNT_DESC, MetricType.LIMITED.name, resource + ) + } + + private fun getExceptionCounter(resource: String): Counter { + return getMetricsCount( + RATE_LIMITER_EXCEPTION_COUNT, RATE_LIMITER_EXCEPTION_COUNT_DESC, MetricType.EXCEPTION.name, resource + ) + } + + private fun getMetricsCount(metricsName: String, metricsDes: String, status: String, resource: String): Counter { + return Counter.builder(metricsName) + .description(metricsDes) + .tag(TAG_STATUS, status) + .tag(TAG_NAME, resource) + .register(registry) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/RateLimitCreatOrUpdateRequest.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/RateLimitCreatOrUpdateRequest.kt new file mode 100644 index 0000000000..4d2b2ddd99 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/RateLimitCreatOrUpdateRequest.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.model + +data class RateLimitCreatOrUpdateRequest( + val id: String? = null, + // 算法选择 + var algo: String, + // 资源标识 + var resource: String, + // 限流维度 + var limitDimension: String, + // 限流值 + var limit: Long, + // 限流周期 + var duration: Long, + // 桶容量(令牌桶和漏桶使用) + var capacity: Long? = null, + // 生效范围 + var scope: String, + // 指定机器上运行 + var targets: List? = emptyList(), + // 模块名 + var moduleName: List +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/TRateLimit.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/TRateLimit.kt new file mode 100644 index 0000000000..8b00c843df --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/TRateLimit.kt @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.model + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document +import java.time.Duration + +@Document(collection = "rate_limit") +@CompoundIndexes( + CompoundIndex( + name = "resource_limitDimension_idx", + def = "{'resource': 1,'limitDimension': 1}", + background = true + ), + CompoundIndex( + name = "limitDimension_idx", + def = "{'limitDimension': 1}", + background = true + ) +) +data class TRateLimit( + var id: String?, + // 算法选择 + var algo: String = Algorithms.FIXED_WINDOW.name, + // 资源标识 + var resource: String = "/", + // 限流维度 + var limitDimension: String = LimitDimension.URL.name, + // 限流值 + var limit: Long = -1, + // 限流周期 + var duration: Duration = Duration.ofSeconds(1), + // 桶容量(令牌桶和漏桶使用) + var capacity: Long? = null, + // 生效范围 + var scope: String = WorkScope.LOCAL.name, + // 指定机器上运行 + var targets: List = emptyList(), + // 模块名 + var moduleName: List +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/redis/LuaScript.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/redis/LuaScript.kt new file mode 100644 index 0000000000..9d5e45975c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/redis/LuaScript.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.redis + +import org.slf4j.LoggerFactory +import org.springframework.util.StreamUtils +import java.io.IOException +import java.nio.charset.StandardCharsets + +/** + * lua脚本加载 + */ +object LuaScript { + private val logger = LoggerFactory.getLogger(LuaScript::class.java) + private const val FIX_WINDOW_RATE_LIMITER_FILE_PATH = "fix-window-rate-limiter.lua" + private const val TOKEN_BUCKET_RATE_LIMITER_FILE_PATH = "token-bucket-rate-limiter.lua" + private const val SLIDING_WINDOW_RATE_LIMITER_FILE_PATH = "sliding-window-rate-limiter.lua" + private const val LEAKY_RATE_LIMITER_FILE_PATH = "leaky-rate-limiter.lua" + + lateinit var fixWindowRateLimiterScript: String + lateinit var tokenBucketRateLimiterScript: String + lateinit var slidingWindowRateLimiterScript: String + lateinit var leakyRateLimiterScript: String + + init { + val fixWindowInput = Thread.currentThread().contextClassLoader + .getResourceAsStream(FIX_WINDOW_RATE_LIMITER_FILE_PATH) + val tokenBucketInput = Thread.currentThread().contextClassLoader + .getResourceAsStream(TOKEN_BUCKET_RATE_LIMITER_FILE_PATH) + val slidingWindowInput = Thread.currentThread().contextClassLoader + .getResourceAsStream(SLIDING_WINDOW_RATE_LIMITER_FILE_PATH) + val leakyInput = Thread.currentThread().contextClassLoader + .getResourceAsStream(LEAKY_RATE_LIMITER_FILE_PATH) + try { + fixWindowRateLimiterScript = StreamUtils.copyToString(fixWindowInput, StandardCharsets.UTF_8) + tokenBucketRateLimiterScript = StreamUtils.copyToString(tokenBucketInput, StandardCharsets.UTF_8) + slidingWindowRateLimiterScript = StreamUtils.copyToString(slidingWindowInput, StandardCharsets.UTF_8) + leakyRateLimiterScript = StreamUtils.copyToString(leakyInput, StandardCharsets.UTF_8) + } catch (e: IOException) { + logger.error("lua script Initialization failed, $e") + } + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/repository/RateLimitRepository.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/repository/RateLimitRepository.kt new file mode 100644 index 0000000000..25736c10a9 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/repository/RateLimitRepository.kt @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.repository + +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.common.ratelimiter.model.TRateLimit +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Repository + +@Repository +class RateLimitRepository : SimpleMongoDao() { + + fun existsById(id: String) : Boolean { + return find(Query(TRateLimit::id.isEqualTo(id))).isNotEmpty() + } + + fun existsByResourceAndLimitDimension(resource: String, limitDimension: String) : Boolean { + return exists( + Query( + Criteria.where(TRateLimit::resource.name).isEqualTo(resource) + .and(TRateLimit::limitDimension.name).isEqualTo(limitDimension) + ) + ) + } + + fun findByResourceAndLimitDimension(resource: String, limitDimension: String) : List { + return find( + Query( + Criteria.where(TRateLimit::resource.name).isEqualTo(resource) + .and(TRateLimit::limitDimension.name).isEqualTo(limitDimension) + ) + ) + } + + fun findByModuleNameAndLimitDimension(moduleName: String, limitDimension: String): List { + return find( + Query( + Criteria.where(TRateLimit::moduleName.name).regex("$moduleName") + .and(TRateLimit::limitDimension.name).isEqualTo(limitDimension) + ) + ) + } + + fun findByModuleNameAndLimitDimensionAndResource( + resource: String, + moduleName: List, + limitDimension: String + ): TRateLimit? { + return findOne( + Query( + Criteria.where(TRateLimit::moduleName.name).isEqualTo("$moduleName") + .and(TRateLimit::limitDimension.name).isEqualTo(limitDimension) + .and(TRateLimit::resource.name).isEqualTo(resource) + ) + ) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/PathResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/PathResourceLimitRule.kt new file mode 100644 index 0000000000..4e7fc5add4 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/PathResourceLimitRule.kt @@ -0,0 +1,216 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule + +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap +import java.util.regex.Pattern + +/** + * path资源规则类 + * pathLengthCheck : 当为false时，不校验resource根据/分割后的字符个数与查出的队友匹配规则的resource按照/分割后的字符个数是否相等 + * 如果是url 类型的路径， 如果配置resource为/project1/, 则其子目录都遵循该规则, 此时pathLengthCheck设置为false + * 如果是project/repo类型的路径，如何配置resource为/project1/,则只有当查询资源为/project1/才能够匹配。 此时pathLengthCheck应该设置为 true + */ +open class PathResourceLimitRule( + private val root: PathNode = PathNode("/"), + private val pathLengthCheck: Boolean = false +) { + + fun isEmpty(): Boolean { + return root.getEdges().isEmpty() && root.getResourceLimit() == null + } + + /** + * 添加资源对应的规则 + */ + fun addPathResourceLimit(resourceLimit: ResourceLimit, limits: List) { + if (!limits.contains(resourceLimit.limitDimension)) { + return + } + val resourcePath = resourceLimit.resource + if (!resourcePath.startsWith("/")) { + throw InvalidResourceException(resourcePath) + } + // TODO 配置/*/blueking/* 能识别/api/node/blueking/generic-local + addPathNode(resourcePath, resourceLimit) + } + + fun addPathResourceLimits(resourceLimits: List, limits: List) { + resourceLimits.forEach { + addPathResourceLimit(it, limits) + } + } + + /** + * 根据资源获取对应规则 + */ + open fun getPathResourceLimit(resource: String): ResourceLimit? { + if (resource.isBlank()) { + return null + } + if (resource == "/") { + return root.getResourceLimit() + } + val pathDirs = ResourcePathUtils.tokenizeResourcePath(resource) + if (pathDirs.isEmpty()) { + logger.warn("config resource path $resource is empty!") + return null + } + return findResourceLimit(pathDirs) + } + + private fun findResourceLimit(pathDirs: List): ResourceLimit? { + var p = root + var currentLimit: ResourceLimit? = null + if (p.getResourceLimit() != null) { + currentLimit = p.getResourceLimit() + } + for (path in pathDirs) { + val children = p.getEdges() + var matchedNode = children[path] + if (matchedNode == null) { + val child = findInChildren(children, path) + if (child != null) { + matchedNode = child + } + } + if (matchedNode == null) { + break + } + p = matchedNode + if (matchedNode.getResourceLimit() != null) { + currentLimit = matchedNode.getResourceLimit() + } + } + if (pathLengthCheck) { + return if (pathLengthCheck(currentLimit, pathDirs.size)) { + currentLimit + } else { + null + } + } + return currentLimit + } + + private fun pathLengthCheck(currentLimit: ResourceLimit?, pathDirSize: Int): Boolean { + val length = if (currentLimit?.resource.isNullOrEmpty()) { + 0 + } else { + ResourcePathUtils.tokenizeResourcePath(currentLimit!!.resource).size + } + if (length == pathDirSize) { + return true + } + return false + } + + /** + * 将资源路径按照/拆分，存对应每级对应规则 + */ + private fun addPathNode( + resourcePath: String, + resourceLimit: ResourceLimit + ) { + if (resourcePath == "/") { + root.setResourceLimit(resourceLimit) + return + } + + val pathDirs = ResourcePathUtils.tokenizeResourcePath(resourcePath) + if (pathDirs.isEmpty()) { + logger.warn("config resource path $resourcePath is empty!") + return + } + + var p = root + pathDirs.forEach { + val children = p.getEdges() + var pathDirPattern = it + var isPattern = false + if (isTemplateVariable(it)) { + pathDirPattern = getPathDirPatten(it) + isPattern = true + } + val newNode = PathNode(pathDirPattern, isPattern) + p = children.putIfAbsent(pathDirPattern, newNode) ?: newNode + } + p.setResourceLimit(resourceLimit) + logger.debug("$resourcePath set limit info $resourceLimit") + } + + private fun findInChildren( + children: ConcurrentHashMap, + path: String, + ): PathNode? { + children.entries.forEach { entry -> + val n = entry.value + if (n.isPattern) { + if (Pattern.matches(n.pathDir, path)) { + return n + } + } + } + return null + } + + /** + * 判断是否是模板 + */ + private fun isTemplateVariable(pathDir: String): Boolean { + return pathDir.startsWith("{") && pathDir.endsWith("}") || + pathDir == "*" || pathDir == "**" || pathDir.contains('*') + } + + /** + * 如果模板自带正则表达式，则格式必须为{(^[a-zA-Z]*$)}, ()内味对应正则表达式 + */ + private fun getPathDirPatten(pathDir: String): String { + val patternBuilder = StringBuilder() + val isRegex = pathDir.contains("{(") && pathDir.contains(")}") + if (isRegex) { + val variablePattern = pathDir.substring(2, pathDir.length - 2) + patternBuilder.append('(') + patternBuilder.append(variablePattern) + patternBuilder.append(')') + } else { + patternBuilder.append(PATH_REGEX) + } + return patternBuilder.toString() + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(PathResourceLimitRule::class.java) + private const val PATH_REGEX = ".*" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/RateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/RateLimitRule.kt new file mode 100644 index 0000000000..e570801c3c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/RateLimitRule.kt @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流配置规则处理 + */ +interface RateLimitRule { + + /** + * 是否存在相应配置规则 + */ + fun isEmpty(): Boolean + + /** + * 获取资源对应的规则 + * 优先查找resource， 如查不到对应规则，则通过extraResource查找 + * resource一般是特定类型，如特定用户，特定URL，特定项目仓库等 + * extraResource一般是某一类类型，如所有用户、URL模版、所有仓库等 + */ + fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? + + /** + * 添加限流规则 + */ + fun addRateLimitRule(resourceLimit: ResourceLimit) + + /** + * 批量添加限流规则 + */ + fun addRateLimitRules(resourceLimit: List) + + /** + * 过滤不符合条件的规则 + */ + fun filterResourceLimit(resourceLimit: ResourceLimit) +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/BandwidthResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/BandwidthResourceLimitRule.kt new file mode 100644 index 0000000000..6094dc6f9d --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/BandwidthResourceLimitRule.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 带宽资源规则类 + * 如果配置了仓库级别限流配置，则仓库下所有请求遵循这个配置，即同一仓库下所有请求共享这个带宽限额； + * 如果配置了项目级别限流配置，则项目下所有请求遵循这个配置，即同一项目下所有请求共享这个带宽限额。 + */ +class BandwidthResourceLimitRule( + root: PathNode = PathNode("/"), + pathLengthCheck: Boolean = true +) : PathResourceLimitRule(root, pathLengthCheck) { + + /** + * 添加资源对应对应的规则 + */ + fun addUrlResourceLimit(resourceLimit: ResourceLimit) { + addPathResourceLimit(resourceLimit, bandwidthDimensionList) + } + + fun addUrlResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUrlResourceLimit(it) + } + } + + companion object { + private val bandwidthDimensionList = listOf( + LimitDimension.UPLOAD_BANDWIDTH.name, LimitDimension.DOWNLOAD_BANDWIDTH.name + ) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRule.kt new file mode 100644 index 0000000000..fa5428338d --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRule.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 下载带宽限流配置规则实现 + */ +class DownloadBandwidthRateLimitRule( + bandwidthLimitRules: BandwidthResourceLimitRule = BandwidthResourceLimitRule() +) : UploadBandwidthRateLimitRule(bandwidthLimitRules) { + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.DOWNLOAD_BANDWIDTH.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DownloadBandwidthRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRule.kt new file mode 100644 index 0000000000..8e7b35ca7f --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRule.kt @@ -0,0 +1,96 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 上传带宽限流配置规则实现 + */ +open class UploadBandwidthRateLimitRule( + private val bandwidthLimitRules: BandwidthResourceLimitRule = BandwidthResourceLimitRule() +) : RateLimitRule { + + override fun isEmpty(): Boolean { + return bandwidthLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var realResource = resInfo.resource + if (realResource.isBlank()) { + return null + } + var ruleLimit = bandwidthLimitRules.getPathResourceLimit(realResource) + if (ruleLimit == null && resInfo.extraResource.isNotEmpty()) { + for (res in resInfo.extraResource) { + ruleLimit = bandwidthLimitRules.getPathResourceLimit(res) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + return ResLimitInfo(realResource, ruleLimit) + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + bandwidthLimitRules.addUrlResourceLimit(resourceLimit) + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.UPLOAD_BANDWIDTH.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UploadBandwidthRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/PathNode.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/PathNode.kt new file mode 100644 index 0000000000..6a05e7c575 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/PathNode.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.common + +import java.util.concurrent.ConcurrentHashMap + +/** + * path 对应配置 + */ +class PathNode( + val pathDir: String, + val isPattern: Boolean = false, +) { + private val edges: ConcurrentHashMap = ConcurrentHashMap() + private var resourceLimit: ResourceLimit? = null + + fun getEdges(): ConcurrentHashMap { + return edges + } + + fun setResourceLimit(resourceLimit: ResourceLimit) { + this.resourceLimit = resourceLimit + } + + fun getResourceLimit(): ResourceLimit? { + return this.resourceLimit + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResInfo.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResInfo.kt new file mode 100644 index 0000000000..3e3a1014bb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResInfo.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.common + +/** + * 资源信息 + */ +data class ResInfo( + val resource: String, + val extraResource: List = emptyList() +) + diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResLimitInfo.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResLimitInfo.kt new file mode 100644 index 0000000000..f7dc5d33ce --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResLimitInfo.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.common + +/** + * 请求资源以及对应的限流配置 + */ +data class ResLimitInfo( + // 请求资源 + val resource: String, + // 请求资源对应的限流配置 + val resourceLimit: ResourceLimit +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResourceLimit.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResourceLimit.kt new file mode 100644 index 0000000000..68ebc0526e --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResourceLimit.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.common + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import java.time.Duration + +/** + * 限流规则 + */ +data class ResourceLimit( + // 算法选择 + var algo: String = Algorithms.FIXED_WINDOW.name, + // 资源标识 + var resource: String = "/", + // 限流维度 + var limitDimension: String = LimitDimension.URL.name, + // 限流值 + var limit: Long = -1, + // 限流周期 + var duration: Duration = Duration.ofSeconds(1), + // 桶容量(令牌桶和漏桶使用) + var capacity: Long? = null, + // 生效范围 + var scope: String = WorkScope.LOCAL.name, + // 指定机器上运行 + var targets: List = emptyList() +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRule.kt new file mode 100644 index 0000000000..c5eef29ac6 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRule.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * URL限流配置规则实现 + */ +class UrlRateLimitRule : RateLimitRule { + + // 指定URL规则 + private val urlLimitRules: UrlResourceLimitRule = UrlResourceLimitRule() + + override fun isEmpty(): Boolean { + return urlLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var realResource = resInfo.resource + if (realResource.isBlank()) { + return null + } + var ruleLimit = urlLimitRules.getPathResourceLimit(realResource) + if (ruleLimit == null && resInfo.extraResource.isNotEmpty()) { + for (res in resInfo.extraResource) { + ruleLimit = urlLimitRules.getPathResourceLimit(res) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + return ResLimitInfo(realResource, ruleLimit) + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + urlLimitRules.addUrlResourceLimit(resourceLimit) + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.URL.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UrlRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRule.kt new file mode 100644 index 0000000000..5a698ded10 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRule.kt @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 基于项目/仓库的URL限流配置规则实现 + */ +class UrlRepoRateLimitRule : RateLimitRule { + + val urlRepoLimitRules: PathResourceLimitRule = PathResourceLimitRule(pathLengthCheck = true) + + override fun isEmpty(): Boolean { + return urlRepoLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var realResource = resInfo.resource + if (realResource.isBlank()) { + return null + } + var ruleLimit = urlRepoLimitRules.getPathResourceLimit(realResource) + if (ruleLimit == null && resInfo.extraResource.isNotEmpty()) { + for (res in resInfo.extraResource) { + ruleLimit = urlRepoLimitRules.getPathResourceLimit(res) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + return ResLimitInfo(realResource, ruleLimit) + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + urlRepoLimitRules.addPathResourceLimit(resourceLimit, urlRepoDimensionList) + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.URL_REPO.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UrlRepoRateLimitRule::class.java) + private val urlRepoDimensionList = listOf(LimitDimension.URL_REPO.name) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlResourceLimitRule.kt new file mode 100644 index 0000000000..b921a4c24c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlResourceLimitRule.kt @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * url资源规则类: + * 如果配置了父路径，则子路径也会遵循父路径对应的配置规则，即每个子路径单独拥有对应父路径所配置的限流规则； + * 如果配置指定路径，则只有该指定路径遵循对应的配置规则，即该指定路径独享所配置的限流规则。 + */ +class UrlResourceLimitRule( + root: PathNode = PathNode("/") +) : PathResourceLimitRule(root) { + + /** + * 添加URL对应的规则 + */ + fun addUrlResourceLimit(resourceLimit: ResourceLimit) { + addPathResourceLimit(resourceLimit, urlDimensionList) + } + + fun addUrlResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUrlResourceLimit(it) + } + } + + companion object { + private val urlDimensionList = listOf(LimitDimension.URL.name) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRule.kt new file mode 100644 index 0000000000..9a00605c8a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRule.kt @@ -0,0 +1,128 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.buildUserPath +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.getUserAndPath +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +/** + * 用户级别的URL限流配置规则实现 + */ +class UserUrlRateLimitRule : RateLimitRule { + + // 用户+指定url对应规则(限制每个用户对应URL的请求次数) + private val userUrlLimitRules: ConcurrentHashMap = ConcurrentHashMap() + + // 用户对应规则(限制每个用户的总请求次数) + private val userLimitRules: ConcurrentHashMap = ConcurrentHashMap() + + override fun isEmpty(): Boolean { + return userUrlLimitRules.isEmpty() && userLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var resLimitInfo = findConfigRule(resInfo.resource) + if (resLimitInfo == null) { + resLimitInfo = findConfigRule(resInfo.resource, true) + } + if (resLimitInfo == null && resInfo.extraResource.isNotEmpty()) { + val res = resInfo.extraResource.first() + val (user, _) = getUserAndPath(res) + val userRule = userLimitRules[user] + if (userRule != null) { + resLimitInfo = ResLimitInfo(res, userRule) + } + if (resLimitInfo == null && userLimitRules.containsKey(StringPool.POUND)) { + resLimitInfo = ResLimitInfo(res, userLimitRules[StringPool.POUND]!!) + } + } + return resLimitInfo + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + val (userId, path) = getUserAndPath(resourceLimit.resource) + if (path.isEmpty()) { + userLimitRules[userId] = resourceLimit + } else { + val userUrlResourceLimitRule = userUrlLimitRules.getOrDefault(userId, UserUrlResourceLimitRule()) + userUrlResourceLimitRule.addUserUrlResourceLimit(resourceLimit) + userUrlLimitRules.putIfAbsent(userId, userUrlResourceLimitRule) + } + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.USER_URL.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + /** + * 根据资源获取配置 + */ + private fun findConfigRule(resource: String, userPattern: Boolean = false): ResLimitInfo? { + if (resource.isBlank()) { + return null + } + var (user, resWithoutUser) = getUserAndPath(resource) + if (userPattern) { + user = StringPool.POUND + } + val userUrlRule = userUrlLimitRules[user] + val ruleLimit = userUrlRule?.getPathResourceLimit(resWithoutUser) ?: return null + val resourceLimitCopy = ruleLimit.copy(resource = buildUserPath(user, ruleLimit.resource)) + return ResLimitInfo(resource, resourceLimitCopy) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUrlRateLimitRule::class.java) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRule.kt new file mode 100644 index 0000000000..6c2bf11cc9 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRule.kt @@ -0,0 +1,138 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +class UserUrlRepoRateLimitRule( + // 用户+urlRepo对应规则 + private val userUrlRepoLimitRules: ConcurrentHashMap = ConcurrentHashMap(), + // 用户对应规则 + private val userLimitRules: ConcurrentHashMap = ConcurrentHashMap() +) : RateLimitRule { + + override fun isEmpty(): Boolean { + return userUrlRepoLimitRules.isEmpty() && userLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var resLimitInfo = findConfigRule(resInfo.resource, resInfo.extraResource) + if (resLimitInfo == null) { + resLimitInfo = findConfigRule(resInfo.resource, resInfo.extraResource, true) + } + if (resLimitInfo == null && resInfo.extraResource.isNotEmpty()) { + val res = resInfo.extraResource.last() + val (user, _) = ResourcePathUtils.getUserAndPath(res) + val userRule = userLimitRules[user] + if (userRule != null) { + resLimitInfo = ResLimitInfo(res, userRule) + } + if (resLimitInfo == null && userLimitRules.containsKey(StringPool.POUND)) { + resLimitInfo = ResLimitInfo(res, userLimitRules[StringPool.POUND]!!) + } + } + return resLimitInfo + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + val (userId, path) = ResourcePathUtils.getUserAndPath(resourceLimit.resource) + if (path.isEmpty()) { + userLimitRules[userId] = resourceLimit + } else { + val userUrlRepoResourceLimitRule = + userUrlRepoLimitRules.getOrDefault(userId, UserUrlRepoResourceLimitRule()) + userUrlRepoResourceLimitRule.addUserUrlRepoResourceLimit(resourceLimit) + userUrlRepoLimitRules.putIfAbsent(userId, userUrlRepoResourceLimitRule) + } + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.USER_URL_REPO.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + /** + * 根据资源获取配置 + */ + private fun findConfigRule( + resource: String, extraResource: List, userPattern: Boolean = false + ): ResLimitInfo? { + var realResource = resource + if (realResource.isBlank()) { + return null + } + var (user, resWithoutUser) = ResourcePathUtils.getUserAndPath(realResource) + if (userPattern) { + user = StringPool.POUND + } + val userUsageRule = userUrlRepoLimitRules[user] + var ruleLimit = userUsageRule?.getPathResourceLimit(resWithoutUser) + if (ruleLimit == null && extraResource.isNotEmpty()) { + for (res in extraResource) { + var (_, resWithoutUser) = ResourcePathUtils.getUserAndPath(res) + ruleLimit = userUsageRule?.getPathResourceLimit(resWithoutUser) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + val resourceLimitCopy = ruleLimit.copy(resource = ResourcePathUtils.buildUserPath(user, ruleLimit.resource)) + return ResLimitInfo(realResource, resourceLimitCopy) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUrlRepoRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoResourceLimitRule.kt new file mode 100644 index 0000000000..4f2f04937b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoResourceLimitRule.kt @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.getUserAndPath +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class UserUrlRepoResourceLimitRule( + root: PathNode = PathNode("/"), + pathLengthCheck: Boolean = true, + private var user: String = StringPool.POUND, +) : PathResourceLimitRule(root, pathLengthCheck) { + + fun addUserUrlRepoResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension !in userUrlRepoDimensionList) { + return + } + val (userId, urlPath) = getUserAndPath(resourceLimit.resource) + if (!urlPath.startsWith("/") || user.isBlank()) { + logger.warn("$resourceLimit is invalid") + throw InvalidResourceException(urlPath) + } + user = userId + val resourceLimitCopy = resourceLimit.copy(resource = urlPath) + addPathResourceLimit(resourceLimitCopy, userUrlRepoDimensionList) + } + + fun addUserUrlRepoResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUserUrlRepoResourceLimit(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUrlRepoResourceLimitRule::class.java) + private val userUrlRepoDimensionList = listOf( + LimitDimension.USER_URL_REPO.name + ) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlResourceLimitRule.kt new file mode 100644 index 0000000000..892d837d1c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlResourceLimitRule.kt @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.getUserAndPath +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 用户+url对应规则 + * 如果配置了父路径，则该用户对应访问的子路径也会遵循父路径对应的配置规则，即每个子路径单独拥有对应父路径所配置的限流规则； + * 如果配置指定路径，则只有该用户访问的指定路径遵循对应的配置规则，即该指定路径独享所配置的限流规则。 + */ +class UserUrlResourceLimitRule( + root: PathNode = PathNode("/"), + private var user: String = StringPool.POUND +) : PathResourceLimitRule(root) { + + fun addUserUrlResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension !in userLimitDimensionList) { + return + } + val (userId, urlPath) = getUserAndPath(resourceLimit.resource) + if (!urlPath.startsWith("/") || user.isBlank()) { + logger.warn("$resourceLimit is invalid") + throw InvalidResourceException(urlPath) + } + user = userId + val resourceLimitCopy = resourceLimit.copy(resource = urlPath) + addPathResourceLimit(resourceLimitCopy, userLimitDimensionList) + } + + fun addUserUrlResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUserUrlResourceLimit(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUrlResourceLimitRule::class.java) + private val userLimitDimensionList = listOf(LimitDimension.USER_URL.name) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRule.kt new file mode 100644 index 0000000000..c79f93af95 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRule.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 下载用量限流配置规则实现 + * 如果配置了仓库级别限流配置，则该对应仓库下所有请求遵循这个配置，即同一仓库下所有请求共享这个容量限额； + * 如果配置了项目级别限流配置，则对应项目下所有请求遵循这个配置，即同一项目下所有请求共享这个容量限额。 + */ +class DownloadUsageRateLimitRule : UploadUsageRateLimitRule() { + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.DOWNLOAD_USAGE.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DownloadUsageRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRule.kt new file mode 100644 index 0000000000..55a5759094 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRule.kt @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 上传用量限流配置规则实现 + */ +open class UploadUsageRateLimitRule : RateLimitRule { + + val usageLimitRules: PathResourceLimitRule = PathResourceLimitRule(pathLengthCheck = true) + + override fun isEmpty(): Boolean { + return usageLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var realResource = resInfo.resource + if (realResource.isBlank()) { + return null + } + var ruleLimit = usageLimitRules.getPathResourceLimit(realResource) + if (ruleLimit == null && resInfo.extraResource.isNotEmpty()) { + for (res in resInfo.extraResource) { + ruleLimit = usageLimitRules.getPathResourceLimit(res) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + return ResLimitInfo(realResource, ruleLimit) + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + usageLimitRules.addPathResourceLimit(resourceLimit, usageDimensionList) + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.UPLOAD_USAGE.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UploadUsageRateLimitRule::class.java) + private val usageDimensionList = listOf(LimitDimension.UPLOAD_USAGE.name, LimitDimension.DOWNLOAD_USAGE.name) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRule.kt new file mode 100644 index 0000000000..49d24c8548 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRule.kt @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +/** + * 用户下载用量限流规则实现 + */ +class UserDownloadUsageRateLimitRule( + userUsageLimitRules: ConcurrentHashMap = ConcurrentHashMap(), + userLimitRules: ConcurrentHashMap = ConcurrentHashMap() +) : UserUploadUsageRateLimitRule(userUsageLimitRules, userLimitRules) { + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.USER_DOWNLOAD_USAGE.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserDownloadUsageRateLimitRule::class.java) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRule.kt new file mode 100644 index 0000000000..3f79d1b61a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRule.kt @@ -0,0 +1,140 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +/** + * 用户上传用量限流配置规则实现 + */ +open class UserUploadUsageRateLimitRule( + // 用户+用量对应规则 + private val userUsageLimitRules: ConcurrentHashMap = ConcurrentHashMap(), + // 用户对应规则 + private val userLimitRules: ConcurrentHashMap = ConcurrentHashMap() +) : RateLimitRule { + + override fun isEmpty(): Boolean { + return userUsageLimitRules.isEmpty() && userLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var resLimitInfo = findConfigRule(resInfo.resource, resInfo.extraResource) + if (resLimitInfo == null) { + resLimitInfo = findConfigRule(resInfo.resource, resInfo.extraResource, true) + } + if (resLimitInfo == null && resInfo.extraResource.isNotEmpty()) { + val res = resInfo.extraResource.last() + val (user, _) = ResourcePathUtils.getUserAndPath(res) + val userRule = userLimitRules[user] + if (userRule != null) { + resLimitInfo = ResLimitInfo(res, userRule) + } + if (resLimitInfo == null && userLimitRules.containsKey(StringPool.POUND)) { + resLimitInfo = ResLimitInfo(res, userLimitRules[StringPool.POUND]!!) + } + } + return resLimitInfo + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + val (userId, path) = ResourcePathUtils.getUserAndPath(resourceLimit.resource) + if (path.isEmpty()) { + userLimitRules[userId] = resourceLimit + } else { + val userUsageResourceLimitRule = userUsageLimitRules.getOrDefault(userId, UserUsageResourceLimitRule()) + userUsageResourceLimitRule.addUserUsageResourceLimit(resourceLimit) + userUsageLimitRules.putIfAbsent(userId, userUsageResourceLimitRule) + } + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.USER_UPLOAD_USAGE.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + /** + * 根据资源获取配置 + */ + private fun findConfigRule( + resource: String, extraResource: List, userPattern: Boolean = false + ): ResLimitInfo? { + var realResource = resource + if (realResource.isBlank()) { + return null + } + var (user, resWithoutUser) = ResourcePathUtils.getUserAndPath(realResource) + if (userPattern) { + user = StringPool.POUND + } + val userUsageRule = userUsageLimitRules[user] + var ruleLimit = userUsageRule?.getPathResourceLimit(resWithoutUser) + if (ruleLimit == null && extraResource.isNotEmpty()) { + for (res in extraResource) { + var (_, resWithoutUser) = ResourcePathUtils.getUserAndPath(res) + ruleLimit = userUsageRule?.getPathResourceLimit(resWithoutUser) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + val resourceLimitCopy = ruleLimit.copy(resource = ResourcePathUtils.buildUserPath(user, ruleLimit.resource)) + return ResLimitInfo(realResource, resourceLimitCopy) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUploadUsageRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUsageResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUsageResourceLimitRule.kt new file mode 100644 index 0000000000..0ae86f9fef --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUsageResourceLimitRule.kt @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.getUserAndPath +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 用户+用量对应规则 + * 如果配置了仓库级别限流配置，则该用户下对应仓库下所有请求遵循这个配置，即同一仓库下所有请求共享这个容量限额； + * 如果配置了项目级别限流配置，则用户下对应项目下所有请求遵循这个配置，即同一项目下所有请求共享这个容量限额。 + */ +class UserUsageResourceLimitRule( + root: PathNode = PathNode("/"), + pathLengthCheck: Boolean = true, + private var user: String = StringPool.POUND, +) : PathResourceLimitRule(root, pathLengthCheck) { + + fun addUserUsageResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension !in userUsageDimensionList) { + return + } + val (userId, urlPath) = getUserAndPath(resourceLimit.resource) + if (!urlPath.startsWith("/") || user.isBlank()) { + logger.warn("$resourceLimit is invalid") + throw InvalidResourceException(urlPath) + } + user = userId + val resourceLimitCopy = resourceLimit.copy(resource = urlPath) + addPathResourceLimit(resourceLimitCopy, userUsageDimensionList) + } + + fun addUserUsageResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUserUsageResourceLimit(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUsageResourceLimitRule::class.java) + private val userUsageDimensionList = listOf( + LimitDimension.USER_DOWNLOAD_USAGE.name, + LimitDimension.USER_UPLOAD_USAGE.name + ) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractBandwidthRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractBandwidthRateLimiterService.kt new file mode 100644 index 0000000000..cdcea5f9e0 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractBandwidthRateLimiterService.kt @@ -0,0 +1,158 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + +import com.tencent.bkrepo.common.ratelimiter.algorithm.RateLimiter +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream +import com.tencent.bkrepo.common.ratelimiter.stream.RateCheckContext +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.util.unit.DataSize +import java.io.InputStream +import javax.servlet.http.HttpServletRequest + +/** + * 带宽限流器抽象实现 + */ +abstract class AbstractBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterProperties: RateLimiterProperties, + rateLimiterConfigService: RateLimiterConfigService, +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun limit(request: HttpServletRequest, applyPermits: Long?) { + throw UnsupportedOperationException() + } + + /** + * 根据资源返回对应带限流实现的InputStream + */ + fun bandwidthRateStart( + request: HttpServletRequest, + inputStream: InputStream, + circuitBreakerPerSecond: DataSize, + rangeLength: Long? = null, + ): CommonRateLimitInputStream? { + val resLimitInfo = getResLimitInfo(request) ?: return null + logger.info("will check the bandwidth with length $rangeLength of ${resLimitInfo.resource}") + return try { + interceptorChain.doBeforeLimitCheck(resLimitInfo.resource, resLimitInfo.resourceLimit) + circuitBreakerCheck(resLimitInfo.resourceLimit, circuitBreakerPerSecond.toBytes()) + val rateLimiter = getAlgorithmOfRateLimiter(resLimitInfo.resource, resLimitInfo.resourceLimit) + val context = RateCheckContext( + rateLimiter = rateLimiter, latency = rateLimiterProperties.latency, + waitRound = rateLimiterProperties.waitRound, rangeLength = rangeLength, + dryRun = rateLimiterProperties.dryRun, permitsOnce = rateLimiterProperties.permitsOnce, + limitPerSecond = getPermitsPerSecond(resLimitInfo.resourceLimit) + ) + CommonRateLimitInputStream( + delegate = inputStream, + rateCheckContext = context + ) + } catch (e: AcquireLockFailedException) { + logger.warn( + "acquire lock failed for ${resLimitInfo.resource} " + + "with ${resLimitInfo.resourceLimit}, e: ${e.message}" + ) + null + } catch (e: InvalidResourceException) { + logger.warn("${resLimitInfo.resourceLimit} is invalid for ${resLimitInfo.resource} , e: ${e.message}") + null + } + } + + fun bandwidthRateLimitFinish( + request: HttpServletRequest, + exception: Exception? = null, + ) { + val resLimitInfo = getResLimitInfo(request) ?: return + afterRateLimitCheck(resLimitInfo, exception == null, exception) + } + + fun bandwidthRateLimit( + request: HttpServletRequest, + permits: Long, + circuitBreakerPerSecond: DataSize, + ) { + val resLimitInfo = getResLimitInfo(request) ?: return + rateLimitCatch( + request = request, + resLimitInfo = resLimitInfo, + applyPermits = permits, + circuitBreakerPerSecond = circuitBreakerPerSecond.toBytes() + ) { rateLimiter, realPermits -> + bandwidthLimitHandler(rateLimiter, realPermits) + } + } + + fun bandwidthLimitHandler( + rateLimiter: RateLimiter, + permits: Long + ): Boolean { + var flag = false + var failedNum = 0 + while (!flag) { + flag = rateLimiter.tryAcquire(permits) + if (!flag && failedNum < rateLimiterProperties.waitRound) { + failedNum++ + try { + Thread.sleep(rateLimiterProperties.latency) + } catch (ignore: InterruptedException) { + } + } + if (!flag && failedNum >= rateLimiterProperties.waitRound) { + return false + } + } + return true + } + + private fun getPermitsPerSecond(resourceLimit: ResourceLimit): Long { + return resourceLimit.limit / resourceLimit.duration.seconds + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(AbstractBandwidthRateLimiterService::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterService.kt new file mode 100644 index 0000000000..a9c3c60457 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterService.kt @@ -0,0 +1,560 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + + +import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.api.util.JsonUtils.objectMapper +import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.query.enums.OperationType +import com.tencent.bkrepo.common.query.model.QueryModel +import com.tencent.bkrepo.common.query.model.Rule +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedFixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedLeakyRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedSlidingWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedTokenBucketRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.FixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.LeakyRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.RateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.SlidingWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.TokenBucketRateLimiter +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.interceptor.MonitorRateLimiterInterceptorAdaptor +import com.tencent.bkrepo.common.ratelimiter.interceptor.RateLimiterInterceptor +import com.tencent.bkrepo.common.ratelimiter.interceptor.RateLimiterInterceptorChain +import com.tencent.bkrepo.common.ratelimiter.interceptor.TargetRateLimiterInterceptorAdaptor +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.DownloadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.UploadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.usage.DownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.usage.UploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserDownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserUploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.service.servlet.MultipleReadHttpRequest +import java.util.concurrent.ConcurrentHashMap +import javax.servlet.http.HttpServletRequest +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.http.HttpMethod +import org.springframework.http.MediaType +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.util.unit.DataSize +import org.springframework.web.servlet.HandlerMapping + +/** + * 限流器抽象实现 + */ +abstract class AbstractRateLimiterService( + private val taskScheduler: ThreadPoolTaskScheduler, + val rateLimiterProperties: RateLimiterProperties, + private val rateLimiterMetrics: RateLimiterMetrics, + private val redisTemplate: RedisTemplate? = null, + private val rateLimiterConfigService: RateLimiterConfigService, +) : RateLimiterService { + + @Value("\${spring.application.name}") + var moduleName: String = StringPool.EMPTY + + + // 资源对应限限流算法缓存 + private var rateLimiterCache: ConcurrentHashMap = ConcurrentHashMap(256) + + val interceptorChain: RateLimiterInterceptorChain = + RateLimiterInterceptorChain( + mutableListOf( + MonitorRateLimiterInterceptorAdaptor(rateLimiterMetrics), + TargetRateLimiterInterceptorAdaptor(rateLimiterConfigService) + ) + ) + + // 限流规则配置 + var rateLimitRule: RateLimitRule? = null + + // 当前限流规则配置hashcode + var currentRuleHashCode: Int? = null + + init { + taskScheduler.scheduleWithFixedDelay(this::refreshRateLimitRule, rateLimiterProperties.refreshDuration * 1000) + } + + /** + * 获取对应资源限流规则配置 + */ + fun getResLimitInfo(request: HttpServletRequest): ResLimitInfo? { + if (!rateLimiterProperties.enabled) { + return null + } + val resLimitInfo = try { + val resInfo = ResInfo( + resource = buildResource(request), + extraResource = buildExtraResource(request) + ) + rateLimitRule?.getRateLimitRule(resInfo) + } catch (e: InvalidResourceException) { + logger.warn("Config is invalid for request ${request.requestURI}, e: ${e.message}") + null + } + + if (resLimitInfo == null) { + return null + } + return resLimitInfo + } + + override fun limit(request: HttpServletRequest, applyPermits: Long?) { + if (!rateLimiterProperties.enabled) { + return + } + if (ignoreRequest(request)) return + if (rateLimitRule == null || rateLimitRule!!.isEmpty()) return + val resLimitInfo = getResLimitInfo(request) ?: return + rateLimitCatch( + request = request, + resLimitInfo = resLimitInfo, + applyPermits = applyPermits, + ) { rateLimiter, permits -> + rateLimiter.tryAcquire(permits) + } + } + + override fun addInterceptor(interceptor: RateLimiterInterceptor) { + this.interceptorChain.addInterceptor(interceptor) + } + + override fun addInterceptors(interceptors: List) { + if (interceptors.isNotEmpty()) { + this.interceptorChain.addInterceptors(interceptors) + } + } + + /** + * 生成资源对应的唯一key + */ + abstract fun generateKey(resource: String, resourceLimit: ResourceLimit): String + + /** + * 根据请求获取对应的资源，用于查找对应限流规则 + */ + abstract fun buildResource(request: HttpServletRequest): String + + /** + * 根据请求获取对其他资源信息，用于查找对应限流规则 + */ + abstract fun buildExtraResource(request: HttpServletRequest): List + + /** + * 根据请求获取需要申请的许可数 + */ + abstract fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long + + /** + * 限流器实现对应的维度 + */ + abstract fun getLimitDimensions(): List + + /** + * 获取对应限流规则配置实现 + */ + abstract fun getRateLimitRuleClass(): Class + + /** + * 对请求进行过滤，不进行限流 + */ + open fun ignoreRequest(request: HttpServletRequest): Boolean { + return false + } + + /** + * 根据资源和限流规则生成对应限流算法 + */ + open fun createAlgorithmOfRateLimiter(resource: String, resourceLimit: ResourceLimit): RateLimiter { + if (resourceLimit.limit < 0) { + throw InvalidResourceException("config limit is ${resourceLimit.limit}") + } + return when (resourceLimit.algo) { + Algorithms.FIXED_WINDOW.name -> { + buildFixedWindowRateLimiter(resource, resourceLimit) + } + + Algorithms.TOKEN_BUCKET.name -> { + buildTokenBucketRateLimiter(resource, resourceLimit) + } + + Algorithms.SLIDING_WINDOW.name -> { + buildSlidingWindowRateLimiter(resource, resourceLimit) + } + + Algorithms.LEAKY_BUCKET.name -> { + buildLeakyRateLimiter(resource, resourceLimit) + } + + else -> { + throw InvalidResourceException("config algo is ${resourceLimit.algo}") + } + } + } + + fun getRepoInfoFromAttribute(request: HttpServletRequest): Pair { + var projectId: String? = null + var repoName: String? = null + try { + projectId = ((request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE)) + as Map<*, *>)["projectId"] as String? + repoName = ((request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE)) + as Map<*, *>)["repoName"] as String? + } catch (ignore: Exception) { + } + if (projectId.isNullOrEmpty()) { + throw InvalidResourceException("Could not find projectId from request ${request.requestURI}") + } + return Pair(projectId, repoName) + } + + fun getRepoInfoFromBody(request: HttpServletRequest): Pair { + val limit = DataSize.ofMegabytes(1).toBytes() + val lengthCondition = request.contentLength in 1..limit + val typeCondition = request.contentType?.startsWith(MediaType.APPLICATION_JSON_VALUE) == true + // 限制缓存大小 + if (lengthCondition && typeCondition) { + val multiReadRequest = MultipleReadHttpRequest(request, limit) + val (projectId, repoName) = getRepoInfoFromQueryModel(multiReadRequest) + if (!projectId.isNullOrEmpty()) return Pair(projectId, repoName) + val (newProjectId, newRepoName) = getRepoInfoFromOtherRequest(multiReadRequest) + if (newProjectId.isNullOrEmpty()) { + throw InvalidResourceException("Could not find projectId from request ${request.requestURI}") + } + return Pair(newProjectId, newRepoName) + } + throw InvalidResourceException("Could not find projectId from body of request ${request.requestURI}") + } + + private fun getRepoInfoFromQueryModel(multiReadRequest: MultipleReadHttpRequest): Pair { + try { + val queryModel = multiReadRequest.inputStream.readJsonString() + val rule = queryModel.rule + if (rule is Rule.NestedRule && rule.relation == Rule.NestedRule.RelationType.AND) { + return findRepoInfoFromRule(rule) + } + } catch (ignore: Exception) { + } + return Pair(null, null) + } + + private fun getRepoInfoFromOtherRequest(multiReadRequest: MultipleReadHttpRequest): Pair { + try { + val mappedValue = objectMapper.readValue>(multiReadRequest.inputStream) + return Pair((mappedValue[PROJECT_ID] as? String), (mappedValue[REPO_NAME] as? String)) + } catch (ignore: Exception) { + return Pair(null, null) + } + } + + private fun findRepoInfoFromRule(rule: Rule.NestedRule): Pair { + var projectId: String? = null + var repoName: String? = null + findKeyRule(PROJECT_ID, rule.rules)?.let { + it.value.toString().apply { projectId = this } + } + findKeyRule(REPO_NAME, rule.rules)?.let { + if (it.operation == OperationType.EQ) { + it.value.toString().apply { repoName = this } + } + } + return Pair(projectId, repoName) + } + + private fun findKeyRule(key: String, rules: List): Rule.QueryRule? { + for (rule in rules) { + if (rule is Rule.QueryRule && rule.field == key) { + return rule + } + } + return null + } + + /** + * 配置规则刷新 + */ + fun refreshRateLimitRule() { + if (!rateLimiterProperties.enabled) return + val usageRuleConfigs = rateLimiterProperties.rules.filter { + it.limitDimension in getLimitDimensions() + } + val databaseConfig = try { + rateLimiterConfigService.findByModuleNameAndLimitDimension( + moduleName, getLimitDimensions().first() + ) + } catch (e: Exception) { + logger.error("system error: $e") + listOf() + } + val configs = usageRuleConfigs.plus(databaseConfig.map { tRateLimit -> + ResourceLimit( + algo = tRateLimit.algo, + resource = tRateLimit.resource, + limit = tRateLimit.limit, + limitDimension = tRateLimit.limitDimension, + duration = tRateLimit.duration, + capacity = tRateLimit.capacity, + scope = tRateLimit.scope, + targets = tRateLimit.targets + ) + }) + // 配置规则变更后需要清理缓存的限流算法实现 + val newRuleHashCode = configs.hashCode() + if (currentRuleHashCode == newRuleHashCode) { + if (rateLimiterCache.size > rateLimiterProperties.cacheCapacity) { + clearLimiterCache() + } + return + } + val usageRules = getRuleClass() ?: return + usageRules.addRateLimitRules(configs) + rateLimitRule = usageRules + clearLimiterCache() + currentRuleHashCode = newRuleHashCode + logger.info("rules in ${this.javaClass.simpleName} for request has been refreshed!") + } + + private fun getRuleClass(): RateLimitRule? { + return when (getRateLimitRuleClass()) { + UrlRateLimitRule::class.java -> UrlRateLimitRule() + UploadUsageRateLimitRule::class.java -> UploadUsageRateLimitRule() + DownloadUsageRateLimitRule::class.java -> DownloadUsageRateLimitRule() + UserDownloadUsageRateLimitRule::class.java -> UserDownloadUsageRateLimitRule() + UserUploadUsageRateLimitRule::class.java -> UserUploadUsageRateLimitRule() + UserUrlRateLimitRule::class.java -> UserUrlRateLimitRule() + UploadBandwidthRateLimitRule::class.java -> UploadBandwidthRateLimitRule() + DownloadBandwidthRateLimitRule::class.java -> DownloadBandwidthRateLimitRule() + UrlRepoRateLimitRule::class.java -> UrlRepoRateLimitRule() + UserUrlRepoRateLimitRule::class.java -> UserUrlRepoRateLimitRule() + else -> null + } + } + + private fun clearLimiterCache() { + rateLimiterCache.forEach { + try { + it.value.removeCacheLimit(it.key) + } catch (e: Exception) { + logger.warn("clear limiter cache error: ${e.cause}, ${e.message}") + } + } + rateLimiterCache.clear() + } + + private fun beforeRateLimitCheck( + request: HttpServletRequest, + applyPermits: Long? = null, + resLimitInfo: ResLimitInfo, + circuitBreakerPerSecond: Long? = null, + ): Pair { + with(resLimitInfo) { + val realPermits = getApplyPermits(request, applyPermits) + interceptorChain.doBeforeLimitCheck(resource, resourceLimit) + circuitBreakerCheck(resourceLimit, circuitBreakerPerSecond) + val rateLimiter = getAlgorithmOfRateLimiter(resource, resourceLimit) + return Pair(rateLimiter, realPermits) + } + } + + fun afterRateLimitCheck( + resLimitInfo: ResLimitInfo, + pass: Boolean, + exception: Exception? = null, + ) { + with(resLimitInfo) { + interceptorChain.doAfterLimitCheck(resource, resourceLimit, pass, exception) + } + } + + fun rateLimitCatch( + request: HttpServletRequest, + resLimitInfo: ResLimitInfo, + applyPermits: Long? = null, + circuitBreakerPerSecond: Long? = null, + action: (RateLimiter, Long) -> Boolean + ) { + var exception: Exception? = null + var pass = false + try { + val (rateLimiter, permits) = beforeRateLimitCheck( + request = request, + applyPermits = applyPermits, + resLimitInfo = resLimitInfo, + circuitBreakerPerSecond = circuitBreakerPerSecond + ) + val realPermits = permits + pass = action(rateLimiter, realPermits) + if (!pass) { + val msg = "${resLimitInfo.resource} has exceeded max rate limit: " + + "${resLimitInfo.resourceLimit.limit} /${resLimitInfo.resourceLimit.duration}" + if (rateLimiterProperties.dryRun) { + logger.warn(msg) + } else { + throw OverloadException(msg) + } + } + } catch (e: OverloadException) { + throw e + } catch (e: AcquireLockFailedException) { + logger.warn( + "acquire lock failed for ${resLimitInfo.resource}" + + " with ${resLimitInfo.resourceLimit}, e: ${e.message}" + ) + exception = e + } catch (e: InvalidResourceException) { + logger.warn("${resLimitInfo.resourceLimit} is invalid ${resLimitInfo.resource} , e: ${e.message}") + exception = e + } catch (e: Exception) { + logger.error("internal error: $e") + exception = e + } finally { + afterRateLimitCheck(resLimitInfo, pass, exception) + } + } + + /** + * 获取对应限流算法实现 + */ + fun getAlgorithmOfRateLimiter( + resource: String, resourceLimit: ResourceLimit + ): RateLimiter { + val limitKey = generateKey(resource, resourceLimit) + var rateLimiter = rateLimiterCache[limitKey] + if (rateLimiter == null) { + val newRateLimiter = createAlgorithmOfRateLimiter(limitKey, resourceLimit) + rateLimiter = rateLimiterCache.putIfAbsent(limitKey, newRateLimiter) + if (rateLimiter == null) { + rateLimiter = newRateLimiter + } + } + return rateLimiter + } + + /** + * （特殊操作）如果配置的限流规则比熔断配置小，则直接限流 + */ + fun circuitBreakerCheck( + resourceLimit: ResourceLimit, + circuitBreakerPerSecond: Long? = null, + ) { + if (circuitBreakerPerSecond == null) return + val permitsPerSecond = resourceLimit.limit / resourceLimit.duration.seconds + if (circuitBreakerPerSecond >= permitsPerSecond) { + throw OverloadException( + "The circuit breaker is activated when too many download requests are made to the service!" + ) + } + } + + private fun buildFixedWindowRateLimiter( + resource: String, + resourceLimit: ResourceLimit, + ): RateLimiter { + return if (resourceLimit.scope == WorkScope.LOCAL.name) { + FixedWindowRateLimiter(resourceLimit.limit, resourceLimit.duration) + } else { + DistributedFixedWindowRateLimiter( + resource, resourceLimit.limit, resourceLimit.duration, redisTemplate!! + ) + } + } + + private fun buildTokenBucketRateLimiter( + resource: String, + resourceLimit: ResourceLimit, + ): RateLimiter { + val permitsPerSecond = (resourceLimit.limit / resourceLimit.duration.seconds.toDouble()) + return if (resourceLimit.scope == WorkScope.LOCAL.name) { + TokenBucketRateLimiter(permitsPerSecond) + } else { + if (resourceLimit.capacity == null || resourceLimit.capacity!! <= 0) { + throw InvalidResourceException("Resource limit config $resourceLimit is illegal") + } + DistributedTokenBucketRateLimiter( + resource, permitsPerSecond, resourceLimit.capacity!!, redisTemplate!! + ) + } + } + + private fun buildSlidingWindowRateLimiter( + resource: String, + resourceLimit: ResourceLimit, + ): RateLimiter { + return if (resourceLimit.scope == WorkScope.LOCAL.name) { + SlidingWindowRateLimiter(resourceLimit.limit, resourceLimit.duration) + } else { + DistributedSlidingWindowRateLimiter( + resource, resourceLimit.limit, resourceLimit.duration, redisTemplate!! + ) + } + } + + private fun buildLeakyRateLimiter( + resource: String, + resourceLimit: ResourceLimit, + ): RateLimiter { + if (resourceLimit.capacity == null || resourceLimit.capacity!! <= 0) { + throw InvalidResourceException("Resource limit config $resourceLimit is illegal") + } + val rate = resourceLimit.limit / resourceLimit.duration.seconds.toDouble() + return if (resourceLimit.scope == WorkScope.LOCAL.name) { + LeakyRateLimiter(rate, resourceLimit.capacity!!) + } else { + DistributedLeakyRateLimiter( + resource, rate, resourceLimit.capacity!!, redisTemplate!! + ) + } + } + + companion object { + val logger: Logger = LoggerFactory.getLogger(AbstractRateLimiterService::class.java) + val UPLOAD_REQUEST_METHOD = listOf(HttpMethod.POST.name, HttpMethod.PUT.name, HttpMethod.PATCH.name) + val DOWNLOAD_REQUEST_METHOD = listOf(HttpMethod.GET.name) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RateLimiterService.kt new file mode 100644 index 0000000000..5d1e0761d0 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RateLimiterService.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.interceptor.RateLimiterInterceptor +import javax.servlet.http.HttpServletRequest + +interface RateLimiterService { + + /** + * 限流判断 + */ + @Throws(AcquireLockFailedException::class, InvalidResourceException::class, OverloadException::class) + fun limit(request: HttpServletRequest, applyPermits: Long? = null) + + fun addInterceptor(interceptor: RateLimiterInterceptor) + + fun addInterceptors(interceptors: List) +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RequestLimitCheckService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RequestLimitCheckService.kt new file mode 100644 index 0000000000..315ebe8011 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RequestLimitCheckService.kt @@ -0,0 +1,176 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + +import com.tencent.bkrepo.common.ratelimiter.RateLimiterAutoConfiguration +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.service.bandwidth.DownloadBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.bandwidth.UploadBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.UrlRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.UrlRepoRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.user.UserUrlRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.user.UserUrlRepoRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.DownloadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.UploadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.user.UserDownloadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.user.UserUploadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.util.unit.DataSize +import java.io.InputStream +import javax.servlet.http.HttpServletRequest + +class RequestLimitCheckService( + private val rateLimiterProperties: RateLimiterProperties, +) { + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.URL_REPO_RATELIMITER_SERVICE) + private lateinit var urlRepoRateLimiterService: UrlRepoRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.URL_RATELIMITER_SERVICE) + private lateinit var urlRateLimiterService: UrlRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.USER_URL_REPO_RATELIMITER_SERVICE) + private lateinit var userUrlRepoRateLimiterService: UserUrlRepoRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.UPLOAD_USAGE_RATELIMITER_SERVICE) + private lateinit var uploadUsageRateLimiterService: UploadUsageRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.USER_URL_RATELIMITER_SERVICE) + private lateinit var userUrlRateLimiterService: UserUrlRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.USER_UPLOAD_USAGE_RATELIMITER_SERVICE) + private lateinit var userUploadUsageRateLimiterService: UserUploadUsageRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.DOWNLOAD_USAGE_RATELIMITER_SERVICE) + private lateinit var downloadUsageRateLimiterService: DownloadUsageRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.USER_DOWNLOAD_USAGE_RATELIMITER_SERVICE) + private lateinit var userDownloadUsageRateLimiterService: UserDownloadUsageRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.DOWNLOAD_BANDWIDTH_RATELIMITER_SERVICE) + private lateinit var downloadBandwidthRateLimiterService: DownloadBandwidthRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.UPLOAD_BANDWIDTH_RATELIMITER_ERVICE) + private lateinit var uploadBandwidthRateLimiterService: UploadBandwidthRateLimiterService + + fun preLimitCheck(request: HttpServletRequest) { + if (!rateLimiterProperties.enabled) { + return + } + // TODO 可以优化 + urlRepoRateLimiterService.limit(request) + userUrlRepoRateLimiterService.limit(request) + userUrlRateLimiterService.limit(request) + userUploadUsageRateLimiterService.limit(request) + urlRateLimiterService.limit(request) + uploadUsageRateLimiterService.limit(request) + } + + fun postLimitCheck(applyPermits: Long) { + if (!rateLimiterProperties.enabled) { + return + } + val request = getRequest() ?: return + downloadUsageRateLimiterService.limit(request, applyPermits) + userDownloadUsageRateLimiterService.limit(request, applyPermits) + + } + + fun bandwidthCheck( + inputStream: InputStream, + circuitBreakerPerSecond: DataSize, + rangeLength: Long? = null, + ): CommonRateLimitInputStream? { + if (!rateLimiterProperties.enabled) { + return null + } + val request = getRequest() ?: return null + if (!downloadBandwidthRateLimiterService.ignoreRequest(request)) { + return downloadBandwidthRateLimiterService.bandwidthRateStart( + request, inputStream, circuitBreakerPerSecond, rangeLength + ) + } + if (!uploadBandwidthRateLimiterService.ignoreRequest(request)) { + return uploadBandwidthRateLimiterService.bandwidthRateStart( + request, inputStream, circuitBreakerPerSecond, rangeLength + ) + } + return null + } + + fun bandwidthFinish(exception: Exception? = null) { + if (!rateLimiterProperties.enabled) { + return + } + val request = getRequest() ?: return + if (!downloadBandwidthRateLimiterService.ignoreRequest(request)) { + return downloadBandwidthRateLimiterService.bandwidthRateLimitFinish( + request, exception + ) + } + if (!uploadBandwidthRateLimiterService.ignoreRequest(request)) { + return uploadBandwidthRateLimiterService.bandwidthRateLimitFinish( + request, exception + ) + } + } + + fun uploadBandwidthCheck( + applyPermits: Long, + circuitBreakerPerSecond: DataSize, + ) { + if (!rateLimiterProperties.enabled) { + return + } + val request = getRequest() ?: return + uploadBandwidthRateLimiterService.bandwidthRateLimit( + request, applyPermits, circuitBreakerPerSecond + ) + } + + private fun getRequest(): HttpServletRequest? { + return try { + HttpContextHolder.getRequest() + } catch (e: IllegalArgumentException) { + return null + } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterService.kt new file mode 100644 index 0000000000..ddfd54edc5 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterService.kt @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.DownloadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 下载带宽限流器实现，针对project/repo进行限流 + */ +class DownloadBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : UploadBandwidthRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.DOWNLOAD_BANDWIDTH.name + ) + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in DOWNLOAD_REQUEST_METHOD + } + + override fun getRateLimitRuleClass(): Class { + return DownloadBandwidthRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "DownloadBandwidth:$resource" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterService.kt new file mode 100644 index 0000000000..1c99e58b9c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterService.kt @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.bandwidth + + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.UploadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.AbstractBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 上传带宽限流器实现, 针对project/repo进行限流 + */ +open class UploadBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : AbstractBandwidthRateLimiterService( + taskScheduler, + rateLimiterMetrics, + redisTemplate, + rateLimiterProperties, + rateLimiterConfigService +) { + + override fun buildResource(request: HttpServletRequest): String { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + return if (repoName.isNullOrEmpty()) { + "/$projectId/" + } else { + "/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + if (repoName.isNullOrEmpty()) return emptyList() + return listOf("/$projectId/") + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + if (applyPermits == null) { + throw AcquireLockFailedException("apply permits is null") + } + return applyPermits + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.UPLOAD_BANDWIDTH.name + ) + } + + override fun getRateLimitRuleClass(): Class { + return UploadBandwidthRateLimitRule::class.java + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in UPLOAD_REQUEST_METHOD + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UploadBandwidth:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterService.kt new file mode 100644 index 0000000000..8df65479fc --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterService.kt @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url + + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * url限流器实现 + */ +class UrlRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + + override fun buildResource(request: HttpServletRequest): String { + return request.requestURI + } + + override fun buildExtraResource(request: HttpServletRequest): List { + return emptyList() + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return 1 + } + + override fun getLimitDimensions(): List { + return listOf(LimitDimension.URL.name) + } + + override fun getRateLimitRuleClass(): Class { + return UrlRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "Url:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterService.kt new file mode 100644 index 0000000000..3189f7f0b1 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterService.kt @@ -0,0 +1,112 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.web.servlet.HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE +import javax.servlet.http.HttpServletRequest + +/** + * urlRepo限流器实现 + */ +class UrlRepoRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + if (rateLimiterProperties.specialUrls.contains(StringPool.POUND)) { + return false + } + return !rateLimiterProperties.specialUrls.contains(request.getAttribute(BEST_MATCHING_PATTERN_ATTRIBUTE)) + } + + override fun buildResource(request: HttpServletRequest): String { + val (projectId, repoName) = try { + getRepoInfoFromAttribute(request) + } catch (e: InvalidResourceException) { + getRepoInfoFromBody(request) + } + return if (repoName.isNullOrEmpty()) { + "/$projectId/" + } else { + "/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val (projectId, repoName) = try { + getRepoInfoFromAttribute(request) + } catch (e: InvalidResourceException) { + getRepoInfoFromBody(request) + } + val result = mutableListOf() + if (!repoName.isNullOrEmpty()) { + result.add("/$projectId/") + } + return result + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return 1 + } + + override fun getLimitDimensions(): List { + return listOf(LimitDimension.URL_REPO.name) + } + + override fun getRateLimitRuleClass(): Class { + return UrlRepoRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UrlRepo:$resource" + } + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterService.kt new file mode 100644 index 0000000000..a98975b789 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterService.kt @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url.user + + +import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.api.constant.USER_KEY +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 用户+url限流器实现 + */ +class UserUrlRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService, +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun buildResource(request: HttpServletRequest): String { + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + val realUrl = request.requestURI + return "$userId:$realUrl" + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + return listOf("$userId:") + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return 1 + } + + override fun getLimitDimensions(): List { + return listOf(LimitDimension.USER_URL.name) + } + + override fun getRateLimitRuleClass(): Class { + return UserUrlRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UserUrl:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterService.kt new file mode 100644 index 0000000000..b6e29a06a7 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterService.kt @@ -0,0 +1,118 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url.user + +import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.constant.USER_KEY +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.web.servlet.HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE +import javax.servlet.http.HttpServletRequest + +/** + * user+urlRepo限流器实现 + */ +class UserUrlRepoRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + private val rateLimiterConfigService: RateLimiterConfigService +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + if (rateLimiterProperties.specialUrls.contains(StringPool.POUND)) { + return false + } + return !rateLimiterProperties.specialUrls.contains(request.getAttribute(BEST_MATCHING_PATTERN_ATTRIBUTE)) + } + + override fun buildResource(request: HttpServletRequest): String { + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + val (projectId, repoName) = try { + getRepoInfoFromAttribute(request) + } catch (e: InvalidResourceException) { + getRepoInfoFromBody(request) + } + return if (repoName.isNullOrEmpty()) { + "$userId:/$projectId/" + } else { + "$userId:/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + val (projectId, repoName) = try { + getRepoInfoFromAttribute(request) + } catch (e: InvalidResourceException) { + getRepoInfoFromBody(request) + } + val result = mutableListOf() + if (!repoName.isNullOrEmpty()) { + result.add("$userId:/$projectId/") + } + result.add("$userId:") + return result + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return 1 + } + + override fun getLimitDimensions(): List { + return listOf(LimitDimension.USER_URL_REPO.name) + } + + override fun getRateLimitRuleClass(): Class { + return UserUrlRepoRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UserUrlRepo:$resource" + } + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterService.kt new file mode 100644 index 0000000000..8d8a0cbc64 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterService.kt @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.DownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 下载容量限流器实现, 针对project和repo + */ +class DownloadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : UploadUsageRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + if (applyPermits == null) { + throw AcquireLockFailedException("apply permits is null") + } + return applyPermits + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.DOWNLOAD_USAGE.name + ) + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in DOWNLOAD_REQUEST_METHOD + } + + override fun getRateLimitRuleClass(): Class { + return DownloadUsageRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "DownloadUsage:$resource" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterService.kt new file mode 100644 index 0000000000..2a38210253 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterService.kt @@ -0,0 +1,110 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage + + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.UploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 上传容量限流器实现，针对project和repo + */ +open class UploadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService, +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun buildResource(request: HttpServletRequest): String { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + return if (repoName.isNullOrEmpty()) { + "/$projectId/" + } else { + "/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + val result = mutableListOf() + if (!repoName.isNullOrEmpty()) { + result.add("/$projectId/") + } + return result + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return when (request.method) { + in UPLOAD_REQUEST_METHOD -> { + var length = request.contentLengthLong + if (length == -1L) { + logger.warn("content length of ${request.requestURI} is -1") + length = 0 + } + length + } + else -> 0 + } + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.UPLOAD_USAGE.name + ) + } + + override fun getRateLimitRuleClass(): Class { + return UploadUsageRateLimitRule::class.java + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in UPLOAD_REQUEST_METHOD + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UploadUsage:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterService.kt new file mode 100644 index 0000000000..3952414e63 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterService.kt @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage.user + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserDownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 用户下载容量限流器实现，针对user、project和repo + */ +class UserDownloadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : UserUploadUsageRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + if (applyPermits == null) { + throw AcquireLockFailedException("response content is null") + } + return applyPermits + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.USER_DOWNLOAD_USAGE.name + ) + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in DOWNLOAD_REQUEST_METHOD + } + + override fun getRateLimitRuleClass(): Class { + return UserDownloadUsageRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UserDownloadUsage:$resource" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterService.kt new file mode 100644 index 0000000000..42c714d3ad --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterService.kt @@ -0,0 +1,115 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage.user + +import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.api.constant.USER_KEY +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserUploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 用户上传容量限流器实现, 针对user、project和repo + */ +open class UserUploadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun buildResource(request: HttpServletRequest): String { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + return if (repoName.isNullOrEmpty()) { + "$userId:/$projectId/" + } else { + "$userId:/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + val result = mutableListOf() + if (!repoName.isNullOrEmpty()) { + result.add("$userId:/$projectId/") + } + result.add("$userId:") + return result + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return when (request.method) { + in UPLOAD_REQUEST_METHOD -> { + var length = request.contentLengthLong + if (length == -1L) { + logger.warn("content length of ${request.requestURI} is -1") + length = 0 + } + length + } + else -> 0 + } + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.USER_UPLOAD_USAGE.name + ) + } + + override fun getRateLimitRuleClass(): Class { + return UserUploadUsageRateLimitRule::class.java + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in UPLOAD_REQUEST_METHOD + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UserUploadUsage:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/user/RateLimiterConfigService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/user/RateLimiterConfigService.kt new file mode 100644 index 0000000000..fa8c6c0431 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/user/RateLimiterConfigService.kt @@ -0,0 +1,139 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.user + +import com.tencent.bkrepo.common.ratelimiter.model.RateLimitCreatOrUpdateRequest +import com.tencent.bkrepo.common.ratelimiter.model.TRateLimit +import com.tencent.bkrepo.common.ratelimiter.repository.RateLimitRepository +import java.time.Duration +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Service + +@Service +class RateLimiterConfigService( + private val rateLimitRepository: RateLimitRepository +) { + + @Value("\${spring.cloud.client.ip-address}") + var host: String = "127.0.0.1" + + fun list(): List { + return rateLimitRepository.findAll() + } + + fun create(fileCacheRequest: RateLimitCreatOrUpdateRequest) { + with(fileCacheRequest) { + rateLimitRepository.insert( + TRateLimit( + id = null, + resource = resource, + limitDimension = limitDimension, + algo = algo, + limit = limit, + duration = Duration.ofSeconds(duration), + capacity = capacity, + scope = scope, + moduleName = moduleName + ) + ) + } + } + + fun checkExist(id: String): Boolean { + return rateLimitRepository.existsById(id) + } + + fun checkExist(fileCacheRequest: RateLimitCreatOrUpdateRequest): Boolean { + with(fileCacheRequest) { + return rateLimitRepository.existsByResourceAndLimitDimension(resource, limitDimension) + } + } + + fun delete(id: String) { + rateLimitRepository.removeById(id) + } + + fun getById(id: String): TRateLimit? { + return rateLimitRepository.findById(id) + } + + fun update(fileCacheRequest: RateLimitCreatOrUpdateRequest) { + with(fileCacheRequest) { + targets?.let { + rateLimitRepository.save( + TRateLimit( + id = id, + resource = resource, + limitDimension = limitDimension, + algo = algo, + limit = limit, + duration = Duration.ofSeconds(duration), + capacity = capacity, + scope = scope, + moduleName = moduleName, + targets = it + ) + ) + } ?: run { + rateLimitRepository.save( + TRateLimit( + id = id, + resource = resource, + limitDimension = limitDimension, + algo = algo, + limit = limit, + duration = Duration.ofSeconds(duration), + capacity = capacity, + scope = scope, + moduleName = moduleName + ) + ) + } + } + } + + fun findByModuleNameAndLimitDimension(moduleName: String, limitDimension: String): List { + return rateLimitRepository.findByModuleNameAndLimitDimension(moduleName, limitDimension) + } + + fun findByResourceAndLimitDimension(resource: String, limitDimension: String): List { + return rateLimitRepository.findByResourceAndLimitDimension(resource, limitDimension) + } + + fun findByModuleNameAndLimitDimensionAndResource( + resource: String, + moduleName: List, + limitDimension: String + ): TRateLimit? { + return rateLimitRepository.findByModuleNameAndLimitDimensionAndResource(resource, moduleName, limitDimension) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStream.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStream.kt new file mode 100644 index 0000000000..2ce9d3cc9b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStream.kt @@ -0,0 +1,127 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.stream + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.artifact.stream.DelegateInputStream +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.io.InputStream + +class CommonRateLimitInputStream( + delegate: InputStream, + private val rateCheckContext: RateCheckContext +) : DelegateInputStream(delegate) { + + private var bytesRead: Long = 0 + private var applyNum: Long = 0 + + override fun read(): Int { + tryAcquire(1) + val data = super.read() + if (data != -1) { + bytesRead++ + } + return data + } + + override fun read(byteArray: ByteArray): Int { + tryAcquire(byteArray.size) + val readLen = super.read(byteArray) + if (readLen != -1) { + bytesRead += readLen + } + return readLen + } + + override fun read(byteArray: ByteArray, off: Int, len: Int): Int { + tryAcquire(len) + val readLen = super.read(byteArray, off, len) + if (readLen != -1) { + bytesRead += readLen + } + return readLen + } + + private fun tryAcquire(bytes: Int) { + with(rateCheckContext) { + if (rangeLength == null || rangeLength!! <= 0) { + // 当不知道文件大小时，没办法进行大小预估，无法降低申请频率， 只能每次读取都进行判断 + acquire(bytes.toLong()) + } else { + // 避免频繁申请，增加耗时，降低申请频率， 每次申请一定数量 + // 当申请的bytes比limitPerSecond还大时,直接限流 + if (limitPerSecond < bytes) { + if (rateCheckContext.dryRun) { + return + } + throw OverloadException("request reached bandwidth limit") + } + // 此处避免限流带宽大小比每次申请的还少的情况下，每次都被限流 + val realPermitOnce = limitPerSecond.coerceAtMost(permitsOnce) + if (bytesRead == 0L || (bytesRead + bytes) > applyNum) { + val leftLength = rangeLength!! - bytesRead + val permits = if (leftLength >= 0) { + leftLength.coerceAtMost(realPermitOnce) + } else { + // 当剩余文件大小小于0时，说明文件大小不正确，无法确认剩余多少 + realPermitOnce + } + acquire(permits) + applyNum += permits + } + } + } + } + + private fun acquire(permits: Long) { + var flag = false + var failedNum = 0 + while (!flag) { + // 当限制小于读取大小时，会进入死循环，增加等待轮次，如果达到等待轮次上限后还是无法获取，则抛异常结束 + try { + flag = rateCheckContext.rateLimiter.tryAcquire(permits) + } catch (ignore: AcquireLockFailedException) { + return + } + if (!flag && failedNum < rateCheckContext.waitRound) { + failedNum++ + try { + Thread.sleep(rateCheckContext.latency * failedNum) + } catch (ignore: InterruptedException) { + } + continue + } + if (!flag && failedNum >= rateCheckContext.waitRound) { + if (rateCheckContext.dryRun) { + return + } + throw OverloadException("request reached bandwidth limit") + } + } + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/RateCheckContext.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/RateCheckContext.kt new file mode 100644 index 0000000000..588266debe --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/RateCheckContext.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.stream + +import com.tencent.bkrepo.common.ratelimiter.algorithm.RateLimiter + +data class RateCheckContext( + var rateLimiter: RateLimiter, + var latency: Long, + var waitRound: Int, + var limitPerSecond: Long, + var rangeLength: Long? = null, + var dryRun: Boolean = false, + var permitsOnce: Long = 1024 * 1024, +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtils.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtils.kt new file mode 100644 index 0000000000..3e777bdedf --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtils.kt @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.utils + +import com.tencent.bkrepo.common.api.constant.CharPool +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import java.net.URI +import java.net.URISyntaxException + +object ResourcePathUtils { + + /** + * 分割路径， 支持带template变量路径 + */ + @Throws(InvalidResourceException::class) + fun tokenizeResourcePath(resourcePath: String): List { + if (resourcePath.isBlank()) { + return emptyList() + } + if (!resourcePath.startsWith("/")) { + throw InvalidResourceException("invalid resource path: $resourcePath") + } + val dirs = resourcePath.split("/").toTypedArray() + val dirList: MutableList = ArrayList() + for (i in dirs.indices) { + if (dirs[i].contains("?") + && (!dirs[i].startsWith("{") || !dirs[i].endsWith("}"))) { + throw InvalidResourceException("invalid resource path: $resourcePath") + } + if (dirs[i].isNotEmpty()) { + dirList.add(dirs[i]) + } + } + return dirList + } + + /** + * 获取URL路径，不带host等信息 + */ + fun getPathOfUrl(url: String): String? { + if (url.isBlank()) { + return null + } + val uri: URI + try { + uri = URI(url) + } catch (e: URISyntaxException) { + throw InvalidResourceException(url) + } + val path = uri.path + if (path.isNullOrEmpty()) { + return "/" + } + return path + } + + /** + * 截取user和path + */ + fun getUserAndPath(resource: String): Pair { + val index = resource.indexOfFirst { it == CharPool.COLON } + if (index == -1) { + throw InvalidResourceException(resource) + } + return Pair(resource.substring(0, index), resource.substring(index + 1)) + } + + /** + * 根据userId和path生成对应配置格式 + */ + fun buildUserPath(userId: String, resource: String): String { + return "$userId:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/resources/META-INF/spring.factories b/src/backend/common/common-ratelimiter/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..4de49fafff --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bkrepo.common.ratelimiter.RateLimiterAutoConfiguration diff --git a/src/backend/common/common-ratelimiter/src/main/resources/fix-window-rate-limiter.lua b/src/backend/common/common-ratelimiter/src/main/resources/fix-window-rate-limiter.lua new file mode 100644 index 0000000000..9d3c918b1b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/fix-window-rate-limiter.lua @@ -0,0 +1,14 @@ +local key = KEYS[1] +local limit = tonumber(ARGV[1]) +local adder = tonumber(ARGV[2]) +local ttl = tonumber(ARGV[3]) +local current = tonumber(redis.call('get', key) or "0") +if current + adder > limit then + return 0 +else + redis.call('incrby', key, adder) + if (current == 0) then + redis.call('EXPIRE', key, ttl) + end + return 1 +end \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/resources/leaky-rate-limiter.lua b/src/backend/common/common-ratelimiter/src/main/resources/leaky-rate-limiter.lua new file mode 100644 index 0000000000..7819c1cb6a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/leaky-rate-limiter.lua @@ -0,0 +1,58 @@ +local leaky_bucket_key = KEYS[1] +-- last update key +local last_bucket_key = KEYS[2] +-- the rate of leak water +local rate = tonumber(ARGV[1]) +-- capacity +local capacity = tonumber(ARGV[2]) +-- request count +local requested = tonumber(ARGV[3]) +-- current timestamp seconds +local now = tonumber(ARGV[4]) + +-- the key life time +local key_lifetime = math.ceil((capacity / rate) + 1) + +-- the yield of water in the bucket default 0 +local key_bucket_count = tonumber(redis.call("GET", leaky_bucket_key)) or 0 + +-- the last update time default now +local last_time = tonumber(redis.call("GET", last_bucket_key)) or now + +-- the time difference +local millis_since_last_leak = now - last_time + +-- the yield of water had lasted +local leaks = millis_since_last_leak * rate + +if leaks > 0 then + -- clean up the bucket + if leaks >= key_bucket_count then + key_bucket_count = 0 + else + -- compute the yield of water in the bucket + key_bucket_count = key_bucket_count - leaks + end + last_time = now +end + +-- is allowed pass default not allow +local is_allow = 0 + +local new_bucket_count = key_bucket_count + requested +-- allow +if new_bucket_count <= capacity then + is_allow = 1 +else + -- not allow + return {is_allow, new_bucket_count} +end + +-- update the key bucket water yield +redis.call("SETEX", leaky_bucket_key, key_lifetime, new_bucket_count) + +-- update last update time +redis.call("SETEX", last_bucket_key, key_lifetime, now) + +-- return +return {is_allow, new_bucket_count} diff --git a/src/backend/common/common-ratelimiter/src/main/resources/sliding-window-rate-limiter.lua b/src/backend/common/common-ratelimiter/src/main/resources/sliding-window-rate-limiter.lua new file mode 100644 index 0000000000..7c8b5d5cd5 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/sliding-window-rate-limiter.lua @@ -0,0 +1,36 @@ +-- key: 限流器的键名 +-- limit: 限流器的容量 +-- interval: 时间窗口的长度（单位为秒） +-- count: 一次获取的令牌数量 +local key = KEYS[1] +local limit = tonumber(ARGV[1]) +local interval = tonumber(ARGV[2]) +local count = tonumber(ARGV[3]) +local now_sec = tonumber(ARGV[4]) +local random = tonumber(ARGV[5]) + +-- 删除时间窗口之外的令牌 +redis.call('zremrangebyscore', key, 0, now_sec - interval) + +-- 获取当前时间窗口内的令牌数量 +local current = tonumber(redis.call('zcard', key)) + +-- 如果当前令牌数量已经达到限流器的容量，则不再生成新的令牌 +if current >= limit then + return 0 +end + +-- 计算需要生成的令牌数量 +local remaining = limit - current +local allowed_num = 0 +if (remaining < count) then + return { allowed_num, remaining } +end +allowed_num = 1 +-- 生成令牌，并返回生成的令牌数量 +local tokens = {} +for i = 1, count do + redis.call('zadd', key, now_sec, random..i) +end +redis.call('expire', key, interval) +return { allowed_num, remaining } diff --git a/src/backend/common/common-ratelimiter/src/main/resources/token-bucket-rate-limiter.lua b/src/backend/common/common-ratelimiter/src/main/resources/token-bucket-rate-limiter.lua new file mode 100644 index 0000000000..b0c8f44645 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/token-bucket-rate-limiter.lua @@ -0,0 +1,51 @@ +local tokens_key = KEYS[1] +local timestamp_key = KEYS[2] +-- 每秒填充速率 +local rate = tonumber(ARGV[1]) +-- 令牌桶最大数量 +local capacity = tonumber(ARGV[2]) +-- 消耗令牌数量 +local requested = tonumber(ARGV[3]) +-- now 当前时间秒 +local now = tonumber(ARGV[4]) + +-- 计算令牌桶填充满需要多久 +local fill_time = capacity/rate +-- *2保证时间充足 +local ttl = math.floor(fill_time*2) + +-- 防止小于0 +if ttl < 1 then + ttl = 10 +end +-- 获取令牌桶内剩余数量 +local last_tokens = tonumber(redis.call("get", tokens_key)) +-- 第一次没有数值，设置桶为满的 +if last_tokens == nil then + last_tokens = capacity +end +-- 获取上次更新时间 +local last_refreshed = tonumber(redis.call("get", timestamp_key)) +if last_refreshed == nil then + last_refreshed = 0 +end +-- 本次校验和上次更新时间的间隔 +local delta = math.max(0, now-last_refreshed) +-- 填充令牌，计算新的令牌桶剩余令牌数，填充不超过令牌桶上限 +local filled_tokens = math.min(capacity, last_tokens+(delta*rate)) + +-- 判断令牌数量是否足够 +local allowed = filled_tokens >= requested +local new_tokens = filled_tokens +local allowed_num = 0 +if allowed then + -- 如成功，令牌桶剩余令牌数减消耗令牌数 + new_tokens = filled_tokens - requested + allowed_num = 1 +end + +-- 设置令牌桶剩余令牌数，令牌桶最后填充时间now, ttl超时时间 +redis.call("setex", tokens_key, ttl, new_tokens) +redis.call("setex", timestamp_key, ttl, now) + +return { allowed_num, new_tokens } diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfigurationTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfigurationTest.kt new file mode 100644 index 0000000000..3d08fcdb28 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfigurationTest.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter + +import org.springframework.boot.SpringBootConfiguration +import org.springframework.boot.autoconfigure.EnableAutoConfiguration + +@SpringBootConfiguration +@EnableAutoConfiguration +class RateLimiterAutoConfigurationTest \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiterTest.kt new file mode 100644 index 0000000000..33d1a40b0d --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiterTest.kt @@ -0,0 +1,105 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.springframework.test.annotation.DirtiesContext +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DistributedFixedWindowRateLimiterTest : DistributedTest() { + + + @Test + fun testTryAcquire() { + val key = KEY + "testTryAcquire" + val ratelimiter = DistributedFixedWindowRateLimiter(key, 5, Duration.ofSeconds(1), redisTemplate) + println(System.currentTimeMillis()) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + Thread.sleep(1000) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + println(System.currentTimeMillis()) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + println(System.currentTimeMillis()) + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed7) + clean(key) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val key = KEY + "testTryAcquireOnMultiThreads" + val ratelimiter = DistributedFixedWindowRateLimiter(key, 5, Duration.ofSeconds(1), redisTemplate) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + clean(key) + } + + + companion object { + private const val KEY = "DistributedFixedKey" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiterTest.kt new file mode 100644 index 0000000000..4815debe65 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiterTest.kt @@ -0,0 +1,105 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.springframework.test.annotation.DirtiesContext +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DistributedLeakyRateLimiterTest : DistributedTest() { + + @Test + fun testTryAcquire() { + val key = KEY + "testTryAcquire" + val ratelimiter = DistributedLeakyRateLimiter(key, 5.0, 5, redisTemplate) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + println(System.currentTimeMillis()) + Thread.sleep(1000) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + println(System.currentTimeMillis()) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + println(System.currentTimeMillis()) + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed7) + Thread.sleep(1000) + val passed8 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed8) + clean(key) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val key = KEY + "testTryAcquireOnMultiThreads" + val ratelimiter = DistributedLeakyRateLimiter(key, 5.0, 5, redisTemplate) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + clean(key) + } + + companion object { + private const val KEY = "DistributedLeakyKey" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiterTest.kt new file mode 100644 index 0000000000..3bd57c3844 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiterTest.kt @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.springframework.test.annotation.DirtiesContext +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DistributedSlidingWindowRateLimiterTest : DistributedTest() { + @Test + fun testTryAcquire() { + val key = KEY + "testTryAcquire" + val ratelimiter = DistributedSlidingWindowRateLimiter(key, 5, Duration.ofSeconds(1), redisTemplate) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + var passed6 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed6) + Thread.sleep(1000) + var passed7 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed7) + val passed8 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed8) + clean(key) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val key = KEY + "testTryAcquireOnMultiThreads" + val ratelimiter = DistributedSlidingWindowRateLimiter(key, 5, Duration.ofSeconds(1), redisTemplate) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + clean(key) + } + + companion object { + private const val KEY = "DistributedSlidingKey" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTest.kt new file mode 100644 index 0000000000..ef38441481 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTest.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.redis.RedisAutoConfiguration +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.ImportAutoConfiguration +import org.springframework.boot.test.autoconfigure.data.redis.DataRedisTest +import org.springframework.data.redis.core.RedisTemplate + +@DataRedisTest +@ImportAutoConfiguration(RedisTestConfiguration::class, RedisAutoConfiguration::class) +open class DistributedTest : BaseRuleTest() { + + @Autowired + lateinit var redisTemplate: RedisTemplate + + fun clean(key: String) { + redisTemplate.delete(key) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiterTest.kt new file mode 100644 index 0000000000..e8efdd8ee0 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiterTest.kt @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.springframework.test.annotation.DirtiesContext +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DistributedTokenBucketRateLimiterTest : DistributedTest() { + @Test + fun testTryAcquire1() { + val key = KEY + "testTryAcquire1" + var ratelimiter = DistributedTokenBucketRateLimiter(key, 5.0, 5, redisTemplate) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + Thread.sleep(1000) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + println(System.currentTimeMillis()) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + println(System.currentTimeMillis()) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + println(System.currentTimeMillis()) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + println(System.currentTimeMillis()) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed7) + clean(key) + } + + + @Test + fun testTryAcquireOnMultiThreads() { + val key = KEY + "testTryAcquireOnMultiThreads" + var ratelimiter = DistributedTokenBucketRateLimiter(key, 5.0, 5, redisTemplate) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + clean(key) + } + + companion object { + private const val KEY = "DistributedTokenKey" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiterTest.kt new file mode 100644 index 0000000000..49487c7165 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiterTest.kt @@ -0,0 +1,106 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.google.common.base.Stopwatch +import com.google.common.base.Ticker +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.mockito.Mockito +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.random.Random +import kotlin.system.measureTimeMillis + +class FixedWindowRateLimiterTest { + + @Test + fun testTryAcquire() { + val ticker = Mockito.mock(Ticker::class.java) + Mockito.`when`(ticker.read()).thenReturn(0 * 1000 * 1000L) + val ratelimiter: RateLimiter = FixedWindowRateLimiter(5, Duration.ofSeconds(1), Stopwatch.createStarted(ticker)) + Mockito.`when`(ticker.read()).thenReturn(100 * 1000 * 1000L) + val passed1 = ratelimiter.tryAcquire(1) + assertTrue(passed1) + Mockito.`when`(ticker.read()).thenReturn(200 * 1000 * 1000L) + val passed2 = ratelimiter.tryAcquire(1) + assertTrue(passed2) + Mockito.`when`(ticker.read()).thenReturn(300 * 1000 * 1000L) + val passed3 = ratelimiter.tryAcquire(1) + assertTrue(passed3) + Mockito.`when`(ticker.read()).thenReturn(400 * 1000 * 1000L) + val passed4 = ratelimiter.tryAcquire(1) + assertTrue(passed4) + Mockito.`when`(ticker.read()).thenReturn(500 * 1000 * 1000L) + val passed5 = ratelimiter.tryAcquire(1) + assertTrue(passed5) + Mockito.`when`(ticker.read()).thenReturn(600 * 1000 * 1000L) + val passed6 = ratelimiter.tryAcquire(1) + assertFalse(passed6) + Mockito.`when`(ticker.read()).thenReturn(1001 * 1000 * 1000L) + val passed7 = ratelimiter.tryAcquire(1) + assertTrue(passed7) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val ticker = Mockito.mock(Ticker::class.java) + Mockito.`when`(ticker.read()).thenReturn(0 * 1000 * 1000L) + val ratelimiter: RateLimiter = FixedWindowRateLimiter(5, Duration.ofSeconds(1), Stopwatch.createStarted(ticker)) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + Thread.sleep((Random.nextInt(5) * 2).toLong()) + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiterTest.kt new file mode 100644 index 0000000000..77924e66d6 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiterTest.kt @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +class LeakyRateLimiterTest { + + @Test + fun testTryAcquire() { + val ratelimiter = LeakyRateLimiter(5.0, 5) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed6) + try { + Thread.sleep(1000) + } catch (ignore: InterruptedException) { + } + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed7) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val ratelimiter = LeakyRateLimiter(5.0, 5) + var successNum = 0 + var failedNum = 0 + var errorNun = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNun++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RedisTestConfiguration.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RedisTestConfiguration.kt new file mode 100644 index 0000000000..710c226a80 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RedisTestConfiguration.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import org.springframework.boot.test.context.TestConfiguration +import redis.embedded.RedisServer +import javax.annotation.PostConstruct +import javax.annotation.PreDestroy + + +@TestConfiguration +class RedisTestConfiguration { + private val redisServer = RedisServer.builder().build() + + @PostConstruct + fun postConstruct() { + redisServer.start() + } + + @PreDestroy + fun preDestroy() { + redisServer.stop() + } +} diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiterTest.kt new file mode 100644 index 0000000000..6d742545bb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiterTest.kt @@ -0,0 +1,92 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +class SlidingWindowRateLimiterTest { + + @Test + fun testTryAcquire() { + val ratelimiter = SlidingWindowRateLimiter(5, Duration.ofMillis(100)) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + Thread.sleep(1200) + var passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + var passed7 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed7) + val passed8 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed8) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val ratelimiter = SlidingWindowRateLimiter(5, Duration.ofMillis(100)) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiterTest.kt new file mode 100644 index 0000000000..96cea7389a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiterTest.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +class TokenBucketRateLimiterTest { + + @Test + fun testTryAcquire() { + var ratelimiter = TokenBucketRateLimiter(5.0) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + Thread.sleep(1000) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + println(System.currentTimeMillis()) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + println(System.currentTimeMillis()) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + println(System.currentTimeMillis()) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + println(System.currentTimeMillis()) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed7) + } + + + @Test + fun testTryAcquireOnMultiThreads() { + val ratelimiter = TokenBucketRateLimiter(5.0) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + Thread.sleep(1000) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChainTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChainTest.kt new file mode 100644 index 0000000000..f850ea4252 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChainTest.kt @@ -0,0 +1,151 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.repository.RateLimitRepository +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + + +class RateLimiterInterceptorChainTest { + + open class InterceptorA : RateLimiterInterceptor { + + override fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) { + list.add(identity() + ":before") + } + + + override fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) { + list.add(identity() + ":after") + } + + protected open fun identity(): String { + return InterceptorA::class.java.simpleName + } + } + + class InterceptorB : InterceptorA() { + override fun identity(): String { + return InterceptorB::class.java.simpleName + } + } + + class InterceptorC : InterceptorA() { + override fun identity(): String { + return InterceptorC::class.java.simpleName + } + } + + @BeforeEach + fun reInit() { + list.clear() + } + + @Test + fun testDoBeforeLimit() { + val chain = RateLimiterInterceptorChain() + chain.addInterceptor(InterceptorA()) + chain.addInterceptor(InterceptorB()) + chain.addInterceptor(InterceptorC()) + val resourceLimit = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + chain.doBeforeLimitCheck("test1", resourceLimit) + assertEquals(Companion.list.size, 3) + assertEquals(Companion.list[0], "InterceptorA:before") + assertEquals(Companion.list[1], "InterceptorB:before") + assertEquals(Companion.list[2], "InterceptorC:before") + } + + @Test + fun testTargetDoBeforeLimit() { + val chain = RateLimiterInterceptorChain() + val rateLimitRepository: RateLimitRepository = RateLimitRepository() + chain.addInterceptor(TargetRateLimiterInterceptorAdaptor(RateLimiterConfigService(rateLimitRepository))) + val resourceLimit = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name, + targets = listOf("127.0.0.2") + ) + assertThrows { + chain.doBeforeLimitCheck("test1", resourceLimit) + } + } + + @Test + fun testDoAfterLimit() { + val chain = RateLimiterInterceptorChain() + chain.addInterceptor(InterceptorC()) + chain.addInterceptor(InterceptorB()) + chain.addInterceptor(InterceptorA()) + chain.doAfterLimitCheck("test1", null, true, null) + assertEquals(Companion.list.size, 3) + assertEquals(Companion.list[0], "InterceptorC:after") + assertEquals(Companion.list[1], "InterceptorB:after") + assertEquals(Companion.list[2], "InterceptorA:after") + } + + @Test + fun testIsEmpty() { + val chain = RateLimiterInterceptorChain() + val isEmpty = chain.isEmpty() + Assertions.assertTrue(isEmpty) + } + + @Test + fun testClear() { + val chain = RateLimiterInterceptorChain() + chain.addInterceptor(InterceptorB()) + chain.addInterceptor(InterceptorA()) + chain.addInterceptor(InterceptorC()) + assertEquals(chain.size(), 3) + chain.clear() + assertEquals(chain.size(), 0) + } + + companion object { + val list: MutableList = ArrayList() + } +} diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/BaseRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/BaseRuleTest.kt new file mode 100644 index 0000000000..b309208bbf --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/BaseRuleTest.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions + +open class BaseRuleTest { + + fun assertEqualsLimitInfo(actualInfo: ResourceLimit?, expectedInfo: ResourceLimit?) { + Assertions.assertEquals(actualInfo?.limit, expectedInfo?.limit) + Assertions.assertEquals(actualInfo?.algo, expectedInfo?.algo) + Assertions.assertEquals(actualInfo?.limitDimension, expectedInfo?.limitDimension) + Assertions.assertEquals(actualInfo?.capacity, expectedInfo?.capacity) + Assertions.assertEquals(actualInfo?.resource, expectedInfo?.resource) + Assertions.assertEquals(actualInfo?.scope, expectedInfo?.scope) + Assertions.assertEquals(actualInfo?.duration, expectedInfo?.duration) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRuleTest.kt new file mode 100644 index 0000000000..26817c58e9 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class DownloadBandwidthRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val downloadBandwidthRateLimitRule = DownloadBandwidthRateLimitRule() + assertEquals(downloadBandwidthRateLimitRule.isEmpty(), true) + downloadBandwidthRateLimitRule.addRateLimitRule(l1) + assertEquals(downloadBandwidthRateLimitRule.isEmpty(), false) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRule() { + val downloadBandwidthRateLimitRule = DownloadBandwidthRateLimitRule() + downloadBandwidthRateLimitRule.addRateLimitRule(l1) + downloadBandwidthRateLimitRule.addRateLimitRule(l2) + downloadBandwidthRateLimitRule.addRateLimitRule(l3) + downloadBandwidthRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + downloadBandwidthRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + downloadBandwidthRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + downloadBandwidthRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = DownloadBandwidthRateLimitRule() + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = DownloadBandwidthRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = DownloadBandwidthRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l2) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = DownloadBandwidthRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testBandwidthRateLimitRuleWithInvalidLimitInfo() { + val rule = DownloadBandwidthRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testBandwidthRateLimitRuleWithInvalidUrl() { + val rule = DownloadBandwidthRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRuleTest.kt new file mode 100644 index 0000000000..33ac193447 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UploadBandwidthRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val uploadBandwidthRateLimitRule = UploadBandwidthRateLimitRule() + assertEquals(uploadBandwidthRateLimitRule.isEmpty(), true) + uploadBandwidthRateLimitRule.addRateLimitRule(l1) + assertEquals(uploadBandwidthRateLimitRule.isEmpty(), false) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRule() { + val uploadBandwidthRateLimitRule = UploadBandwidthRateLimitRule() + uploadBandwidthRateLimitRule.addRateLimitRule(l1) + uploadBandwidthRateLimitRule.addRateLimitRule(l2) + uploadBandwidthRateLimitRule.addRateLimitRule(l3) + uploadBandwidthRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + uploadBandwidthRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + uploadBandwidthRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + uploadBandwidthRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UploadBandwidthRateLimitRule() + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UploadBandwidthRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UploadBandwidthRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l2) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UploadBandwidthRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testBandwidthRateLimitRuleWithInvalidLimitInfo() { + val rule = UploadBandwidthRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testBandwidthRateLimitRuleWithInvalidUrl() { + val rule = UploadBandwidthRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRuleTest.kt new file mode 100644 index 0000000000..a043023d71 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRuleTest.kt @@ -0,0 +1,296 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UrlRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project2/*/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/{repo}}/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val urlRateLimitRule = UrlRateLimitRule() + assertEquals(urlRateLimitRule.isEmpty(), true) + urlRateLimitRule.addRateLimitRule(l1) + assertEquals(urlRateLimitRule.isEmpty(), false) + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRule() { + val urlRateLimitRule = UrlRateLimitRule() + urlRateLimitRule.addRateLimitRule(l1) + urlRateLimitRule.addRateLimitRule(l2) + urlRateLimitRule.addRateLimitRule(l3) + urlRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("///") + var actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertNull(actualInfo) + + resInfo = ResInfo("/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/") + + resInfo = ResInfo("/project1/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/") + + resInfo = ResInfo("/project2/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project2/") + + urlRateLimitRule.addRateLimitRule(l11) + resInfo = ResInfo("/project3/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + assertEquals(actualInfo?.resource, "/project3/") + + resInfo = ResInfo("/project1/repo1/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project3/repo3/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + assertEquals(actualInfo?.resource, "/project3/repo3/") + + urlRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + urlRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project3/repo1/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + assertEquals(actualInfo?.resource, "/project3/repo1/") + + resInfo = ResInfo("/project4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + resInfo = ResInfo("/project4/repo4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + urlRateLimitRule.addRateLimitRule(l7) + urlRateLimitRule.addRateLimitRule(l8) + urlRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("/project3/repo3/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + + resInfo = ResInfo("/project3/repo4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + assertEquals(actualInfo?.resource, "/project3/repo4/") + + resInfo = ResInfo("/project3/xxxx/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + assertEquals(actualInfo?.resource, "/project3/xxxx/") + + resInfo = ResInfo("/project3/1234/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + assertEquals(actualInfo?.resource, "/project3/1234/") + + urlRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("/project3/xxxx/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + assertEquals(actualInfo?.resource, "/project3/xxxx/") + + + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UrlRateLimitRule() + var resInfo = ResInfo("/project1/repo1/") + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UrlRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UrlRateLimitRule() + rule.addRateLimitRule(l5) + + val resInfo = ResInfo("/project1/repo1/") + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UrlRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project1/repo1/") + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUrlRateLimitRuleWithInvalidLimitInfo() { + val rule = UrlRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUrlRateLimitRuleWithInvalidUrl() { + val rule = UrlRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRuleTest.kt new file mode 100644 index 0000000000..f74009cf46 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import java.time.Duration +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class UrlRepoRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val urlRepoRateLimitRule = UrlRepoRateLimitRule() + assertEquals(urlRepoRateLimitRule.isEmpty(), true) + urlRepoRateLimitRule.addRateLimitRule(l1) + assertEquals(urlRepoRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRule() { + val urlRepoRateLimitRule = UrlRepoRateLimitRule() + urlRepoRateLimitRule.addRateLimitRule(l1) + urlRepoRateLimitRule.addRateLimitRule(l2) + urlRepoRateLimitRule.addRateLimitRule(l3) + urlRepoRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + urlRepoRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + urlRepoRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + urlRepoRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UrlRepoRateLimitRule() + var resInfo = ResInfo("/project1/repo1/") + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UrlRepoRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UrlRepoRateLimitRule() + rule.addRateLimitRule(l5) + + val resInfo = ResInfo("/project1/repo1/") + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UrlRepoRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project1/repo1/") + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUrlRateLimitRuleWithInvalidLimitInfo() { + val rule = UrlRepoRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserUrlRateLimitRuleWithInvalidUrl() { + val rule = UrlRepoRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRuleTest.kt new file mode 100644 index 0000000000..ae75f399a4 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRuleTest.kt @@ -0,0 +1,339 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UserUrlRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project2/*/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/*/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/repo3/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{repo}}/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l12 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l13 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l14 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val userUrlRateLimitRule = UserUrlRateLimitRule() + assertEquals(userUrlRateLimitRule.isEmpty(), true) + userUrlRateLimitRule.addRateLimitRule(l1) + assertEquals(userUrlRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRule() { + val userUrlRateLimitRule = UserUrlRateLimitRule() + userUrlRateLimitRule.addRateLimitRule(l13) + var resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:")) + var actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUrlRateLimitRule.addRateLimitRule(l14) + resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + userUrlRateLimitRule.addRateLimitRule(l1) + + resInfo = ResInfo("user1:///", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + resInfo = ResInfo("user1:/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/") + + + + userUrlRateLimitRule.addRateLimitRule(l2) + userUrlRateLimitRule.addRateLimitRule(l3) + userUrlRateLimitRule.addRateLimitRule(l4) + + resInfo = ResInfo("user1:/project1/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/") + + userUrlRateLimitRule.addRateLimitRule(l11) + resInfo = ResInfo("user1:/project3/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + userUrlRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userUrlRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project1/repo2/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo2/") + + resInfo = ResInfo("user1:/project2/repo2/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/repo2/") + + resInfo = ResInfo("user1:/project3/repo1/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo1/") + + resInfo = ResInfo("user1:/project4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userUrlRateLimitRule.addRateLimitRule(l7) + userUrlRateLimitRule.addRateLimitRule(l8) + userUrlRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + resInfo = ResInfo("user1:/project3/repo4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo4/") + + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + resInfo = ResInfo("user1:/project3/1234/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/1234/") + + userUrlRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + + resInfo = ResInfo("user2:/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/") + + resInfo = ResInfo("user2:/project3/xxxx/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/project3/xxxx/") + + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UserUrlRateLimitRule() + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + var info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + resInfo = ResInfo("user1:/", listOf("user1:")) + info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UserUrlRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf("user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + Assertions.assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UserUrlRateLimitRule() + rule.addRateLimitRule(l5) + + val resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UserUrlRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l12) + val resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l12) + } + + @Test + fun testUserUrlRateLimitRuleWithInvalidLimitInfo() { + val rule = UserUrlRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserUrlRateLimitRuleWithInvalidUrl() { + val rule = UserUrlRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url") + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRuleTest.kt new file mode 100644 index 0000000000..efdd708692 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRuleTest.kt @@ -0,0 +1,340 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class UserUrlRepoRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project2/*/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/*/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/repo3/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{repo}}/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l12 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l13 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l14 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val userUrlRepoRateLimitRule = UserUrlRepoRateLimitRule() + assertEquals(userUrlRepoRateLimitRule.isEmpty(), true) + userUrlRepoRateLimitRule.addRateLimitRule(l1) + assertEquals(userUrlRepoRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRule() { + val userUrlRepoRateLimitRule = UserUrlRepoRateLimitRule() + userUrlRepoRateLimitRule.addRateLimitRule(l13) + var resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + var actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUrlRepoRateLimitRule.addRateLimitRule(l14) + + resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + userUrlRepoRateLimitRule.addRateLimitRule(l1) + + resInfo = ResInfo("user1:///", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + resInfo = ResInfo("user1:/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/") + + + + userUrlRepoRateLimitRule.addRateLimitRule(l2) + userUrlRepoRateLimitRule.addRateLimitRule(l3) + userUrlRepoRateLimitRule.addRateLimitRule(l4) + + resInfo = ResInfo("user1:/project1/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUrlRepoRateLimitRule.addRateLimitRule(l11) + + resInfo = ResInfo("user1:/project3/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + userUrlRepoRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + userUrlRepoRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project1/repo2/", listOf("user1:/project1/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/repo2/", listOf("user1:/project2/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/repo2/") + + resInfo = ResInfo("user1:/project3/repo1/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project4/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userUrlRepoRateLimitRule.addRateLimitRule(l7) + userUrlRepoRateLimitRule.addRateLimitRule(l8) + userUrlRepoRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + resInfo = ResInfo("user1:/project3/repo4/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + resInfo = ResInfo("user1:/project3/1234/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/1234/") + + userUrlRepoRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + + resInfo = ResInfo("user2:/", listOf("user2:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/") + + resInfo = ResInfo("user2:/project3/xxxx/", listOf("user2:/project3/", "user2:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user2:") + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UserUrlRepoRateLimitRule() + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + var info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + resInfo = ResInfo("user1:/", listOf("user1:")) + info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UserUrlRepoRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf("user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + Assertions.assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UserUrlRepoRateLimitRule() + rule.addRateLimitRule(l5) + + val resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UserUrlRepoRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l12) + val resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l12) + } + + @Test + fun testUserUrlRepoRateLimitRuleWithInvalidLimitInfo() { + val rule = UserUrlRepoRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserUrlRepoRateLimitRuleWithInvalidUrl() { + val rule = UserUrlRepoRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url") + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRuleTest.kt new file mode 100644 index 0000000000..04aac62941 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class DownloadUsageRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val downloadUsageRateLimitRule = DownloadUsageRateLimitRule() + assertEquals(downloadUsageRateLimitRule.isEmpty(), true) + downloadUsageRateLimitRule.addRateLimitRule(l1) + assertEquals(downloadUsageRateLimitRule.isEmpty(), false) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRule() { + val downloadUsageRateLimitRule = DownloadUsageRateLimitRule() + downloadUsageRateLimitRule.addRateLimitRule(l1) + downloadUsageRateLimitRule.addRateLimitRule(l2) + downloadUsageRateLimitRule.addRateLimitRule(l3) + downloadUsageRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + downloadUsageRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + downloadUsageRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + downloadUsageRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = DownloadUsageRateLimitRule() + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = DownloadUsageRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = DownloadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l2) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = DownloadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUsageRateLimitRuleWithInvalidLimitInfo() { + val rule = DownloadUsageRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUsageRateLimitRuleWithInvalidUrl() { + val rule = DownloadUsageRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRuleTest.kt new file mode 100644 index 0000000000..8d634f519b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UploadUsageRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val uploadUsageRateLimitRule = UploadUsageRateLimitRule() + assertEquals(uploadUsageRateLimitRule.isEmpty(), true) + uploadUsageRateLimitRule.addRateLimitRule(l1) + assertEquals(uploadUsageRateLimitRule.isEmpty(), false) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRule() { + val uploadUsageRateLimitRule = UploadUsageRateLimitRule() + uploadUsageRateLimitRule.addRateLimitRule(l1) + uploadUsageRateLimitRule.addRateLimitRule(l2) + uploadUsageRateLimitRule.addRateLimitRule(l3) + uploadUsageRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + uploadUsageRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + uploadUsageRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + uploadUsageRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UploadUsageRateLimitRule() + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UploadUsageRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UploadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l2) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UploadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUsageRateLimitRuleWithInvalidLimitInfo() { + val rule = UploadUsageRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUsageRateLimitRuleWithInvalidUrl() { + val rule = UploadUsageRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRuleTest.kt new file mode 100644 index 0000000000..5ec3ae822b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRuleTest.kt @@ -0,0 +1,340 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UserDownloadUsageRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project2/*/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/*/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/repo3/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{repo}}/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l12 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l13 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l14 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val userDownloadUsageRateLimitRule = UserDownloadUsageRateLimitRule() + assertEquals(userDownloadUsageRateLimitRule.isEmpty(), true) + userDownloadUsageRateLimitRule.addRateLimitRule(l1) + assertEquals(userDownloadUsageRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRule() { + val userDownloadUsageRateLimitRule = UserDownloadUsageRateLimitRule() + userDownloadUsageRateLimitRule.addRateLimitRule(l13) + var resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + var actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userDownloadUsageRateLimitRule.addRateLimitRule(l14) + + resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + userDownloadUsageRateLimitRule.addRateLimitRule(l1) + + resInfo = ResInfo("user1:///", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + resInfo = ResInfo("user1:/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/") + + + + userDownloadUsageRateLimitRule.addRateLimitRule(l2) + userDownloadUsageRateLimitRule.addRateLimitRule(l3) + userDownloadUsageRateLimitRule.addRateLimitRule(l4) + + resInfo = ResInfo("user1:/project1/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userDownloadUsageRateLimitRule.addRateLimitRule(l11) + + resInfo = ResInfo("user1:/project3/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + userDownloadUsageRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + userDownloadUsageRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project1/repo2/", listOf("user1:/project1/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/repo2/", listOf("user1:/project2/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/repo2/") + + resInfo = ResInfo("user1:/project3/repo1/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project4/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userDownloadUsageRateLimitRule.addRateLimitRule(l7) + userDownloadUsageRateLimitRule.addRateLimitRule(l8) + userDownloadUsageRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + resInfo = ResInfo("user1:/project3/repo4/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + resInfo = ResInfo("user1:/project3/1234/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/1234/") + + userDownloadUsageRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + + resInfo = ResInfo("user2:/", listOf("user2:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/") + + resInfo = ResInfo("user2:/project3/xxxx/", listOf("user2:/project3/", "user2:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user2:") + + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UserDownloadUsageRateLimitRule() + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + var info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + resInfo = ResInfo("user1:/", listOf("user1:")) + info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UserDownloadUsageRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf("")) + assertThrows { rule.getRateLimitRule(resInfo) } + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UserDownloadUsageRateLimitRule() + rule.addRateLimitRule(l5) + + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UserDownloadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l12) + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l12) + } + + @Test + fun testUserDownloadUsageRateLimitRuleWithInvalidLimitInfo() { + val rule = UserDownloadUsageRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserDownloadUsageRateLimitRuleWithInvalidUrl() { + val rule = UserDownloadUsageRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url") + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRuleTest.kt new file mode 100644 index 0000000000..65fa3d0723 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRuleTest.kt @@ -0,0 +1,340 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UserUploadUsageRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project2/*/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/*/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/repo3/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{repo}}/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l12 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l13 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l14 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val userUploadUsageRateLimitRule = UserUploadUsageRateLimitRule() + assertEquals(userUploadUsageRateLimitRule.isEmpty(), true) + userUploadUsageRateLimitRule.addRateLimitRule(l1) + assertEquals(userUploadUsageRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRule() { + val userUploadUsageRateLimitRule = UserUploadUsageRateLimitRule() + userUploadUsageRateLimitRule.addRateLimitRule(l13) + var resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + var actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUploadUsageRateLimitRule.addRateLimitRule(l14) + + resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + userUploadUsageRateLimitRule.addRateLimitRule(l1) + + resInfo = ResInfo("user1:///", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + resInfo = ResInfo("user1:/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/") + + + + userUploadUsageRateLimitRule.addRateLimitRule(l2) + userUploadUsageRateLimitRule.addRateLimitRule(l3) + userUploadUsageRateLimitRule.addRateLimitRule(l4) + + resInfo = ResInfo("user1:/project1/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUploadUsageRateLimitRule.addRateLimitRule(l11) + + resInfo = ResInfo("user1:/project3/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + userUploadUsageRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + userUploadUsageRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project1/repo2/", listOf("user1:/project1/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/repo2/", listOf("user1:/project2/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/repo2/") + + resInfo = ResInfo("user1:/project3/repo1/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project4/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userUploadUsageRateLimitRule.addRateLimitRule(l7) + userUploadUsageRateLimitRule.addRateLimitRule(l8) + userUploadUsageRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + resInfo = ResInfo("user1:/project3/repo4/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + resInfo = ResInfo("user1:/project3/1234/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/1234/") + + userUploadUsageRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + + resInfo = ResInfo("user2:/", listOf("user2:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/") + + resInfo = ResInfo("user2:/project3/xxxx/", listOf("user2:/project3/", "user2:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user2:") + + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UserUploadUsageRateLimitRule() + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + var info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + resInfo = ResInfo("user1:/", listOf("user1:")) + info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UserUploadUsageRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf("")) + assertThrows { rule.getRateLimitRule(resInfo) } + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UserUploadUsageRateLimitRule() + rule.addRateLimitRule(l5) + + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UserUploadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l12) + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l12) + } + + @Test + fun testUserUploadUsageRateLimitRuleWithInvalidLimitInfo() { + val rule = UserUploadUsageRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserUploadUsageRateLimitRuleWithInvalidUrl() { + val rule = UserUploadUsageRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url") + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterServiceTest.kt new file mode 100644 index 0000000000..090da664cf --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterServiceTest.kt @@ -0,0 +1,275 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedFixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedLeakyRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedSlidingWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedTest +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedTokenBucketRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.FixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.LeakyRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.SlidingWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.TokenBucketRateLimiter +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.repository.RateLimitRepository +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import io.micrometer.core.instrument.MeterRegistry +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.TestInstance +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.mock.web.MockHttpServletRequest +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import java.time.Duration + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +open class AbstractRateLimiterServiceTest : DistributedTest() { + + lateinit var request: MockHttpServletRequest + lateinit var rateLimiterService: RateLimiterService + lateinit var rateLimiterProperties: RateLimiterProperties + lateinit var taskScheduler: ThreadPoolTaskScheduler + lateinit var rateLimiterMetrics: RateLimiterMetrics + lateinit var rateLimiterConfigService: RateLimiterConfigService + + @MockBean + private lateinit var meterRegistry: MeterRegistry + + @MockBean + private lateinit var rateLimitRepository: RateLimitRepository + + fun init() { + request = MockHttpServletRequest() + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + taskScheduler = scheduler + rateLimiterProperties = RateLimiterProperties() + rateLimiterMetrics = RateLimiterMetrics(meterRegistry) + rateLimiterConfigService = RateLimiterConfigService(rateLimitRepository) + } + + + open fun createAlgorithmOfRateLimiterTest() { + val resource = (rateLimiterService as AbstractRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as AbstractRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as AbstractRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + // 测试固定窗口本地算法生成 + var rateLimiter = (rateLimiterService as AbstractRateLimiterService) + .createAlgorithmOfRateLimiter(resource, resourceLimit!!.resourceLimit) + Assertions.assertInstanceOf( + FixedWindowRateLimiter::class.java, + rateLimiter + ) + // 测试固定窗口分布式算法生成 + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/", + limitDimension = LimitDimension.URL.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.GLOBAL.name + ) + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l1) + Assertions.assertInstanceOf( + DistributedFixedWindowRateLimiter::class.java, + rateLimiter + ) + // 测试limit < 0的场景 + val l2 = ResourceLimit( + algo = Algorithms.SLIDING_WINDOW.name, resource = "/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.URL.name, limit = -1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + Assertions.assertThrows(InvalidResourceException::class.java) { + (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l2) + } + // 测试滑动窗口本地算法生成 + l2.limit = 1 + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l2) + Assertions.assertInstanceOf( + SlidingWindowRateLimiter::class.java, + rateLimiter + ) + // 测试滑动窗口分布式算法生成 + l2.scope = WorkScope.GLOBAL.name + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l2) + Assertions.assertInstanceOf( + DistributedSlidingWindowRateLimiter::class.java, + rateLimiter + ) + // 测试不支持的算法类型 + val l3 = ResourceLimit( + algo = "", resource = "/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + Assertions.assertThrows(InvalidResourceException::class.java) { + (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l3) + } + //测试 capacity 不合规的场景 + val l4 = ResourceLimit( + algo = Algorithms.LEAKY_BUCKET.name, resource = "/project3/{repo}}/", + limitDimension = LimitDimension.URL.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + Assertions.assertThrows(InvalidResourceException::class.java) { + (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l4) + } + l4.capacity = -2 + Assertions.assertThrows(InvalidResourceException::class.java) { + (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l4) + } + // 测试漏桶本地算法生成 + l4.capacity = 5 + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l4) + Assertions.assertInstanceOf( + LeakyRateLimiter::class.java, + rateLimiter + ) + // 测试漏桶分布式算法生成 + l4.scope = WorkScope.GLOBAL.name + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l4) + Assertions.assertInstanceOf( + DistributedLeakyRateLimiter::class.java, + rateLimiter + ) + // 测试令牌桶本地算法生成 + val l5 = ResourceLimit( + algo = Algorithms.TOKEN_BUCKET.name, resource = "/project3/", + limitDimension = LimitDimension.URL.name, limit = 1, capacity = 5, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l5) + Assertions.assertInstanceOf( + TokenBucketRateLimiter::class.java, + rateLimiter + ) + // 测试令牌桶分布式算法生成 + l5.scope = WorkScope.GLOBAL.name + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l5) + Assertions.assertInstanceOf( + DistributedTokenBucketRateLimiter::class.java, + rateLimiter + ) + } + + + open fun refreshRateLimitRuleTest() { + val hashCode = rateLimiterProperties.rules.hashCode() + (rateLimiterService as AbstractRateLimiterService).refreshRateLimitRule() + Assertions.assertEquals(hashCode, (rateLimiterService as AbstractRateLimiterService).currentRuleHashCode) + } + + + open fun getAlgorithmOfRateLimiterTest() { + val resource = (rateLimiterService as AbstractRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as AbstractRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as AbstractRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + assertEqualsLimitInfo(resourceLimit!!.resourceLimit, rateLimiterProperties.rules.first()) + val rateLimiter = (rateLimiterService as AbstractRateLimiterService) + .getAlgorithmOfRateLimiter(resource, resourceLimit.resourceLimit) + Assertions.assertInstanceOf( + FixedWindowRateLimiter::class.java, + rateLimiter + ) + Assertions.assertEquals( + rateLimiter, + (rateLimiterService as AbstractRateLimiterService) + .getAlgorithmOfRateLimiter(resource, resourceLimit.resourceLimit) + ) + } + + + open fun getResLimitInfoTest() { + val resourceLimit = (rateLimiterService as AbstractRateLimiterService).getResLimitInfo(request) + Assertions.assertNotNull(resourceLimit) + assertEqualsLimitInfo(resourceLimit!!.resourceLimit, rateLimiterProperties.rules.first()) + } + + + open fun circuitBreakerCheckTest() { + + var circuitBreakerPerSecond: Long? = null + (rateLimiterService as AbstractRateLimiterService).circuitBreakerCheck( + rateLimiterProperties.rules.first(), circuitBreakerPerSecond + ) + + circuitBreakerPerSecond = Long.MAX_VALUE + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as AbstractRateLimiterService).circuitBreakerCheck( + rateLimiterProperties.rules.first(), circuitBreakerPerSecond + ) + } + circuitBreakerPerSecond = Long.MIN_VALUE + (rateLimiterService as AbstractRateLimiterService).circuitBreakerCheck( + rateLimiterProperties.rules.first(), circuitBreakerPerSecond + ) + } + + + open fun rateLimitCatchTest() { + val resourceLimit = (rateLimiterService as AbstractRateLimiterService).getResLimitInfo(request) + Assertions.assertNotNull(resourceLimit) + val rateLimiter = (rateLimiterService as AbstractRateLimiterService) + .getAlgorithmOfRateLimiter(resourceLimit!!.resource, resourceLimit.resourceLimit) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as AbstractRateLimiterService).rateLimitCatch( + request = request, + resLimitInfo = resourceLimit, + applyPermits = Long.MAX_VALUE, + circuitBreakerPerSecond = null, + ) { rateLimiter, permits -> + false + } + } + rateLimiterProperties.dryRun = true + (rateLimiterService as AbstractRateLimiterService).rateLimitCatch( + request = request, + resLimitInfo = resourceLimit, + applyPermits = Long.MAX_VALUE, + circuitBreakerPerSecond = null, + ) { rateLimiter, permits -> + false + } + rateLimiterProperties.dryRun = false + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterServiceTest.kt new file mode 100644 index 0000000000..c3126425fc --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterServiceTest.kt @@ -0,0 +1,239 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.DownloadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.servlet.HandlerMapping + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DownloadBandwidthRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.method = "GET" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = DownloadBandwidthRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals( + false, + (rateLimiterService as DownloadBandwidthRateLimiterService).ignoreRequest(request) + ) + request.method = "POST" + Assertions.assertEquals( + true, + (rateLimiterService as DownloadBandwidthRateLimiterService).ignoreRequest(request) + ) + request.method = "PATCH" + Assertions.assertEquals( + true, + (rateLimiterService as DownloadBandwidthRateLimiterService).ignoreRequest(request) + ) + request.method = "PUT" + Assertions.assertEquals( + true, + (rateLimiterService as DownloadBandwidthRateLimiterService).ignoreRequest(request) + ) + request.method = "GET" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = + (rateLimiterService as DownloadBandwidthRateLimiterService).getRepoInfoFromAttribute(request) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as DownloadBandwidthRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as DownloadBandwidthRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertThrows(AcquireLockFailedException::class.java) { + (rateLimiterService as DownloadBandwidthRateLimiterService).getApplyPermits(request, null) + } + Assertions.assertEquals( + 10, + (rateLimiterService as DownloadBandwidthRateLimiterService).getApplyPermits(request, 10) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + DownloadBandwidthRateLimitRule::class.java, + (rateLimiterService as DownloadBandwidthRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.DOWNLOAD_BANDWIDTH.name), + (rateLimiterService as DownloadBandwidthRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as DownloadBandwidthRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as DownloadBandwidthRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as DownloadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "DownloadBandwidth:" + "/blueking/", + (rateLimiterService as DownloadBandwidthRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as DownloadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "DownloadBandwidth:" + "/blueking/generic-local/", + (rateLimiterService as DownloadBandwidthRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as DownloadBandwidthRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as DownloadBandwidthRateLimiterService).buildExtraResource(request) + ) + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + val resourceLimit = (rateLimiterService as DownloadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + Assertions.assertThrows(UnsupportedOperationException::class.java) { rateLimiterService.limit(request) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterServiceTest.kt new file mode 100644 index 0000000000..108c7eb237 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterServiceTest.kt @@ -0,0 +1,295 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.bandwidth + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.UploadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.util.unit.DataSize +import org.springframework.web.servlet.HandlerMapping + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UploadBandwidthRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + val content = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + request.setContent(content) + request.method = "PUT" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UploadBandwidthRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UploadBandwidthRateLimiterService).ignoreRequest(request)) + request.method = "POST" + Assertions.assertEquals(false, (rateLimiterService as UploadBandwidthRateLimiterService).ignoreRequest(request)) + request.method = "PATCH" + Assertions.assertEquals(false, (rateLimiterService as UploadBandwidthRateLimiterService).ignoreRequest(request)) + request.method = "GET" + Assertions.assertEquals(true, (rateLimiterService as UploadBandwidthRateLimiterService).ignoreRequest(request)) + request.method = "PUT" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = (rateLimiterService as UploadBandwidthRateLimiterService).getRepoInfoFromAttribute( + request + ) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as UploadBandwidthRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as UploadBandwidthRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertThrows(AcquireLockFailedException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).getApplyPermits(request, null) + } + Assertions.assertEquals( + 10, + (rateLimiterService as UploadBandwidthRateLimiterService).getApplyPermits(request, 10) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UploadBandwidthRateLimitRule::class.java, + (rateLimiterService as UploadBandwidthRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.UPLOAD_BANDWIDTH.name), + (rateLimiterService as UploadBandwidthRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UploadBandwidthRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UploadBandwidthRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UploadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UploadBandwidth:" + "/blueking/", + (rateLimiterService as UploadBandwidthRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UploadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UploadBandwidth:" + "/blueking/generic-local/", + (rateLimiterService as UploadBandwidthRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as UploadBandwidthRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UploadBandwidthRateLimiterService).buildExtraResource(request) + ) + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + val resourceLimit = (rateLimiterService as UploadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + Assertions.assertThrows(UnsupportedOperationException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).limit(request) + } + } + + @Test + fun bandwidthRateLimitTest() { + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateLimit( + request = request, + permits = 1, + circuitBreakerPerSecond = DataSize.ofBytes(0) + ) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateLimit( + request = request, + permits = 10000, + circuitBreakerPerSecond = DataSize.ofBytes(0) + ) + } + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateLimit( + request = request, + permits = 1, + circuitBreakerPerSecond = DataSize.ofTerabytes(1) + ) + } + } + + @Test + fun bandwidthRateLimit1Test() { + val content = "1234567891" + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateStart( + request = request, + inputStream = content.byteInputStream(), + circuitBreakerPerSecond = DataSize.ofTerabytes(1), + rangeLength = null + ) + } + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateStart( + request = request, + inputStream = content.byteInputStream(), + circuitBreakerPerSecond = DataSize.ofBytes(1), + rangeLength = null + ) + } + + @Test + fun bandwidthLimitHandlerTest() { + val resourceLimit = (rateLimiterService as UploadBandwidthRateLimiterService).getResLimitInfo(request) + Assertions.assertNotNull(resourceLimit) + assertEqualsLimitInfo(resourceLimit!!.resourceLimit, rateLimiterProperties.rules.first()) + val rateLimiter = (rateLimiterService as UploadBandwidthRateLimiterService) + .getAlgorithmOfRateLimiter(resourceLimit.resource, resourceLimit.resourceLimit) + Assertions.assertEquals( + true, + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthLimitHandler(rateLimiter, 1) + ) + Assertions.assertEquals( + false, + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthLimitHandler(rateLimiter, 100) + ) + } + + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterServiceTest.kt new file mode 100644 index 0000000000..ec3baac926 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterServiceTest.kt @@ -0,0 +1,215 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UrlRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/", + limitDimension = LimitDimension.URL.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UrlRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UrlRateLimiterService).ignoreRequest(request)) + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/test.txt", + (rateLimiterService as UrlRateLimiterService).buildResource(request) + ) + request.requestURI = "/api/node/batch/blueking/generic-local" + Assertions.assertEquals( + "/api/node/batch/blueking/generic-local", + (rateLimiterService as UrlRateLimiterService).buildResource(request) + ) + request.requestURI = "/blueking/generic-local/test.txt" + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + emptyList(), + (rateLimiterService as UrlRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals(1, (rateLimiterService as UrlRateLimiterService).getApplyPermits(request, null)) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UrlRateLimitRule::class.java, + (rateLimiterService as UrlRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.URL.name), + (rateLimiterService as UrlRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UrlRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UrlRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as UrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "Url:" + "/blueking/generic-local/test.txt", + (rateLimiterService as UrlRateLimiterService).generateKey( + resourceLimit!!.resource, resourceLimit.resourceLimit + ) + ) + } + + @Test + fun generateKeyTestNull() { + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + + val resource = (rateLimiterService as UrlRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UrlRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as UrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRateLimiterService).limit(request) + } + + l1.resource = "/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UrlRateLimiterService).limit(request) + (rateLimiterService as UrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRateLimiterService).limit(request) + } + } + +} diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterServiceTest.kt new file mode 100644 index 0000000000..80af9fbb42 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterServiceTest.kt @@ -0,0 +1,287 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import com.tencent.bkrepo.repository.pojo.repo.UserRepoCreateRequest +import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.servlet.HandlerMapping +import org.springframework.web.servlet.HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UrlRepoRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.URL_REPO.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.setAttribute(BEST_MATCHING_PATTERN_ATTRIBUTE, "/{projectId}/{repoName}/**") + request.contentType = "application/json" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UrlRepoRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + rateLimiterProperties.specialUrls = listOf("*") + Assertions.assertEquals(false, (rateLimiterService as UrlRepoRateLimiterService).ignoreRequest(request)) + rateLimiterProperties.specialUrls = listOf("/{projectId}/{repoName}/**") + Assertions.assertEquals(false, (rateLimiterService as UrlRepoRateLimiterService).ignoreRequest(request)) + rateLimiterProperties.specialUrls = listOf() + Assertions.assertEquals(true, (rateLimiterService as UrlRepoRateLimiterService).ignoreRequest(request)) + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as UrlRepoRateLimiterService).buildResource(request) + ) + request.requestURI = "/api/node/batch/blueking/generic-local" + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as UrlRepoRateLimiterService).buildResource(request) + ) + request.requestURI = "/blueking/generic-local/test.txt" + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as UrlRepoRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals(1, (rateLimiterService as UrlRepoRateLimiterService).getApplyPermits(request, null)) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UrlRepoRateLimitRule::class.java, + (rateLimiterService as UrlRepoRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.URL_REPO.name), + (rateLimiterService as UrlRepoRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UrlRepoRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UrlRepoRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as UrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UrlRepo:" + "/blueking/", + (rateLimiterService as UrlRepoRateLimiterService).generateKey( + resourceLimit!!.resource, resourceLimit.resourceLimit + ) + ) + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = (rateLimiterService as UrlRepoRateLimiterService).getRepoInfoFromAttribute( + request + ) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + } + + @Test + fun getRepoInfoFromBodyTest() { + request.requestURI = "/api/node/search" + request.contentType = "application/json" + val queryModelBuilder = NodeQueryBuilder() + .select(PROJECT_ID, REPO_NAME) + .sortByAsc("fullPath") + .page(1, 10) + .projectId("test-projectId") + .repoName("test-repoName") + + val queryModel = queryModelBuilder.build() + request.setContent(queryModel.toJsonString().toByteArray()) + val (projectId, repoName) = (rateLimiterService as UrlRepoRateLimiterService).getRepoInfoFromBody( + request + ) + Assertions.assertEquals("test-projectId", projectId) + Assertions.assertEquals("test-repoName", repoName) + + request.setContent( + UserRepoCreateRequest( + projectId = "test-projectId1", + name = "test-repoName1", + type = RepositoryType.GENERIC, + category = RepositoryCategory.COMPOSITE, + display = false + ).toJsonString().toByteArray() + ) + val (projectId1, repoName1) = (rateLimiterService as UrlRepoRateLimiterService).getRepoInfoFromBody( + request + ) + Assertions.assertEquals("test-projectId1", projectId1) + Assertions.assertEquals(null, repoName1) + + request.requestURI = "/blueking/generic-local/test.txt" + } + + @Test + fun generateKeyTestNull() { + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + + val resource = (rateLimiterService as UrlRepoRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UrlRepoRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as UrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + rateLimiterProperties.specialUrls = listOf("*") + // 本地限流验证 + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + } + + l1.resource = "/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + } + + l1.resource = "/*/" + l1.limit = 1 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + } + +} diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterServiceTest.kt new file mode 100644 index 0000000000..fe89d23cbb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterServiceTest.kt @@ -0,0 +1,263 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url.user + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UserUrlRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_URL.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + val attributes = ServletRequestAttributes(request) + RequestContextHolder.setRequestAttributes(attributes) + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UserUrlRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UserUrlRateLimiterService).ignoreRequest(request)) + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "admin:/blueking/generic-local/test.txt", + (rateLimiterService as UserUrlRateLimiterService).buildResource(request) + ) + request.requestURI = "/api/node/batch/blueking/generic-local" + request.setAttribute("userId", "test") + Assertions.assertEquals( + "test:/api/node/batch/blueking/generic-local", + (rateLimiterService as UserUrlRateLimiterService).buildResource(request) + ) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("admin:"), + (rateLimiterService as UserUrlRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals( + 1, + (rateLimiterService as UserUrlRateLimiterService).getApplyPermits(request, null) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UserUrlRateLimitRule::class.java, + (rateLimiterService as UserUrlRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.USER_URL.name), + (rateLimiterService as UserUrlRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UserUrlRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUrlRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UserUrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUrl:" + "admin:/blueking/generic-local/test.txt", + (rateLimiterService as UserUrlRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "admin:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + + resourceLimit = (rateLimiterService as UserUrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUrl:" + "admin:", + (rateLimiterService as UserUrlRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "*:/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + l1.resource = "test:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + + val resource = (rateLimiterService as UserUrlRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUrlRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UserUrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "test:/blueking/generic-local/test.txt" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + + l1.resource = "*:/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRateLimiterService).limit(request) + } + + l1.resource = "admin:/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRateLimiterService).limit(request) + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRateLimiterService).limit(request) + } + + l1.resource = "admin:" + l1.limit = 1 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRateLimiterService).limit(request) + } + + l1.resource = "test:/blueking" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRateLimiterService).limit(request) + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertDoesNotThrow { (rateLimiterService as UserUrlRateLimiterService).limit(request) } + } + + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterServiceTest.kt new file mode 100644 index 0000000000..74fa8ccfd0 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterServiceTest.kt @@ -0,0 +1,337 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url.user + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import com.tencent.bkrepo.repository.pojo.repo.UserRepoCreateRequest +import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import org.springframework.web.servlet.HandlerMapping +import org.springframework.web.servlet.HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UserUrlRepoRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/*/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.setAttribute(BEST_MATCHING_PATTERN_ATTRIBUTE, "/{projectId}/{repoName}/**") + val attributes = ServletRequestAttributes(request) + RequestContextHolder.setRequestAttributes(attributes) + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UserUrlRepoRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + rateLimiterProperties.specialUrls = listOf("*") + Assertions.assertEquals(false, (rateLimiterService as UserUrlRepoRateLimiterService).ignoreRequest(request)) + rateLimiterProperties.specialUrls = listOf("/{projectId}/{repoName}/**") + Assertions.assertEquals(false, (rateLimiterService as UserUrlRepoRateLimiterService).ignoreRequest(request)) + rateLimiterProperties.specialUrls = listOf() + Assertions.assertEquals(true, (rateLimiterService as UserUrlRepoRateLimiterService).ignoreRequest(request)) + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "admin:/blueking/generic-local/", + (rateLimiterService as UserUrlRepoRateLimiterService).buildResource(request) + ) + request.requestURI = "/api/node/batch/blueking/generic-local" + request.setAttribute("userId", "test") + Assertions.assertEquals( + "test:/blueking/generic-local/", + (rateLimiterService as UserUrlRepoRateLimiterService).buildResource(request) + ) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("admin:/blueking/", "admin:"), + (rateLimiterService as UserUrlRepoRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals( + 1, + (rateLimiterService as UserUrlRepoRateLimiterService).getApplyPermits(request, null) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UserUrlRepoRateLimitRule::class.java, + (rateLimiterService as UserUrlRepoRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.USER_URL_REPO.name), + (rateLimiterService as UserUrlRepoRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UserUrlRepoRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUrlRepoRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = + (rateLimiterService as UserUrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUrlRepo:" + "admin:/blueking/", + (rateLimiterService as UserUrlRepoRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "admin:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + + resourceLimit = (rateLimiterService as UserUrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUrlRepo:" + "admin:", + (rateLimiterService as UserUrlRepoRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + } + + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = (rateLimiterService as UserUrlRepoRateLimiterService).getRepoInfoFromAttribute( + request + ) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + } + + @Test + fun getRepoInfoFromBodyTest() { + request.requestURI = "/api/node/search" + request.contentType = "application/json" + val queryModelBuilder = NodeQueryBuilder() + .select(PROJECT_ID, REPO_NAME) + .sortByAsc("fullPath") + .page(1, 10) + .projectId("test-projectId") + .repoName("test-repoName") + + val queryModel = queryModelBuilder.build() + request.setContent(queryModel.toJsonString().toByteArray()) + val (projectId, repoName) = (rateLimiterService as UserUrlRepoRateLimiterService).getRepoInfoFromBody( + request + ) + Assertions.assertEquals("test-projectId", projectId) + Assertions.assertEquals("test-repoName", repoName) + + request.setContent( + UserRepoCreateRequest( + projectId = "test-projectId1", + name = "test-repoName1", + type = RepositoryType.GENERIC, + category = RepositoryCategory.COMPOSITE, + display = false + ).toJsonString().toByteArray() + ) + val (projectId1, repoName1) = (rateLimiterService as UserUrlRepoRateLimiterService).getRepoInfoFromBody( + request + ) + Assertions.assertEquals("test-projectId1", projectId1) + Assertions.assertEquals(null, repoName1) + + request.requestURI = "/blueking/generic-local/test.txt" + } + + @Test + fun generateKeyTestNull() { + l1.resource = "test:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + + val resource = (rateLimiterService as UserUrlRepoRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUrlRepoRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = + (rateLimiterService as UserUrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "test:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + rateLimiterProperties.specialUrls = listOf("*") + // 本地限流验证 + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + } + + l1.resource = "admin:/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + } + + l1.resource = "admin:" + l1.limit = 1 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + } + + l1.resource = "test:/blueking" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertDoesNotThrow { (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) } + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + } + + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterServiceTest.kt new file mode 100644 index 0000000000..16cdf93eb1 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterServiceTest.kt @@ -0,0 +1,262 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.DownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.servlet.HandlerMapping +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DownloadUsageRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.method = "GET" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = DownloadUsageRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "POST" + Assertions.assertEquals(true, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PATCH" + Assertions.assertEquals(true, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PUT" + Assertions.assertEquals(true, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "HEAD" + Assertions.assertEquals(true, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "GET" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = + (rateLimiterService as DownloadUsageRateLimiterService).getRepoInfoFromAttribute(request) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as DownloadUsageRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as DownloadUsageRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertThrows(AcquireLockFailedException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).getApplyPermits(request, null) + } + + Assertions.assertEquals( + 10, + (rateLimiterService as DownloadUsageRateLimiterService).getApplyPermits(request, 10) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + DownloadUsageRateLimitRule::class.java, + (rateLimiterService as DownloadUsageRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.DOWNLOAD_USAGE.name), + (rateLimiterService as DownloadUsageRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as DownloadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as DownloadUsageRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as DownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "DownloadUsage:" + "/blueking/", + (rateLimiterService as DownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as DownloadUsageRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "DownloadUsage:" + "/blueking/generic-local/", + (rateLimiterService as DownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as DownloadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as DownloadUsageRateLimiterService).buildExtraResource(request) + ) + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + val resourceLimit = (rateLimiterService as DownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "/blueking/" + l1.limit = 20 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + } + } + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterServiceTest.kt new file mode 100644 index 0000000000..96e2e9a3bb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterServiceTest.kt @@ -0,0 +1,263 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.UploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.servlet.HandlerMapping +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UploadUsageRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + val content = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + request.setContent(content) + request.method = "PUT" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UploadUsageRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "POST" + Assertions.assertEquals(false, (rateLimiterService as UploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PATCH" + Assertions.assertEquals(false, (rateLimiterService as UploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "GET" + Assertions.assertEquals(true, (rateLimiterService as UploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PUT" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = + (rateLimiterService as UploadUsageRateLimiterService).getRepoInfoFromAttribute(request) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as UploadUsageRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as UploadUsageRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals( + 10, + (rateLimiterService as UploadUsageRateLimiterService).getApplyPermits(request, null) + ) + request.method = "GET" + Assertions.assertEquals( + 0, + (rateLimiterService as UploadUsageRateLimiterService).getApplyPermits(request, null) + ) + request.method = "PUT" + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UploadUsageRateLimitRule::class.java, + (rateLimiterService as UploadUsageRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.UPLOAD_USAGE.name), + (rateLimiterService as UploadUsageRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UploadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UploadUsageRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UploadUsage:" + "/blueking/", + (rateLimiterService as UploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UploadUsageRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UploadUsage:" + "/blueking/generic-local/", + (rateLimiterService as UploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as UploadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UploadUsageRateLimiterService).buildExtraResource(request) + ) + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + val resourceLimit = (rateLimiterService as UploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + } + + l1.resource = "/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + } + + l1.resource = "/blueking/" + l1.limit = 20 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + } + } + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterServiceTest.kt new file mode 100644 index 0000000000..89829d5b98 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterServiceTest.kt @@ -0,0 +1,309 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage.user + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserDownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import org.springframework.web.servlet.HandlerMapping +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UserDownloadUsageRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + var l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/*/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.method = "GET" + val attributes = ServletRequestAttributes(request) + RequestContextHolder.setRequestAttributes(attributes) + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UserDownloadUsageRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals( + false, + (rateLimiterService as UserDownloadUsageRateLimiterService).ignoreRequest(request) + ) + request.method = "POST" + Assertions.assertEquals( + true, + (rateLimiterService as UserDownloadUsageRateLimiterService).ignoreRequest(request) + ) + request.method = "PATCH" + + Assertions.assertEquals( + true, + (rateLimiterService as UserDownloadUsageRateLimiterService).ignoreRequest(request) + ) + request.method = "PUT" + + Assertions.assertEquals( + true, + (rateLimiterService as UserDownloadUsageRateLimiterService).ignoreRequest(request) + ) + request.method = "GET" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = + (rateLimiterService as UserDownloadUsageRateLimiterService).getRepoInfoFromAttribute(request) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "admin:/blueking/generic-local/", + (rateLimiterService as UserDownloadUsageRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("admin:/blueking/", "admin:"), + (rateLimiterService as UserDownloadUsageRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertThrows(AcquireLockFailedException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).getApplyPermits(request, null) + } + + Assertions.assertEquals( + 10, + (rateLimiterService as UserDownloadUsageRateLimiterService).getApplyPermits(request, 10) + ) + + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UserDownloadUsageRateLimitRule::class.java, + (rateLimiterService as UserDownloadUsageRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.USER_DOWNLOAD_USAGE.name), + (rateLimiterService as UserDownloadUsageRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as UserDownloadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserDownloadUsageRateLimiterService).buildExtraResource(request) + ) + + l1.resource = "test:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + var resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "test:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UserDownloadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserDownloadUsageRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserDownloadUsage:" + "admin:/blueking/", + (rateLimiterService as UserDownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "admin:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserDownloadUsage:" + "admin:/blueking/generic-local/", + (rateLimiterService as UserDownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "admin:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserDownloadUsage:" + "admin:", + (rateLimiterService as UserDownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "admin:/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "admin:" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "test:" + l1.limit = 20 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertDoesNotThrow { (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) } + } + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterServiceTest.kt new file mode 100644 index 0000000000..9b0cc43f92 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterServiceTest.kt @@ -0,0 +1,298 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage.user + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserUploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import org.springframework.web.servlet.HandlerMapping + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UserUploadUsageRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + var l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/*/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + val content = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + request.setContent(content) + request.method = "PUT" + val attributes = ServletRequestAttributes(request) + RequestContextHolder.setRequestAttributes(attributes) + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UserUploadUsageRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UserUploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "POST" + Assertions.assertEquals(false, (rateLimiterService as UserUploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PATCH" + Assertions.assertEquals(false, (rateLimiterService as UserUploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "GET" + Assertions.assertEquals(true, (rateLimiterService as UserUploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PUT" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = (rateLimiterService as UserUploadUsageRateLimiterService).getRepoInfoFromAttribute( + request + ) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "admin:/blueking/generic-local/", + (rateLimiterService as UserUploadUsageRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("admin:/blueking/", "admin:"), + (rateLimiterService as UserUploadUsageRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals( + 10, + (rateLimiterService as UserUploadUsageRateLimiterService).getApplyPermits(request, null) + ) + request.method = "GET" + Assertions.assertEquals( + 0, + (rateLimiterService as UserUploadUsageRateLimiterService).getApplyPermits(request, null) + ) + request.method = "PUT" + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UserUploadUsageRateLimitRule::class.java, + (rateLimiterService as UserUploadUsageRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.USER_UPLOAD_USAGE.name), + (rateLimiterService as UserUploadUsageRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as UserUploadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUploadUsageRateLimiterService).buildExtraResource(request) + ) + + l1.resource = "test:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + var resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "test:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UserUploadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUploadUsageRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUploadUsage:" + "admin:/blueking/", + (rateLimiterService as UserUploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "admin:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUploadUsage:" + "admin:/blueking/generic-local/", + (rateLimiterService as UserUploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "admin:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUploadUsage:" + "admin:", + (rateLimiterService as UserUploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + } + + l1.resource = "admin:/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + } + + l1.resource = "admin:" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + } + + l1.resource = "test:" + l1.limit = 20 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + Assertions.assertDoesNotThrow { (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) } + } + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStreamTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStreamTest.kt new file mode 100644 index 0000000000..d554ac25a1 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStreamTest.kt @@ -0,0 +1,160 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.stream + +import com.google.common.base.Stopwatch +import com.google.common.base.Ticker +import com.tencent.bkrepo.common.api.util.HumanReadable +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedFixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedTest +import com.tencent.bkrepo.common.ratelimiter.algorithm.FixedWindowRateLimiter +import com.tencent.bkrepo.common.api.exception.OverloadException +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.mockito.Mockito +import org.springframework.test.annotation.DirtiesContext +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CommonRateLimitInputStreamTest : DistributedTest() { + + lateinit var ticker: Ticker + private val content = "1234567891" + val keyStr = "CommonRateLimitInputStreamTest" + + @BeforeAll + fun before() { + ticker = Mockito.mock(Ticker::class.java) + } + + @Test + fun readTestOncePermitsGreaterThanLength() { + val (context, _) = createContext(1024 * 1024) + inputStreamReadTest(context) + } + + @Test + fun readTestOncePermitsLessThanLength() { + val (context, _) = createContext(5) + inputStreamReadTest(context) + } + + @Test + fun readTestOncePermitsEqualLength() { + val (context, _) = createContext(10) + inputStreamReadTest(context) + } + + @Test + fun readTestOncePermitsGreaterThanLimit() { + val (context, _) = createContext(10, 5) + CommonRateLimitInputStream( + delegate = content.byteInputStream(), + rateCheckContext = context + ).use { `is` -> + val buf = ByteArray(3) + `is`.read(buf) + Assertions.assertThrows(OverloadException::class.java) { `is`.read(buf) } + } + } + + + + @Test + fun readTestOnMultiThreads() { + val (context, key) = createContext(10, 100, true, keyStr) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + inputStreamReadTest(context) + successNum++ + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + key?.let { clean(key) } + } + + private fun inputStreamReadTest(context: RateCheckContext) { + CommonRateLimitInputStream( + delegate = content.byteInputStream(), + rateCheckContext = context + ).use { `is` -> + val buf = ByteArray(3) + assertEquals(`is`.read(buf, 0, 3), 3) + assertEquals(String(buf), "123") + assertEquals(`is`.read().toChar(), '4') + assertEquals(`is`.read(buf), 3) + assertEquals(String(buf), "567") + assertEquals(`is`.read().toChar(), '8') + assertEquals(`is`.read().toChar(), '9') + assertEquals(`is`.read().toChar(), '1') + assertEquals(`is`.read(), -1) + println("finished") + } + + } + + private fun createContext( + permitsOnce: Long, limit: Long = 1024 * 1024 * 100, + distributed: Boolean = false, keyStr: String? = null, + ): Pair { + val (rateLimiter, key) = if (distributed) { + Pair(DistributedFixedWindowRateLimiter(keyStr!!, limit, Duration.ofSeconds(1), redisTemplate), keyStr) + } else { + Pair(FixedWindowRateLimiter(limit, Duration.ofSeconds(1), Stopwatch.createStarted(ticker)), keyStr) + } + return Pair( + RateCheckContext( + rateLimiter = rateLimiter, latency = 10, + waitRound = 3, rangeLength = content.length.toLong(), + dryRun = false, permitsOnce = permitsOnce, limitPerSecond = limit + ), key + ) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtilsTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtilsTest.kt new file mode 100644 index 0000000000..893b43f6da --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtilsTest.kt @@ -0,0 +1,155 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.utils + +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import org.hamcrest.CoreMatchers +import org.hamcrest.MatcherAssert +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class ResourcePathUtilsTest { + + @Test + fun testTokenizeUrlPath() { + val url = "/test1/test2" + try { + val actualSegments = ResourcePathUtils.tokenizeResourcePath(url) + assertEquals(actualSegments.size, 2) + MatcherAssert.assertThat(actualSegments, CoreMatchers.hasItems("test1", "test2")) + } catch (e: InvalidResourceException) { + fail("tokenizeUrlPath should not throw InvalidResourceException.") + } + } + + @Test + fun testTokenizeUrlPathWithUrlPatten() { + val url = "/test1/test2/{projectId}/{repoName}/{*:(.*)}" + try { + val actualSegments = ResourcePathUtils.tokenizeResourcePath(url) + assertEquals(actualSegments.size, 5) + MatcherAssert.assertThat( + actualSegments, + CoreMatchers.hasItems("test1", "test2", "{projectId}", "{repoName}", "{*:(.*)}") + ) + } catch (e: InvalidResourceException) { + fail("tokenizeUrlPath should not throw InvalidResourceException.") + } + } + + @Test + fun testTokenizeUrlPathWithEmptyUrl() { + try { + val actualSegments = ResourcePathUtils.tokenizeResourcePath("") + assertNotNull(actualSegments) + assertEquals(actualSegments.size, 0) + val actualSegments2 = ResourcePathUtils.tokenizeResourcePath("/") + assertNotNull(actualSegments2) + assertEquals(actualSegments2.size, 0) + } catch (e: InvalidResourceException) { + fail("tokenizeUrlPath should not throw InvalidResourceException.") + } + } + + @Test + fun testGetUrlPath() { + try { + var actualPath: String? = ResourcePathUtils.getPathOfUrl("http://www.bkrepo.com/") + assertEquals(actualPath, "/") + actualPath = ResourcePathUtils.getPathOfUrl("http://www.bkrepo.com") + assertEquals(actualPath, "/") + actualPath = ResourcePathUtils.getPathOfUrl("http://www.bkrepo.com/test1/test2") + assertEquals(actualPath, "/test1/test2") + actualPath = ResourcePathUtils.getPathOfUrl("http://www.bkrepo.com/test1/test2?user=xxx") + assertEquals(actualPath, "/test1/test2") + actualPath = ResourcePathUtils.getPathOfUrl("/test1/test2") + assertEquals(actualPath, "/test1/test2") + actualPath = ResourcePathUtils.getPathOfUrl("/test1/test2?user=xxx") + assertEquals(actualPath, "/test1/test2") + actualPath = ResourcePathUtils.getPathOfUrl("/test1/test2/") + assertEquals(actualPath, "/test1/test2/") + } catch (e: InvalidResourceException) { + fail("getPathOfUrl() should not throw exception here.") + } + } + + @Test + fun testGetUrlPathWithEmptyUrl() { + try { + val actualPath: String? = ResourcePathUtils.getPathOfUrl("") + assertNull(actualPath) + } catch (e: InvalidResourceException) { + fail("getPathOfUrl() should not throw exception here.") + } + } + + @Test + fun testGetUserAndPathWithEmptyUrl() { + try { + assertThrows { ResourcePathUtils.getUserAndPath("") } + } catch (e: InvalidResourceException) { + fail("getUserAndPath() should not throw exception here.") + } + } + + @Test + fun testGetUserAndPath() { + try { + assertThrows { ResourcePathUtils.getUserAndPath("a") } + var (userId, path) = ResourcePathUtils.getUserAndPath("a:") + assertEquals(userId, "a") + assertEquals(path, "") + val (userId1, path1) = ResourcePathUtils.getUserAndPath("a:1") + assertEquals(userId1, "a") + assertEquals(path1, "1") + val (userId2, path2) = ResourcePathUtils.getUserAndPath("a:1:12") + assertEquals(userId2, "a") + assertEquals(path2, "1:12") + } catch (e: InvalidResourceException) { + fail("getPathOfUrl() should not throw exception here.") + } + } + + @Test + fun testBuildUserPath() { + var actualPath: String? = ResourcePathUtils.buildUserPath("a", "b") + assertEquals(actualPath, "a:b") + actualPath = ResourcePathUtils.buildUserPath("1", "2") + assertEquals(actualPath, "1:2") + } + + @Test + fun testBuildUserPathWithEmptyUrl() { + var actualPath: String? = ResourcePathUtils.buildUserPath("a", "") + assertEquals(actualPath, "a:") + } +} \ No newline at end of file diff --git a/src/backend/common/common-service/service-servlet/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt b/src/backend/common/common-service/service-servlet/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt index 1df6f9540c..c80bd8746c 100644 --- a/src/backend/common/common-service/service-servlet/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt +++ b/src/backend/common/common-service/service-servlet/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt @@ -33,10 +33,15 @@ package com.tencent.bkrepo.common.service.exception import com.fasterxml.jackson.module.kotlin.MissingKotlinParameterException import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.exception.OverloadException import com.tencent.bkrepo.common.api.exception.TooManyRequestsException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.common.service.util.LocaleMessageUtils import org.springframework.core.Ordered import org.springframework.core.annotation.Order import org.springframework.http.converter.HttpMessageNotReadableException @@ -143,6 +148,21 @@ class GlobalExceptionHandler : AbstractExceptionHandler() { return response(exception) } + @ExceptionHandler(OverloadException::class) + fun handleException(exception: OverloadException): Response? { + if (HttpContextHolder.getResponse().isCommitted) { + // 当返回已写入部分数据后，无法正常返回429 + val errorMessage = LocaleMessageUtils.getLocalizedMessage(exception.messageCode, exception.params) + LoggerHolder.logErrorCodeException(exception, "[${exception.messageCode.getCode()}]$errorMessage") + HttpContextHolder.getResponse().outputStream.close() + return null + } else { + HttpContextHolder.getResponse().reset() + HttpContextHolder.getResponse().contentType = MediaTypes.APPLICATION_JSON_WITHOUT_CHARSET + return response(exception) + } + } + @ExceptionHandler(Exception::class) fun handleException(exception: Exception): Response { return response(exception) diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt index 506efdbddb..67248ff1c3 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt @@ -72,5 +72,9 @@ data class ReceiveProperties( /** * 每秒接收数据量 */ - var rateLimit: DataSize = DataSize.ofBytes(-1) + var rateLimit: DataSize = DataSize.ofBytes(-1), + /** + * 限速熔断阈值，当仓库配置的rateLimit小于等于限速熔断阈值时则直接将请求断开 + */ + var circuitBreakerThreshold: DataSize = DataSize.ofKilobytes(1), ) diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt index 8a333b99c2..43a81db429 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt @@ -44,6 +44,7 @@ import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.query.model.Sort +import com.tencent.bkrepo.common.api.exception.OverloadException import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -226,6 +227,8 @@ class ChartRepositoryServiceImpl( context.putAttribute(FULL_PATH, HelmUtils.getIndexCacheYamlFullPath()) try { ArtifactContextHolder.getRepository().download(context) + } catch (e: OverloadException) { + throw e } catch (e: Exception) { logger.warn("Error occurred while downloading index.yaml, error: ${e.message}") throw HelmFileNotFoundException( @@ -333,6 +336,8 @@ class ChartRepositoryServiceImpl( context.putAttribute(FILE_TYPE, CHART) try { ArtifactContextHolder.getRepository().download(context) + } catch (e: OverloadException) { + throw e } catch (e: ArtifactDownloadForbiddenException) { throw e } catch (e: Exception) { @@ -350,6 +355,8 @@ class ChartRepositoryServiceImpl( context.putAttribute(FILE_TYPE, PROV) try { ArtifactContextHolder.getRepository().download(context) + } catch (e: OverloadException) { + throw e } catch (e: Exception) { logger.warn("Error occurred while installing prov, error: ${e.message}") throw HelmFileNotFoundException( diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt index 88aac6b016..104ef8da19 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt @@ -49,6 +49,7 @@ import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_DESCRIPTION import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_MESSAGE import com.tencent.bkrepo.oci.pojo.response.OciErrorResponse import com.tencent.bkrepo.oci.pojo.response.OciResponse +import javax.servlet.http.HttpServletResponse import org.slf4j.LoggerFactory import org.springframework.core.Ordered import org.springframework.core.annotation.Order @@ -56,7 +57,6 @@ import org.springframework.http.HttpStatus import org.springframework.web.bind.annotation.ExceptionHandler import org.springframework.web.bind.annotation.ResponseStatus import org.springframework.web.bind.annotation.RestControllerAdvice -import javax.servlet.http.HttpServletResponse @Order(Ordered.HIGHEST_PRECEDENCE + 1) @RestControllerAdvice("com.tencent.bkrepo.oci") @@ -64,7 +64,7 @@ class OciExceptionHandler( private val ociProperties: OciProperties ) { -/** + /** * 单独处理认证失败异常，需要添加WWW_AUTHENTICATE响应头触发浏览器登录 */ @ExceptionHandler(AuthenticationException::class) @@ -122,12 +122,6 @@ class OciExceptionHandler( ociResponse(responseObject, exception) } - @ExceptionHandler(ErrorCodeException::class) - @ResponseStatus(HttpStatus.BAD_REQUEST) - fun handleException(exception: ErrorCodeException) { - ociResponse(exception) - } - @ExceptionHandler(PermissionException::class) @ResponseStatus(HttpStatus.FORBIDDEN) fun handleException(exception: PermissionException) { @@ -162,7 +156,7 @@ class OciExceptionHandler( val uri = HttpContextHolder.getRequest().requestURI logger.warn( "User[$userId] access oci resource[$uri] failed[${exception.javaClass.simpleName}]:" + - " ${responseObject.message}" + " ${responseObject.message}" ) } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/RateLimitController.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/RateLimitController.kt new file mode 100644 index 0000000000..fe1468e306 --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/RateLimitController.kt @@ -0,0 +1,118 @@ +package com.tencent.bkrepo.opdata.controller + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.exception.NotFoundException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.model.RateLimitCreatOrUpdateRequest +import com.tencent.bkrepo.common.ratelimiter.model.TRateLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.security.permission.Principal +import com.tencent.bkrepo.common.security.permission.PrincipalType +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestParam + +@RestController +@RequestMapping("/api/rateLimit") +@Principal(PrincipalType.ADMIN) +class RateLimitController( + private val rateLimiterConfigService: RateLimiterConfigService, + private val rateLimiterProperties: RateLimiterProperties +) { + + @GetMapping("/list") + fun list(): Response> { + return ResponseBuilder.success(rateLimiterConfigService.list()) + } + + @PostMapping("/update") + fun update(@RequestBody request: RateLimitCreatOrUpdateRequest): Response { + if (request.id.isNullOrBlank()) { + throw NotFoundException(CommonMessageCode.PARAMETER_EMPTY, "ID") + } + if (!rateLimiterConfigService.checkExist(request.id!!)) { + throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, request.id!!) + } + val tRateLimit = rateLimiterConfigService.findByModuleNameAndLimitDimensionAndResource( + request.resource, + request.moduleName, + request.limitDimension + ) + if (tRateLimit == null || !tRateLimit.id.equals(request.id)) { + checkResource(request) + } + rateLimiterConfigService.update(request) + return ResponseBuilder.success() + } + + private fun checkResource(request: RateLimitCreatOrUpdateRequest) { + with(request) { + val tRateLimits = rateLimiterConfigService.findByResourceAndLimitDimension( + resource = resource, + limitDimension = limitDimension + ) + val modules = ArrayList() + tRateLimits.forEach { tRateLimit -> + if(id == null || !tRateLimit.id.equals(id)) { + modules.addAll(tRateLimit.moduleName) + } + } + if (modules.isNotEmpty()) { + modules.retainAll(moduleName) + if (modules.isNotEmpty()) { + throw ErrorCodeException( + CommonMessageCode.RESOURCE_EXISTED, + "resource:$resource,limitDimension:$limitDimension,module:${modules}" + ) + } + } + } + } + + // 新增 + @PostMapping("/create") + fun create(@RequestBody request:RateLimitCreatOrUpdateRequest): Response { + checkResource(request) + rateLimiterConfigService.create(request) + return ResponseBuilder.success() + } + + // 删除 + @DeleteMapping("/delete/{id}") + fun delete(@PathVariable id:String): Response { + if (!rateLimiterConfigService.checkExist(id)) { + throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, id) + } + rateLimiterConfigService.delete(id) + return ResponseBuilder.success() + } + + // 获取配置中的属性 + @GetMapping("/config") + fun getConfig(): Response { + val config = rateLimiterProperties + return ResponseBuilder.success(config.rules.toJsonString()) + } + + // 获取数据库里面的模块名 + @PostMapping("/getExistModule") + fun getExistModule(@RequestParam resource:String,@RequestParam limitDimension:String): Response> { + val tRateLimits = rateLimiterConfigService.findByResourceAndLimitDimension( + resource = resource, + limitDimension = limitDimension + ) + val modules = ArrayList() + tRateLimits.forEach { tRateLimit -> modules.addAll(tRateLimit.moduleName) } + return ResponseBuilder.success(modules) + } + +} \ No newline at end of file diff --git a/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/fdtp/FdtpAFTTest.kt b/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/fdtp/FdtpAFTTest.kt index 8402ab5397..40ee61c8a4 100644 --- a/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/fdtp/FdtpAFTTest.kt +++ b/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/fdtp/FdtpAFTTest.kt @@ -36,6 +36,8 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactClient import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.proxy.ProxyRepository import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurity import com.tencent.bkrepo.common.security.service.ServiceAuthManager import com.tencent.bkrepo.common.security.service.ServiceAuthProperties @@ -51,11 +53,7 @@ import io.micrometer.core.instrument.simple.SimpleMeterRegistry import io.mockk.every import io.mockk.mockk import io.mockk.mockkObject -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.* import org.mockito.Mockito import org.springframework.cloud.loadbalancer.support.SimpleObjectProvider import org.springframework.cloud.sleuth.Tracer @@ -95,6 +93,7 @@ class FdtpAFTTest { val proxyRepository = Mockito.mock(ProxyRepository::class.java) val artifactClient = Mockito.mock(ArtifactClient::class.java) val httpAuthSecurity = SimpleObjectProvider(null) + val limitCheckService = RequestLimitCheckService(RateLimiterProperties()) ArtifactContextHolder( listOf(artifactConfigurer), compositeRepository, @@ -103,7 +102,7 @@ class FdtpAFTTest { httpAuthSecurity, ) val helper = StorageHealthMonitorHelper(ConcurrentHashMap()) - ArtifactFileFactory(StorageProperties(), helper) + ArtifactFileFactory(StorageProperties(), helper, limitCheckService) mockkObject(ArtifactMetrics) every { ArtifactMetrics.getUploadingCounters(any()) } returns emptyList() every { ArtifactMetrics.getUploadingTimer(any()) } returns Timer.builder(ARTIFACT_UPLOADING_TIME) diff --git a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/configuration/ResourceWriterConfigurer.kt b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/configuration/ResourceWriterConfigurer.kt index 402419b406..5a964cc4c8 100644 --- a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/configuration/ResourceWriterConfigurer.kt +++ b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/configuration/ResourceWriterConfigurer.kt @@ -33,17 +33,23 @@ package com.tencent.bkrepo.s3.artifact.configuration import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.s3.artifact.response.S3ArtifactResourceWriter import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Primary @Configuration -class ResourceWriterConfigurer{ +class ResourceWriterConfigurer { @Primary @Bean - fun artifactResourceWriter(storageProperties: StorageProperties): ArtifactResourceWriter { - return S3ArtifactResourceWriter(storageProperties) + fun artifactResourceWriter( + storageProperties: StorageProperties, + requestLimitCheckService: RequestLimitCheckService + ): ArtifactResourceWriter { + return S3ArtifactResourceWriter( + storageProperties, requestLimitCheckService + ) } } diff --git a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/response/S3ArtifactResourceWriter.kt b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/response/S3ArtifactResourceWriter.kt index 7c9eec2736..8fd9134352 100644 --- a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/response/S3ArtifactResourceWriter.kt +++ b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/response/S3ArtifactResourceWriter.kt @@ -38,6 +38,8 @@ import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException import com.tencent.bkrepo.common.artifact.resolve.response.AbstractArtifactResourceHandler import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.common.storage.monitor.Throughput @@ -51,13 +53,17 @@ import javax.servlet.http.HttpServletResponse /** * S3协议的响应输出 */ -class S3ArtifactResourceWriter ( - storageProperties: StorageProperties -) : AbstractArtifactResourceHandler(storageProperties) { +class S3ArtifactResourceWriter( + storageProperties: StorageProperties, + requestLimitCheckService: RequestLimitCheckService +) : AbstractArtifactResourceHandler( + storageProperties, requestLimitCheckService +) { - @Throws(ArtifactResponseException::class) + @Throws(ArtifactResponseException::class, OverloadException::class) override fun write(resource: ArtifactResource): Throughput { responseRateLimitCheck() + downloadRateLimitCheck(resource) return writeArtifact(resource) } diff --git a/src/backend/settings.gradle.kts b/src/backend/settings.gradle.kts index 8792746a34..23128e5dcc 100644 --- a/src/backend/settings.gradle.kts +++ b/src/backend/settings.gradle.kts @@ -62,6 +62,7 @@ includeAll(":common:common-storage") includeAll(":common:common-query") includeAll(":common:common-artifact") includeAll(":common:common-notify") +includeAll(":common:common-ratelimiter") includeAll(":composer") includeAll(":generic") includeAll(":helm") diff --git a/src/frontend/devops-op/src/api/rateLimit.js b/src/frontend/devops-op/src/api/rateLimit.js new file mode 100644 index 0000000000..95917ceab1 --- /dev/null +++ b/src/frontend/devops-op/src/api/rateLimit.js @@ -0,0 +1,51 @@ +import request from '@/utils/request' + +const PREFIX_SERVICES = '/opdata/api/rateLimit' + +export function queryRateLimits() { + return request({ + url: `${PREFIX_SERVICES}/list/`, + method: 'get' + }) +} + +export function deleteRateLimit(id) { + return request({ + url: `${PREFIX_SERVICES}/delete/${id}`, + method: 'delete' + }) +} + +export function createRateLimit(data) { + return request({ + url: `${PREFIX_SERVICES}/create/`, + method: 'post', + data: data + }) +} + +export function updateRateLimit(data) { + return request({ + url: `${PREFIX_SERVICES}/update/`, + method: 'post', + data: data + }) +} + +export function getRateLimitConfig() { + return request({ + url: `${PREFIX_SERVICES}/config/`, + method: 'get' + }) +} + +export function getExistModule(resource, limitDimension) { + return request({ + url: `${PREFIX_SERVICES}/getExistModule`, + method: 'post', + params: { + resource: resource, + limitDimension: limitDimension + } + }) +} diff --git a/src/frontend/devops-op/src/router/index.js b/src/frontend/devops-op/src/router/index.js index 24eaa9159d..06600c068c 100644 --- a/src/frontend/devops-op/src/router/index.js +++ b/src/frontend/devops-op/src/router/index.js @@ -24,6 +24,7 @@ export const ROUTER_NAME_FILE_SYSTEM = 'FileSystem' export const ROUTER_NAME_FILE_CACHE = 'FileCache' export const ROUTER_NAME_FILE_SYSTEM_RECORD = 'FileSystemRecord' export const ROUTER_NAME_REPO_CONFIG = 'RepoConfig' +export const ROUTER_NAME_RATE_LIMITER_CONFIG = 'RateLimiterConfig' Vue.use(Router) @@ -313,6 +314,18 @@ export const asyncRoutes = [ } ] }, + { + path: '/rateLimiter', + component: Layout, + children: [ + { + path: '/', + name: ROUTER_NAME_RATE_LIMITER_CONFIG, + meta: { title: '限流管理', icon: 'permission' }, + component: () => import('@/views/rateLimitConfg/RateLimiter') + } + ] + }, // 404 page must be placed at the end !!! { path: '*', redirect: '/404', hidden: true } ] diff --git a/src/frontend/devops-op/src/views/rateLimitConfg/RateLimiter.vue b/src/frontend/devops-op/src/views/rateLimitConfg/RateLimiter.vue new file mode 100644 index 0000000000..6f5e419fce --- /dev/null +++ b/src/frontend/devops-op/src/views/rateLimitConfg/RateLimiter.vue @@ -0,0 +1,246 @@ + + + + + diff --git a/src/frontend/devops-op/src/views/rateLimitConfg/components/CreateOrUpdateRateLimitDialog.vue b/src/frontend/devops-op/src/views/rateLimitConfg/components/CreateOrUpdateRateLimitDialog.vue new file mode 100644 index 0000000000..6caec34c72 --- /dev/null +++ b/src/frontend/devops-op/src/views/rateLimitConfg/components/CreateOrUpdateRateLimitDialog.vue @@ -0,0 +1,577 @@ + + + + + + diff --git a/src/frontend/devops-repository/src/views/repoGeneric/index.vue b/src/frontend/devops-repository/src/views/repoGeneric/index.vue index e1c9444eab..c87649f2e8 100644 --- a/src/frontend/devops-repository/src/views/repoGeneric/index.vue +++ b/src/frontend/devops-repository/src/views/repoGeneric/index.vue @@ -1063,6 +1063,11 @@ }) } }) + } else if (e.status === 429) { + this.$bkMessage({ + theme: 'error', + message: e.message + }) } else { const message = this.$t('fileError') this.$bkMessage({