diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/ProjectRepoMetricsStatJobContext.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/ProjectRepoMetricsStatJobContext.kt index cfa9264ed5..775ab67c2b 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/ProjectRepoMetricsStatJobContext.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/ProjectRepoMetricsStatJobContext.kt @@ -27,7 +27,7 @@ package com.tencent.bkrepo.job.batch.context -import com.tencent.bkrepo.job.batch.ProjectRepoMetricsStatJob +import com.tencent.bkrepo.job.batch.stat.ProjectRepoMetricsStatJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.pojo.project.TProjectMetrics import com.tencent.bkrepo.job.pojo.project.TRepoMetrics @@ -38,9 +38,13 @@ import java.util.concurrent.atomic.LongAdder data class ProjectRepoMetricsStatJobContext( var metrics: ConcurrentHashMap = ConcurrentHashMap(), var statDate: LocalDateTime, - var activeProjects: Set = emptySet() + var statProjects: Set = emptySet() ) : JobContext() { + override fun toString(): String { + return "Success[$success], failed[$failed], total[$total], statDate[$statDate]" + } + data class ProjectMetrics( val projectId: String, var nodeNum: LongAdder = LongAdder(), @@ -59,7 +63,10 @@ data class ProjectRepoMetricsStatJobContext( repo.size.add(row.size) } - fun toDO(statDate: LocalDateTime = LocalDateTime.now()): TProjectMetrics { + fun toDO( + active: Boolean, + statDate: LocalDateTime = LocalDateTime.now() + ): TProjectMetrics { val repoMetrics = ArrayList(repoMetrics.size) this.repoMetrics.values.forEach { repo -> val num = repo.num.toLong() @@ -75,7 +82,8 @@ data class ProjectRepoMetricsStatJobContext( nodeNum = nodeNum.toLong(), capSize = capSize.toLong(), repoMetrics = repoMetrics, - createdDate = statDate + createdDate = statDate, + active = active ) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ActiveProjectRepoMetricsStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ActiveProjectRepoMetricsStatJob.kt new file mode 100644 index 0000000000..4bb0f47e08 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ActiveProjectRepoMetricsStatJob.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.stat + +import com.tencent.bkrepo.job.batch.base.ActiveProjectService +import com.tencent.bkrepo.job.batch.context.ProjectRepoMetricsStatJobContext +import com.tencent.bkrepo.job.config.properties.ActiveProjectRepoMetricsStatJobProperties +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.stereotype.Component + +/** + * 活跃项目仓库指标统计任务 + */ +@Component +@EnableConfigurationProperties(ActiveProjectRepoMetricsStatJobProperties::class) +class ActiveProjectRepoMetricsStatJob( + properties: ActiveProjectRepoMetricsStatJobProperties, + activeProjectService: ActiveProjectService, +) : ProjectRepoMetricsStatJob(properties, activeProjectService) { + + override fun statProjectCheck( + projectId: String, + context: ProjectRepoMetricsStatJobContext + ): Boolean { + if (context.statProjects.isEmpty() || + context.statProjects.contains(projectId)) return true + return false + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/FolderStatChildJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/FolderStatChildJob.kt similarity index 99% rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/FolderStatChildJob.kt rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/FolderStatChildJob.kt index 9351d55211..8daa1e0b84 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/FolderStatChildJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/FolderStatChildJob.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.job.batch.node +package com.tencent.bkrepo.job.batch.stat import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.path.PathUtils @@ -84,9 +84,8 @@ class FolderStatChildJob( require(properties is NodeStatCompositeMongoDbBatchJobProperties) if (!context.runFlag) return if (!collectionRunCheck(collectionName)) return - if (row.deleted != null) return //只统计非目录类节点；没有根目录这个节点，不需要统计 - if (row.folder || row.path == PathUtils.ROOT) { + if (row.path == PathUtils.ROOT) { return } // 判断是否在不统计项目或者仓库列表中 diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/InactiveProjectRepoMetricsStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/InactiveProjectRepoMetricsStatJob.kt new file mode 100644 index 0000000000..7c31ee3874 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/InactiveProjectRepoMetricsStatJob.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.stat + +import com.tencent.bkrepo.job.batch.base.ActiveProjectService +import com.tencent.bkrepo.job.batch.context.ProjectRepoMetricsStatJobContext +import com.tencent.bkrepo.job.config.properties.InactiveProjectRepoMetricsStatJobProperties +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.stereotype.Component + +/** + * 非活跃项目仓库指标统计任务 + */ +@Component +@EnableConfigurationProperties(InactiveProjectRepoMetricsStatJobProperties::class) +class InactiveProjectRepoMetricsStatJob( + properties: InactiveProjectRepoMetricsStatJobProperties, + activeProjectService: ActiveProjectService, +) : ProjectRepoMetricsStatJob(properties, activeProjectService, false) { + + override fun statProjectCheck( + projectId: String, + context: ProjectRepoMetricsStatJobContext + ): Boolean { + if (context.statProjects.isNotEmpty() && + !context.statProjects.contains(projectId)) return true + return false + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/NodeStatCompositeMongoDbBatchJob.kt similarity index 89% rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/NodeStatCompositeMongoDbBatchJob.kt index da3697ac22..1f292fb98f 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/NodeStatCompositeMongoDbBatchJob.kt @@ -1,16 +1,18 @@ -package com.tencent.bkrepo.job.batch.node +package com.tencent.bkrepo.job.batch.stat +import com.tencent.bkrepo.job.DELETED_DATE +import com.tencent.bkrepo.job.FOLDER import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.ActiveProjectService import com.tencent.bkrepo.job.batch.base.ChildMongoDbBatchJob import com.tencent.bkrepo.job.batch.base.CompositeMongoDbBatchJob import com.tencent.bkrepo.job.config.properties.NodeStatCompositeMongoDbBatchJobProperties import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.redis.core.RedisTemplate import org.springframework.stereotype.Component import java.time.Duration -import java.util.Date import kotlin.reflect.KClass @Component @@ -25,7 +27,10 @@ class NodeStatCompositeMongoDbBatchJob( return (0 until SHARDING_COUNT).map { "node_$it" }.toList() } - override fun buildQuery(): Query = Query() + override fun buildQuery(): Query = Query( + Criteria.where(DELETED_DATE).`is`(null) + .and(FOLDER).`is`(false) + ) override fun mapToEntity(row: Map): Node = Node(row) @@ -62,9 +67,6 @@ class NodeStatCompositeMongoDbBatchJob( @JvmField val size: Long - @JvmField - val deleted: Date? - @JvmField val projectId: String @@ -78,8 +80,6 @@ class NodeStatCompositeMongoDbBatchJob( fullPath = map[Node::fullPath.name] as String name = map[Node::name.name] as String size = map[Node::size.name].toString().toLong() - // 查询出的deleted默认为Date类型 - deleted = map[Node::deleted.name] as Date? projectId = map[Node::projectId.name] as String repoName = map[Node::repoName.name] as String } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectMetricsReport2BkbaseJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ProjectMetricsReport2BkbaseJob.kt similarity index 98% rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectMetricsReport2BkbaseJob.kt rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ProjectMetricsReport2BkbaseJob.kt index 5ccb48a814..f6f2fe8eaf 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectMetricsReport2BkbaseJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ProjectMetricsReport2BkbaseJob.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.job.batch +package com.tencent.bkrepo.job.batch.stat import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString @@ -124,6 +124,7 @@ class ProjectMetricsReport2BkbaseJob( centerName = project.metadata.firstOrNull { it.key == ProjectMetadata.KEY_CENTER_NAME }?.value as? String, productId = project.metadata.firstOrNull { it.key == ProjectMetadata.KEY_PRODUCT_ID }?.value as? Int, enabled = project.metadata.firstOrNull { it.key == ProjectMetadata.KEY_ENABLED }?.value as? Boolean, + active = current.active ) } @@ -156,6 +157,7 @@ class ProjectMetricsReport2BkbaseJob( var helmRepoCapSize: Long = 0, var dockerRepoCapSize: Long = 0, val createdDate: LocalDateTime, + val active: Boolean = true ) data class ProjectInfo( diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectRepoMetricsStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ProjectRepoMetricsStatJob.kt similarity index 79% rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectRepoMetricsStatJob.kt rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ProjectRepoMetricsStatJob.kt index cbca5178f1..b435cf28a6 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectRepoMetricsStatJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ProjectRepoMetricsStatJob.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.job.batch +package com.tencent.bkrepo.job.batch.stat import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.job.CREATED_DATE @@ -40,16 +40,12 @@ import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.batch.context.ProjectRepoMetricsStatJobContext import com.tencent.bkrepo.job.batch.utils.FolderUtils import com.tencent.bkrepo.job.batch.utils.MongoShardingUtils -import com.tencent.bkrepo.job.batch.utils.TimeUtils -import com.tencent.bkrepo.job.config.properties.ProjectRepoMetricsStatJobProperties +import com.tencent.bkrepo.job.config.properties.MongodbJobProperties import com.tencent.bkrepo.job.pojo.project.TProjectMetrics import org.slf4j.LoggerFactory -import org.springframework.boot.context.properties.EnableConfigurationProperties -import org.springframework.data.mongodb.core.BulkOperations import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo -import org.springframework.stereotype.Component import java.time.Duration import java.time.LocalDate import java.time.LocalDateTime @@ -58,17 +54,17 @@ import kotlin.reflect.KClass /** * 项目仓库指标统计任务 */ -@Component -@EnableConfigurationProperties(ProjectRepoMetricsStatJobProperties::class) -class ProjectRepoMetricsStatJob( - private val properties: ProjectRepoMetricsStatJobProperties, +open class ProjectRepoMetricsStatJob( + properties: MongodbJobProperties, private val activeProjectService: ActiveProjectService, + private val active: Boolean = true, ) : DefaultContextMongoDbJob(properties) { + override fun collectionNames(): List { return listOf(COLLECTION_REPOSITORY_NAME) } - override fun buildQuery(): Query = Query() + override fun buildQuery(): Query = Query(Criteria.where(DELETED_DATE).`is`(null)) override fun mapToEntity(row: Map): Repository { return Repository(row) @@ -78,12 +74,15 @@ class ProjectRepoMetricsStatJob( return Repository::class } + open fun statProjectCheck( + projectId: String, + context: ProjectRepoMetricsStatJobContext + ): Boolean = true + override fun run(row: Repository, collectionName: String, context: JobContext) { require(context is ProjectRepoMetricsStatJobContext) with(row) { - if (deleted != null) return - if (context.activeProjects.isNotEmpty() && !properties.runAllProjects && - !context.activeProjects.contains(row.projectId)) return + if (!statProjectCheck(projectId, context)) return val query = Query( Criteria.where(PROJECT).isEqualTo(projectId).and(REPO).isEqualTo(name) .and(PATH).isEqualTo(PathUtils.ROOT).and(DELETED_DATE).isEqualTo(null) @@ -116,40 +115,31 @@ class ProjectRepoMetricsStatJob( override fun onRunCollectionFinished(collectionName: String, context: JobContext) { super.onRunCollectionFinished(collectionName, context) require(context is ProjectRepoMetricsStatJobContext) - logger.info("start to insert project's metrics ") - val projectMetrics = ArrayList(context.metrics.size) + logger.info("start to insert [active: ${this.active}] project's metrics") for (entry in context.metrics) { - projectMetrics.add(entry.value.toDO(context.statDate)) + storeMetrics(context.statDate, entry.value.toDO(active, context.statDate)) } - storeMetrics(context.statDate, projectMetrics) context.metrics.clear() - projectMetrics.clear() - logger.info("stat project metrics done") + logger.info("stat [active: ${this.active}] project metrics done") } override fun createJobContext(): ProjectRepoMetricsStatJobContext{ return ProjectRepoMetricsStatJobContext( statDate = LocalDate.now().atStartOfDay(), - activeProjects = if (properties.runAllProjects) { - emptySet() - } else { - activeProjectService.getActiveProjects() - }, + statProjects = activeProjectService.getActiveProjects() ) } - private fun storeMetrics( statDate: LocalDateTime, - projectMetrics: List + projectMetric: TProjectMetrics ) { // insert project repo metrics - val criteria = Criteria.where(CREATED_DATE).isEqualTo(statDate) + val criteria = Criteria.where(CREATED_DATE).isEqualTo(statDate).and(PROJECT).`is`(projectMetric.projectId) mongoTemplate.remove(Query(criteria), COLLECTION_NAME_PROJECT_METRICS) - mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, COLLECTION_NAME_PROJECT_METRICS) - .insert(projectMetrics) - .execute() + logger.info("stat project: [${projectMetric.projectId}], size: [${projectMetric.capSize}]") + mongoTemplate.insert(projectMetric, COLLECTION_NAME_PROJECT_METRICS) } data class Repository( @@ -157,14 +147,12 @@ class ProjectRepoMetricsStatJob( var name: String, var type: String, var credentialsKey: String = "default", - val deleted: LocalDateTime? = null ) { constructor(map: Map) : this( map[Repository::projectId.name].toString(), map[Repository::name.name].toString(), map[Repository::type.name].toString(), map[Repository::credentialsKey.name]?.toString() ?: "default", - map[Repository::deleted.name]?.let { TimeUtils.parseMongoDateTimeStr(it.toString()) } ) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectRepoMetricsStatJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ActiveProjectRepoMetricsStatJobProperties.kt similarity index 85% rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectRepoMetricsStatJobProperties.kt rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ActiveProjectRepoMetricsStatJobProperties.kt index 0aa08ffe74..6caea516a4 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectRepoMetricsStatJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ActiveProjectRepoMetricsStatJobProperties.kt @@ -30,14 +30,10 @@ package com.tencent.bkrepo.job.config.properties import org.springframework.boot.context.properties.ConfigurationProperties /** - * 项目仓库指标统计任务配置项 + * 活跃项目仓库指标统计任务配置项 */ -@ConfigurationProperties("job.project-repo-metrics-stat") -data class ProjectRepoMetricsStatJobProperties( +@ConfigurationProperties("job.active-project-repo-metrics-stat") +data class ActiveProjectRepoMetricsStatJobProperties( override var enabled: Boolean = true, - override var cron: String = "0 0 18 * * ?", - /** - * 是否遍历所有项目记录 - */ - var runAllProjects: Boolean = true + override var cron: String = "0 0 0/6 * * ?", ) : MongodbJobProperties() \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/InactiveProjectRepoMetricsStatJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/InactiveProjectRepoMetricsStatJobProperties.kt new file mode 100644 index 0000000000..79f43ce4f3 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/InactiveProjectRepoMetricsStatJobProperties.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +/** + * 非活跃项目仓库指标统计任务配置项 + */ +@ConfigurationProperties("job.inactive-project-repo-metrics-stat") +data class InactiveProjectRepoMetricsStatJobProperties( + override var enabled: Boolean = true, + override var cron: String = "0 0 1 * * ?", +) : MongodbJobProperties() \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetrics.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetrics.kt index 673ce96383..e3778f2f07 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetrics.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/project/TProjectMetrics.kt @@ -34,5 +34,6 @@ data class TProjectMetrics( var nodeNum: Long, var capSize: Long, val repoMetrics: List, - val createdDate: LocalDateTime? = LocalDateTime.now() + val createdDate: LocalDateTime? = LocalDateTime.now(), + val active: Boolean = true ) diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/constant/Constants.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/constant/Constants.kt index 23b1e62387..f13955a6b7 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/constant/Constants.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/constant/Constants.kt @@ -77,6 +77,7 @@ const val START_DATE = "startDate" const val END_DATE = "endDate" const val DURATION = "duration" const val DELTA_POSITIVE = "deltaPositive" +const val ACTIVE = "active" const val B_0 = "0" diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/MetricFilterInfo.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/MetricFilterInfo.kt index a6a1c75cc4..23bfd684a0 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/MetricFilterInfo.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/MetricFilterInfo.kt @@ -37,4 +37,5 @@ data class MetricFilterInfo( val filterValue: String? = null, val compareFlag: Boolean = false, val deltaPositive: Int? = null, + val active: Boolean? = null ) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/BaseHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/BaseHandler.kt index d643a3de31..9b1eb618ee 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/BaseHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/BaseHandler.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.opdata.handler.impl import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.opdata.constant.ACTIVE import com.tencent.bkrepo.opdata.constant.DELTA_POSITIVE import com.tencent.bkrepo.opdata.constant.DOCKER_TYPES import com.tencent.bkrepo.opdata.constant.DURATION @@ -55,7 +56,8 @@ import java.time.format.DateTimeFormatter * "startDate":"2023-11-10", * "endDate":"2023-11-14", * "duration":1, - * "deltaPositive: 1" 0表示不变的；1 表示增加的；2 表示减少的 + * "deltaPositive: 1" 0表示不变的；1 表示增加的；2 表示减少的, + * "active": true" true表示活跃项目、false 表示非活跃项目、 null表示所有项目 * } */ open class BaseHandler( @@ -68,14 +70,16 @@ open class BaseHandler( date = metricFilterInfo.endDate!!, target = target.target, filterType = metricFilterInfo.filterType, - filterValue = metricFilterInfo.filterValue + filterValue = metricFilterInfo.filterValue, + active = metricFilterInfo.active ) if (!metricFilterInfo.compareFlag) return latestMetricsResult val olderMetricsResult = getMetricsResult( date = metricFilterInfo.startDate!!, target = target.target, filterType = metricFilterInfo.filterType, - filterValue = metricFilterInfo.filterValue + filterValue = metricFilterInfo.filterValue, + active = metricFilterInfo.active ) val deltaValue = findDeltaValue(olderMetricsResult, latestMetricsResult) return if (metricFilterInfo.deltaPositive == null) { @@ -94,10 +98,14 @@ open class BaseHandler( target: Metrics, filterType: FilterType?, filterValue: String?, + active: Boolean? = null ): HashMap { - val projectMetrics = MetricsCacheUtil.getProjectMetrics( + var projectMetrics = MetricsCacheUtil.getProjectMetrics( date.format(DateTimeFormatter.ISO_DATE_TIME) ) + active?.let { + projectMetrics = projectMetrics.filter { it.active == active } + } return when (filterType) { FilterType.REPO_TYPE -> { @@ -150,6 +158,7 @@ open class BaseHandler( val endDateStr = reqData?.get(END_DATE) as? String val duration = reqData?.get(DURATION)?.toString()?.toLongOrNull() val deltaPositive = reqData?.get(DELTA_POSITIVE)?.toString()?.toIntOrNull() + val active = reqData?.get(ACTIVE)?.toString()?.toBoolean() val endDate = if (endDateStr.isNullOrEmpty()) { statDateModel.getShedLockInfo() @@ -168,7 +177,8 @@ open class BaseHandler( startDate = startDate, endDate = endDate, compareFlag = !startDateStr.isNullOrEmpty() || duration != null, - deltaPositive = deltaPositive + deltaPositive = deltaPositive, + active = active ) } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt index 3dbdd1f467..d899df7d94 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt @@ -43,7 +43,7 @@ class StatDateModel @Autowired constructor( ) { companion object { - private val JOB_NAMES = listOf("ProjectRepoMetricsStatJob") + private val JOB_NAMES = listOf("ActiveProjectRepoMetricsStatJob", "InactiveProjectRepoMetricsStatJob") } fun getShedLockInfo(ids: List = JOB_NAMES): LocalDateTime { diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/TProjectMetrics.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/TProjectMetrics.kt index 38fffede20..864537bd14 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/TProjectMetrics.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/TProjectMetrics.kt @@ -47,5 +47,6 @@ data class TProjectMetrics( var nodeNum: Long, var capSize: Long, val repoMetrics: List, - val createdDate: LocalDateTime? = LocalDateTime.now() + val createdDate: LocalDateTime? = LocalDateTime.now(), + val active: Boolean = true )