feat: 统计任务拆分 TencentBlueKing#1673

* feat: 统计任务拆分 TencentBlueKing#1673

* feat: 去除多余引用 TencentBlueKing#1673

* feat: 代码调整 TencentBlueKing#1673

* feat: 项目metrics指标查下调整 TencentBlueKing#1673

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/ProjectRepoMetricsStatJobContext.kt

@@ -27,7 +27,7 @@
 
 package com.tencent.bkrepo.job.batch.context
 
-import com.tencent.bkrepo.job.batch.ProjectRepoMetricsStatJob
+import com.tencent.bkrepo.job.batch.stat.ProjectRepoMetricsStatJob
 import com.tencent.bkrepo.job.batch.base.JobContext
 import com.tencent.bkrepo.job.pojo.project.TProjectMetrics
 import com.tencent.bkrepo.job.pojo.project.TRepoMetrics
@@ -38,9 +38,13 @@ import java.util.concurrent.atomic.LongAdder
 data class ProjectRepoMetricsStatJobContext(
     var metrics: ConcurrentHashMap<String, ProjectMetrics> = ConcurrentHashMap(),
     var statDate: LocalDateTime,
-    var activeProjects: Set<String> = emptySet()
+    var statProjects: Set<String> = emptySet()
 ) : JobContext() {
 
+    override fun toString(): String {
+        return "Success[$success], failed[$failed], total[$total], statDate[$statDate]"
+    }
+
     data class ProjectMetrics(
         val projectId: String,
         var nodeNum: LongAdder = LongAdder(),
@@ -59,7 +63,10 @@ data class ProjectRepoMetricsStatJobContext(
             repo.size.add(row.size)
         }
 
-        fun toDO(statDate: LocalDateTime = LocalDateTime.now()): TProjectMetrics {
+        fun toDO(
+            active: Boolean,
+            statDate: LocalDateTime = LocalDateTime.now()
+        ): TProjectMetrics {
             val repoMetrics = ArrayList<TRepoMetrics>(repoMetrics.size)
             this.repoMetrics.values.forEach { repo ->
                 val num = repo.num.toLong()
@@ -75,7 +82,8 @@ data class ProjectRepoMetricsStatJobContext(
                 nodeNum = nodeNum.toLong(),
                 capSize = capSize.toLong(),
                 repoMetrics = repoMetrics,
-                createdDate = statDate
+                createdDate = statDate,
+                active = active
             )
         }
     }

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ActiveProjectRepoMetricsStatJob.kt

@@ -0,0 +1,54 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.batch.stat
+
+import com.tencent.bkrepo.job.batch.base.ActiveProjectService
+import com.tencent.bkrepo.job.batch.context.ProjectRepoMetricsStatJobContext
+import com.tencent.bkrepo.job.config.properties.ActiveProjectRepoMetricsStatJobProperties
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.stereotype.Component
+
+/**
+ * 活跃项目仓库指标统计任务
+ */
+@Component
+@EnableConfigurationProperties(ActiveProjectRepoMetricsStatJobProperties::class)
+class ActiveProjectRepoMetricsStatJob(
+    properties: ActiveProjectRepoMetricsStatJobProperties,
+    activeProjectService: ActiveProjectService,
+) : ProjectRepoMetricsStatJob(properties, activeProjectService) {
+
+    override fun statProjectCheck(
+        projectId: String,
+        context: ProjectRepoMetricsStatJobContext
+    ): Boolean {
+        if (context.statProjects.isEmpty() ||
+            context.statProjects.contains(projectId)) return true
+        return false
+    }
+}

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/FolderStatChildJob.kt → src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/FolderStatChildJob.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.job.batch.node
+package com.tencent.bkrepo.job.batch.stat
 
 import com.tencent.bkrepo.common.api.constant.StringPool
 import com.tencent.bkrepo.common.artifact.path.PathUtils
@@ -84,9 +84,8 @@ class FolderStatChildJob(
         require(properties is NodeStatCompositeMongoDbBatchJobProperties)
         if (!context.runFlag) return
         if (!collectionRunCheck(collectionName)) return
-        if (row.deleted != null) return
         //只统计非目录类节点；没有根目录这个节点，不需要统计
-        if (row.folder || row.path == PathUtils.ROOT) {
+        if (row.path == PathUtils.ROOT) {
             return
         }
         // 判断是否在不统计项目或者仓库列表中

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/InactiveProjectRepoMetricsStatJob.kt

@@ -0,0 +1,54 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.batch.stat
+
+import com.tencent.bkrepo.job.batch.base.ActiveProjectService
+import com.tencent.bkrepo.job.batch.context.ProjectRepoMetricsStatJobContext
+import com.tencent.bkrepo.job.config.properties.InactiveProjectRepoMetricsStatJobProperties
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.stereotype.Component
+
+/**
+ * 非活跃项目仓库指标统计任务
+ */
+@Component
+@EnableConfigurationProperties(InactiveProjectRepoMetricsStatJobProperties::class)
+class InactiveProjectRepoMetricsStatJob(
+    properties: InactiveProjectRepoMetricsStatJobProperties,
+    activeProjectService: ActiveProjectService,
+) : ProjectRepoMetricsStatJob(properties, activeProjectService, false) {
+
+    override fun statProjectCheck(
+        projectId: String,
+        context: ProjectRepoMetricsStatJobContext
+    ): Boolean {
+        if (context.statProjects.isNotEmpty() &&
+            !context.statProjects.contains(projectId)) return true
+        return false
+    }
+}

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt → src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/NodeStatCompositeMongoDbBatchJob.kt

@@ -1,16 +1,18 @@
-package com.tencent.bkrepo.job.batch.node
+package com.tencent.bkrepo.job.batch.stat
 
+import com.tencent.bkrepo.job.DELETED_DATE
+import com.tencent.bkrepo.job.FOLDER
 import com.tencent.bkrepo.job.SHARDING_COUNT
 import com.tencent.bkrepo.job.batch.base.ActiveProjectService
 import com.tencent.bkrepo.job.batch.base.ChildMongoDbBatchJob
 import com.tencent.bkrepo.job.batch.base.CompositeMongoDbBatchJob
 import com.tencent.bkrepo.job.config.properties.NodeStatCompositeMongoDbBatchJobProperties
 import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.data.mongodb.core.query.Criteria
 import org.springframework.data.mongodb.core.query.Query
 import org.springframework.data.redis.core.RedisTemplate
 import org.springframework.stereotype.Component
 import java.time.Duration
-import java.util.Date
 import kotlin.reflect.KClass
 
 @Component
@@ -25,7 +27,10 @@ class NodeStatCompositeMongoDbBatchJob(
         return (0 until SHARDING_COUNT).map { "node_$it" }.toList()
     }
 
-    override fun buildQuery(): Query = Query()
+    override fun buildQuery(): Query = Query(
+        Criteria.where(DELETED_DATE).`is`(null)
+            .and(FOLDER).`is`(false)
+    )
 
     override fun mapToEntity(row: Map<String, Any?>): Node = Node(row)
 
@@ -62,9 +67,6 @@ class NodeStatCompositeMongoDbBatchJob(
         @JvmField
         val size: Long
 
-        @JvmField
-        val deleted: Date?
-
         @JvmField
         val projectId: String
 
@@ -78,8 +80,6 @@ class NodeStatCompositeMongoDbBatchJob(
             fullPath = map[Node::fullPath.name] as String
             name = map[Node::name.name] as String
             size = map[Node::size.name].toString().toLong()
-            // 查询出的deleted默认为Date类型
-            deleted = map[Node::deleted.name] as Date?
             projectId = map[Node::projectId.name] as String
             repoName = map[Node::repoName.name] as String
         }

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectMetricsReport2BkbaseJob.kt → src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ProjectMetricsReport2BkbaseJob.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.job.batch
+package com.tencent.bkrepo.job.batch.stat
 
 import com.tencent.bkrepo.common.api.util.readJsonString
 import com.tencent.bkrepo.common.api.util.toJsonString
@@ -124,6 +124,7 @@ class ProjectMetricsReport2BkbaseJob(
             centerName = project.metadata.firstOrNull { it.key == ProjectMetadata.KEY_CENTER_NAME }?.value as? String,
             productId = project.metadata.firstOrNull { it.key == ProjectMetadata.KEY_PRODUCT_ID }?.value as? Int,
             enabled = project.metadata.firstOrNull { it.key == ProjectMetadata.KEY_ENABLED }?.value as? Boolean,
+            active = current.active
         )
     }
 
@@ -156,6 +157,7 @@ class ProjectMetricsReport2BkbaseJob(
         var helmRepoCapSize: Long = 0,
         var dockerRepoCapSize: Long = 0,
         val createdDate: LocalDateTime,
+        val active: Boolean = true
     )
 
     data class ProjectInfo(

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectRepoMetricsStatJob.kt → src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/stat/ProjectRepoMetricsStatJob.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.job.batch
+package com.tencent.bkrepo.job.batch.stat
 
 import com.tencent.bkrepo.common.artifact.path.PathUtils
 import com.tencent.bkrepo.job.CREATED_DATE
@@ -40,16 +40,12 @@ import com.tencent.bkrepo.job.batch.base.JobContext
 import com.tencent.bkrepo.job.batch.context.ProjectRepoMetricsStatJobContext
 import com.tencent.bkrepo.job.batch.utils.FolderUtils
 import com.tencent.bkrepo.job.batch.utils.MongoShardingUtils
-import com.tencent.bkrepo.job.batch.utils.TimeUtils
-import com.tencent.bkrepo.job.config.properties.ProjectRepoMetricsStatJobProperties
+import com.tencent.bkrepo.job.config.properties.MongodbJobProperties
 import com.tencent.bkrepo.job.pojo.project.TProjectMetrics
 import org.slf4j.LoggerFactory
-import org.springframework.boot.context.properties.EnableConfigurationProperties
-import org.springframework.data.mongodb.core.BulkOperations
 import org.springframework.data.mongodb.core.query.Criteria
 import org.springframework.data.mongodb.core.query.Query
 import org.springframework.data.mongodb.core.query.isEqualTo
-import org.springframework.stereotype.Component
 import java.time.Duration
 import java.time.LocalDate
 import java.time.LocalDateTime
@@ -58,17 +54,17 @@ import kotlin.reflect.KClass
 /**
  * 项目仓库指标统计任务
  */
-@Component
-@EnableConfigurationProperties(ProjectRepoMetricsStatJobProperties::class)
-class ProjectRepoMetricsStatJob(
-    private val properties: ProjectRepoMetricsStatJobProperties,
+open class ProjectRepoMetricsStatJob(
+    properties: MongodbJobProperties,
     private val activeProjectService: ActiveProjectService,
+    private val active: Boolean = true,
 ) : DefaultContextMongoDbJob<ProjectRepoMetricsStatJob.Repository>(properties) {
+
     override fun collectionNames(): List<String> {
         return listOf(COLLECTION_REPOSITORY_NAME)
     }
 
-    override fun buildQuery(): Query = Query()
+    override fun buildQuery(): Query = Query(Criteria.where(DELETED_DATE).`is`(null))
 
     override fun mapToEntity(row: Map<String, Any?>): Repository {
         return Repository(row)
@@ -78,12 +74,15 @@ class ProjectRepoMetricsStatJob(
         return Repository::class
     }
 
+    open fun statProjectCheck(
+        projectId: String,
+        context: ProjectRepoMetricsStatJobContext
+    ): Boolean = true
+
     override fun run(row: Repository, collectionName: String, context: JobContext) {
         require(context is ProjectRepoMetricsStatJobContext)
         with(row) {
-            if (deleted != null) return
-            if (context.activeProjects.isNotEmpty() && !properties.runAllProjects &&
-                !context.activeProjects.contains(row.projectId)) return
+            if (!statProjectCheck(projectId, context)) return
             val query = Query(
                 Criteria.where(PROJECT).isEqualTo(projectId).and(REPO).isEqualTo(name)
                     .and(PATH).isEqualTo(PathUtils.ROOT).and(DELETED_DATE).isEqualTo(null)
@@ -116,55 +115,44 @@ class ProjectRepoMetricsStatJob(
     override fun onRunCollectionFinished(collectionName: String, context: JobContext) {
         super.onRunCollectionFinished(collectionName, context)
         require(context is ProjectRepoMetricsStatJobContext)
-        logger.info("start to insert project's metrics ")
-        val projectMetrics = ArrayList<TProjectMetrics>(context.metrics.size)
+        logger.info("start to insert [active: ${this.active}] project's metrics")
         for (entry in context.metrics) {
-            projectMetrics.add(entry.value.toDO(context.statDate))
+            storeMetrics(context.statDate, entry.value.toDO(active, context.statDate))
         }
-        storeMetrics(context.statDate, projectMetrics)
         context.metrics.clear()
-        projectMetrics.clear()
-        logger.info("stat project metrics done")
+        logger.info("stat [active: ${this.active}] project metrics done")
     }
 
     override fun createJobContext(): ProjectRepoMetricsStatJobContext{
         return ProjectRepoMetricsStatJobContext(
             statDate = LocalDate.now().atStartOfDay(),
-            activeProjects = if (properties.runAllProjects) {
-                emptySet()
-            } else {
-                activeProjectService.getActiveProjects()
-            },
+            statProjects = activeProjectService.getActiveProjects()
         )
     }
 
 
-
     private fun storeMetrics(
         statDate: LocalDateTime,
-        projectMetrics: List<TProjectMetrics>
+        projectMetric: TProjectMetrics
     ) {
         // insert project repo metrics
-        val criteria = Criteria.where(CREATED_DATE).isEqualTo(statDate)
+        val criteria = Criteria.where(CREATED_DATE).isEqualTo(statDate).and(PROJECT).`is`(projectMetric.projectId)
         mongoTemplate.remove(Query(criteria), COLLECTION_NAME_PROJECT_METRICS)
-        mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, COLLECTION_NAME_PROJECT_METRICS)
-            .insert(projectMetrics)
-            .execute()
+        logger.info("stat project: [${projectMetric.projectId}], size: [${projectMetric.capSize}]")
+        mongoTemplate.insert(projectMetric, COLLECTION_NAME_PROJECT_METRICS)
     }
 
     data class Repository(
         var projectId: String,
         var name: String,
         var type: String,
         var credentialsKey: String = "default",
-        val deleted: LocalDateTime? = null
     ) {
         constructor(map: Map<String, Any?>) : this(
             map[Repository::projectId.name].toString(),
             map[Repository::name.name].toString(),
             map[Repository::type.name].toString(),
             map[Repository::credentialsKey.name]?.toString() ?: "default",
-            map[Repository::deleted.name]?.let { TimeUtils.parseMongoDateTimeStr(it.toString()) }
         )
     }