feat: 增加仓库备份功能#2837

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt

@@ -103,10 +103,20 @@ val IGNORE_PROJECT_PREFIX_LIST = listOf(CODE_PROJECT_PREFIX, CLOSED_SOURCE_PREFI
 const val RESTORE = "RESTORE"
 const val SEPARATE = "SEPARATE"
 
+const val PROJECT_COLLECTION_NAME = "project"
+const val REPO_COLLECTION_NAME = "repository"
+const val STORAGE_CREDENTIALS_COLLECTION_NAME = "storage_credentials"
+
 const val PACKAGE_COLLECTION_NAME = "package"
 const val PACKAGE_VERSION_COLLECTION_NAME = "package_version"
 const val PACKAGE_DOWNLOADS_COLLECTION_NAME = "package_downloads"
 const val SEPARATION_TASK_COLLECTION_NAME = "separation_task"
 
 const val PACKAGE_VERSION = "version"
-const val PACKAGE_DOWNLOAD_DATE = "date"
+const val PACKAGE_DOWNLOAD_DATE = "date"
+
+/**
+ * 记录备份
+ */
+const val DATA_RECORDS_BACKUP = "DATA_BACKUP"
+const val DATA_RECORDS_RESTORE = "DATA_RESTORE"

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt

@@ -0,0 +1,9 @@
+package com.tencent.bkrepo.job.backup.config
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+@ConfigurationProperties("backup")
+data class DataBackupConfig(
+    // 当仓库容量大于阈值时,不进行备份
+    var usageThreshold: Long = 1024 * 1024 * 1024,
+)

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt

@@ -0,0 +1,103 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.backup.dao
+
+import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao
+import com.tencent.bkrepo.job.backup.model.TBackupTask
+import com.tencent.bkrepo.job.backup.pojo.BackupTaskState
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+import org.springframework.data.domain.PageRequest
+import org.springframework.data.mongodb.core.query.Criteria
+import org.springframework.data.mongodb.core.query.Query
+import org.springframework.data.mongodb.core.query.Update
+import org.springframework.data.mongodb.core.query.isEqualTo
+import org.springframework.stereotype.Repository
+import java.time.LocalDateTime
+
+@Repository
+class BackupTaskDao : SimpleMongoDao<TBackupTask>() {
+
+    fun findTasksById(taskId: String): List<TBackupTask> {
+        val criteria = Criteria().and(TBackupTask::id.name).isEqualTo(taskId)
+        return find(Query(criteria))
+    }
+
+    fun find(state: String?, pageRequest: PageRequest): List<TBackupTask> {
+        val criteria = Criteria()
+        state?.let { criteria.and(TBackupTask::state.name).isEqualTo(it) }
+        return find(Query(criteria).with(pageRequest))
+    }
+
+    fun count(state: String?): Long {
+        val criteria = Criteria()
+        state?.let { criteria.and(TBackupTask::state.name).isEqualTo(it) }
+        return count(Query(criteria))
+    }
+
+    fun updateState(
+        taskId: String,
+        state: BackupTaskState,
+        startDate: LocalDateTime? = null,
+        endDate: LocalDateTime? = null
+    ) {
+        val criteria = Criteria().and(ID).isEqualTo(taskId)
+        val update = Update.update(TBackupTask::lastModifiedBy.name, SYSTEM_USER)
+            .set(TBackupTask::lastModifiedDate.name, LocalDateTime.now())
+            .set(TBackupTask::state.name, state.name)
+        startDate?.let { update.set(TBackupTask::startDate.name, startDate) }
+        endDate?.let { update.set(TBackupTask::endDate.name, endDate) }
+        this.updateFirst(Query(criteria), update)
+    }
+}

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt

@@ -0,0 +1,99 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.backup.model
+
+import com.tencent.bkrepo.job.DATA_RECORDS_BACKUP
+import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_STATE_IDX
+import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_STATE_IDX_DEF
+import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_TYPE_IDX
+import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_TYPE_IDX_DEF
+import com.tencent.bkrepo.job.backup.pojo.task.BackupContent
+import com.tencent.bkrepo.job.backup.pojo.BackupTaskState
+import com.tencent.bkrepo.job.backup.pojo.setting.BackupSetting
+import org.springframework.data.mongodb.core.index.CompoundIndex
+import org.springframework.data.mongodb.core.index.CompoundIndexes
+import org.springframework.data.mongodb.core.mapping.Document
+import java.time.LocalDateTime
+
+/**
+ * 备份与恢复任务
+ */
+@Document("backup_task")
+@CompoundIndexes(
+    CompoundIndex(name = TASK_STATE_IDX, def = TASK_STATE_IDX_DEF, background = true),
+    CompoundIndex(name = TASK_TYPE_IDX, def = TASK_TYPE_IDX_DEF, background = true),
+)
+data class TBackupTask(
+    val id: String? = null,
+    val name: String,
+    val storeLocation: String,
+    val type: String = DATA_RECORDS_BACKUP,
+    var content: BackupContent? = null,
+    val backupSetting: BackupSetting,
+    val state: String = BackupTaskState.PENDING.name,
+    val createdBy: String,
+    val createdDate: LocalDateTime,
+    val lastModifiedBy: String,
+    val lastModifiedDate: LocalDateTime,
+    val startDate: LocalDateTime? = null,
+    val endDate: LocalDateTime? = null,
+) {
+    companion object {
+        const val TASK_STATE_IDX = "state_idx"
+        const val TASK_STATE_IDX_DEF = "{'state': 1}"
+        const val TASK_TYPE_IDX = "type_idx"
+        const val TASK_TYPE_IDX_DEF = "{'type': 1}"
+    }
+}

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt

@@ -0,0 +1,38 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.backup.pojo
+
+
+enum class BackupTaskState {
+
+    PENDING,
+
+    RUNNING,
+
+    FINISHED,
+}

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt

@@ -0,0 +1,39 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2020 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.backup.pojo.query
+
+data class BackupFileReferenceInfo(
+    var id: String? = null,
+    var sha256: String,
+    var credentialsKey: String? = null,
+    var count: Long
+)

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt

@@ -0,0 +1,63 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2020 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.backup.pojo.query
+
+import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel
+import java.time.LocalDateTime
+
+data class BackupNodeInfo(
+    var id: String? = null,
+    var createdBy: String,
+    var createdDate: LocalDateTime,
+    var lastModifiedBy: String,
+    var lastModifiedDate: LocalDateTime,
+    var lastAccessDate: LocalDateTime? = null,
+
+    var folder: Boolean,
+    var path: String,
+    var name: String,
+    var fullPath: String,
+    var size: Long,
+    var expireDate: LocalDateTime? = null,
+    var sha256: String? = null,
+    var md5: String? = null,
+    var deleted: LocalDateTime? = null,
+    var copyFromCredentialsKey: String? = null,
+    var copyIntoCredentialsKey: String? = null,
+    var metadata: MutableList<MetadataModel>? = null,
+    var clusterNames: Set<String>? = null,
+    var nodeNum: Long? = null,
+    var archived: Boolean? = null,
+    var compressed: Boolean? = null,
+    var projectId: String,
+    var repoName: String,
+)