From 8290e97ef45032aab26f1c12781cb6f5a2149f70 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 13 Dec 2024 14:39:24 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=E5=8A=9F=E8=83=BD#2837?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/job/Constants.kt | 12 +- .../job/backup/config/DataBackupConfig.kt | 9 + .../bkrepo/job/backup/dao/BackupTaskDao.kt | 103 ++++++ .../bkrepo/job/backup/model/TBackupTask.kt | 99 ++++++ .../bkrepo/job/backup/pojo/BackupTaskState.kt | 38 ++ .../pojo/query/BackupFileReferenceInfo.kt | 39 ++ .../job/backup/pojo/query/BackupNodeInfo.kt | 63 ++++ .../backup/pojo/query/BackupProjectInfo.kt | 49 +++ .../backup/pojo/query/BackupRepositoryInfo.kt | 61 ++++ .../pojo/query/BackupStorageCredentials.kt | 45 +++ .../job/backup/pojo/record/BackupContext.kt | 53 +++ .../job/backup/pojo/record/BackupProgress.kt | 42 +++ .../pojo/setting/BackupConflictStrategy.kt | 43 +++ .../job/backup/pojo/setting/BackupCronType.kt | 39 ++ .../pojo/setting/BackupErrorStrategy.kt | 43 +++ .../pojo/setting/BackupExecutionPlan.kt | 48 +++ .../pojo/setting/BackupExecutionStrategy.kt | 48 +++ .../job/backup/pojo/setting/BackupSetting.kt | 54 +++ .../job/backup/pojo/task/BackupContent.kt | 59 ++++ .../bkrepo/job/backup/pojo/task/BackupTask.kt | 79 +++++ .../job/backup/pojo/task/BackupTaskRequest.kt | 39 ++ .../backup/pojo/task/ProjectContentInfo.kt | 68 ++++ .../job/backup/service/DataBackupService.kt | 15 + .../service/DataRecordsBackupService.kt | 36 ++ .../service/DataRecordsRestoreService.kt | 36 ++ .../job/backup/service/impl/BaseService.kt | 34 ++ .../service/impl/DataBackupServiceImpl.kt | 104 ++++++ .../impl/DataRecordsBackupServiceImpl.kt | 323 +++++++++++++++++ .../impl/DataRecordsRestoreServiceImpl.kt | 333 ++++++++++++++++++ .../bkrepo/job/backup/util/ZipFileUtil.kt | 62 ++++ .../tencent/bkrepo/job/config/JobConfig.kt | 2 + .../controller/user/UserBackupController.kt | 75 ++++ 32 files changed, 2152 insertions(+), 1 deletion(-) create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupProjectInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupRepositoryInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupStorageCredentials.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupContext.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupProgress.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupConflictStrategy.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupCronType.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupErrorStrategy.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionPlan.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionStrategy.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupSetting.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupContent.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTask.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTaskRequest.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/ProjectContentInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataBackupService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsBackupService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsRestoreService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/BaseService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsBackupServiceImpl.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsRestoreServiceImpl.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/util/ZipFileUtil.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/UserBackupController.kt diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt index e552aed048..0c5ff02ed9 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt @@ -103,10 +103,20 @@ val IGNORE_PROJECT_PREFIX_LIST = listOf(CODE_PROJECT_PREFIX, CLOSED_SOURCE_PREFI const val RESTORE = "RESTORE" const val SEPARATE = "SEPARATE" +const val PROJECT_COLLECTION_NAME = "project" +const val REPO_COLLECTION_NAME = "repository" +const val STORAGE_CREDENTIALS_COLLECTION_NAME = "storage_credentials" + const val PACKAGE_COLLECTION_NAME = "package" const val PACKAGE_VERSION_COLLECTION_NAME = "package_version" const val PACKAGE_DOWNLOADS_COLLECTION_NAME = "package_downloads" const val SEPARATION_TASK_COLLECTION_NAME = "separation_task" const val PACKAGE_VERSION = "version" -const val PACKAGE_DOWNLOAD_DATE = "date" \ No newline at end of file +const val PACKAGE_DOWNLOAD_DATE = "date" + +/** + * 记录备份 + */ +const val DATA_RECORDS_BACKUP = "DATA_BACKUP" +const val DATA_RECORDS_RESTORE = "DATA_RESTORE" \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt new file mode 100644 index 0000000000..9002156a74 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt @@ -0,0 +1,9 @@ +package com.tencent.bkrepo.job.backup.config + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties("backup") +data class DataBackupConfig( + // 当仓库容量大于阈值时,不进行备份 + var usageThreshold: Long = 1024 * 1024 * 1024, +) \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt new file mode 100644 index 0000000000..ab6fa92eb5 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.dao + +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.job.backup.model.TBackupTask +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import org.springframework.data.domain.PageRequest +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Repository +import java.time.LocalDateTime + +@Repository +class BackupTaskDao : SimpleMongoDao() { + + fun findTasksById(taskId: String): List { + val criteria = Criteria().and(TBackupTask::id.name).isEqualTo(taskId) + return find(Query(criteria)) + } + + fun find(state: String?, pageRequest: PageRequest): List { + val criteria = Criteria() + state?.let { criteria.and(TBackupTask::state.name).isEqualTo(it) } + return find(Query(criteria).with(pageRequest)) + } + + fun count(state: String?): Long { + val criteria = Criteria() + state?.let { criteria.and(TBackupTask::state.name).isEqualTo(it) } + return count(Query(criteria)) + } + + fun updateState( + taskId: String, + state: BackupTaskState, + startDate: LocalDateTime? = null, + endDate: LocalDateTime? = null + ) { + val criteria = Criteria().and(ID).isEqualTo(taskId) + val update = Update.update(TBackupTask::lastModifiedBy.name, SYSTEM_USER) + .set(TBackupTask::lastModifiedDate.name, LocalDateTime.now()) + .set(TBackupTask::state.name, state.name) + startDate?.let { update.set(TBackupTask::startDate.name, startDate) } + endDate?.let { update.set(TBackupTask::endDate.name, endDate) } + this.updateFirst(Query(criteria), update) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt new file mode 100644 index 0000000000..0dee9f2adb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.model + +import com.tencent.bkrepo.job.DATA_RECORDS_BACKUP +import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_STATE_IDX +import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_STATE_IDX_DEF +import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_TYPE_IDX +import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_TYPE_IDX_DEF +import com.tencent.bkrepo.job.backup.pojo.task.BackupContent +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.job.backup.pojo.setting.BackupSetting +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document +import java.time.LocalDateTime + +/** + * 备份与恢复任务 + */ +@Document("backup_task") +@CompoundIndexes( + CompoundIndex(name = TASK_STATE_IDX, def = TASK_STATE_IDX_DEF, background = true), + CompoundIndex(name = TASK_TYPE_IDX, def = TASK_TYPE_IDX_DEF, background = true), +) +data class TBackupTask( + val id: String? = null, + val name: String, + val storeLocation: String, + val type: String = DATA_RECORDS_BACKUP, + var content: BackupContent? = null, + val backupSetting: BackupSetting, + val state: String = BackupTaskState.PENDING.name, + val createdBy: String, + val createdDate: LocalDateTime, + val lastModifiedBy: String, + val lastModifiedDate: LocalDateTime, + val startDate: LocalDateTime? = null, + val endDate: LocalDateTime? = null, +) { + companion object { + const val TASK_STATE_IDX = "state_idx" + const val TASK_STATE_IDX_DEF = "{'state': 1}" + const val TASK_TYPE_IDX = "type_idx" + const val TASK_TYPE_IDX_DEF = "{'type': 1}" + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt new file mode 100644 index 0000000000..89cb2e152e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo + + +enum class BackupTaskState { + + PENDING, + + RUNNING, + + FINISHED, +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt new file mode 100644 index 0000000000..22649f57fb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +data class BackupFileReferenceInfo( + var id: String? = null, + var sha256: String, + var credentialsKey: String? = null, + var count: Long +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt new file mode 100644 index 0000000000..84cb18d420 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import java.time.LocalDateTime + +data class BackupNodeInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + var lastAccessDate: LocalDateTime? = null, + + var folder: Boolean, + var path: String, + var name: String, + var fullPath: String, + var size: Long, + var expireDate: LocalDateTime? = null, + var sha256: String? = null, + var md5: String? = null, + var deleted: LocalDateTime? = null, + var copyFromCredentialsKey: String? = null, + var copyIntoCredentialsKey: String? = null, + var metadata: MutableList? = null, + var clusterNames: Set? = null, + var nodeNum: Long? = null, + var archived: Boolean? = null, + var compressed: Boolean? = null, + var projectId: String, + var repoName: String, +) \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupProjectInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupProjectInfo.kt new file mode 100644 index 0000000000..b59bfe23cb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupProjectInfo.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.repository.pojo.project.ProjectMetadata +import java.time.LocalDateTime + +data class BackupProjectInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var name: String, + var displayName: String, + var description: String, + var metadata: List = emptyList(), + var credentialsKey: String? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupRepositoryInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupRepositoryInfo.kt new file mode 100644 index 0000000000..8e8e5f1b96 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupRepositoryInfo.kt @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import java.time.LocalDateTime + +data class BackupRepositoryInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var name: String, + var type: RepositoryType, + var category: RepositoryCategory, + var public: Boolean, + var description: String? = null, + var configuration: String, + var credentialsKey: String? = null, + var oldCredentialsKey: String? = null, + var display: Boolean = true, + + var projectId: String, + + var quota: Long? = null, + var used: Long? = null, + var clusterNames: Set? = null, + var deleted: LocalDateTime? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupStorageCredentials.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupStorageCredentials.kt new file mode 100644 index 0000000000..54006b6d62 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupStorageCredentials.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import java.time.LocalDateTime + +data class BackupStorageCredentials( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var credentials: String, + var region: String? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupContext.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupContext.kt new file mode 100644 index 0000000000..362421f95a --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupContext.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.record + +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.filesystem.FileSystemClient +import com.tencent.bkrepo.job.backup.model.TBackupTask +import com.tencent.bkrepo.job.backup.pojo.query.BackupNodeInfo +import com.tencent.bkrepo.job.backup.pojo.task.ProjectContentInfo +import java.nio.file.Path +import java.time.LocalDateTime + +class BackupContext( + val task: TBackupTask, +) { + lateinit var targertPath: Path + lateinit var tempClient: FileSystemClient + lateinit var startDate: LocalDateTime + var backupProgress = BackupProgress() + var type: String = task.type + var taskId: String = task.id!! + var currrentProjectInfo: ProjectContentInfo? = null + var currentProjectId: String? = null + var currentRepoName: String? = null + var currentStorageCredentials: StorageCredentials? = null + var currentNode: BackupNodeInfo? = null + var currentFile: String? = null +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupProgress.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupProgress.kt new file mode 100644 index 0000000000..6ea05fe885 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupProgress.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.record + +data class BackupProgress( + var success: Long = 0, + var failed: Long = 0, + var skipped: Long = 0, + /** + * 最后执行id + */ + var projectId: String? = null, + var repoName: String? = null, + var packageId: String? = null, + var versionId: String? = null, + var nodeId: String? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupConflictStrategy.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupConflictStrategy.kt new file mode 100644 index 0000000000..1cb18a6638 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupConflictStrategy.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 冲突处理策略 + */ +enum class BackupConflictStrategy { + /** + * 跳过文件 + */ + SKIP, + + /** + * 覆盖 + */ + OVERWRITE +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupCronType.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupCronType.kt new file mode 100644 index 0000000000..2e6b1a94ad --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupCronType.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 备份执行策略 + */ +enum class BackupCronType { + // 调度同步，指定时间/定时执行 + SCHEDULED, + + // 只执行一次，手动调用执行 + RUN_ONCE, +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupErrorStrategy.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupErrorStrategy.kt new file mode 100644 index 0000000000..5eded5eb09 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupErrorStrategy.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 出错时的处理策略 + */ +enum class BackupErrorStrategy { + /** + * 出错时继续执行 + */ + CONTINUE, + + /** + * 出错后本次任务失败 + */ + FAST_FAIL +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionPlan.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionPlan.kt new file mode 100644 index 0000000000..1e29ac845c --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionPlan.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +import java.time.LocalDateTime + +/** + * 执行计划 + */ +data class BackupExecutionPlan( + /** + * 执行一次，创建后立即执行 + */ + val executeImmediately: Boolean = true, + /** + * 执行一次，指定时间执行 + */ + val executeTime: LocalDateTime? = null, + /** + * cron表达式定时执行 + */ + val cronExpression: String? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionStrategy.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionStrategy.kt new file mode 100644 index 0000000000..30f882792a --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionStrategy.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 计划调度策略 + */ +enum class BackupExecutionStrategy { + /** + * 立即执行 + */ + IMMEDIATELY, + + /** + * 指定时间执行 + */ + SPECIFIED_TIME, + + /** + * cron表达式执行 + */ + CRON_EXPRESSION +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupSetting.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupSetting.kt new file mode 100644 index 0000000000..4cb79ba030 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupSetting.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 备份任务设置 + */ +data class BackupSetting( + /** + * 冲突解决策略 + */ + val conflictStrategy: BackupConflictStrategy = BackupConflictStrategy.SKIP, + /** + * 错误处理策略 + */ + val errorStrategy: BackupErrorStrategy = BackupErrorStrategy.CONTINUE, + /** + * 执行计划策略 + */ + val executionStrategy: BackupExecutionStrategy = BackupExecutionStrategy.IMMEDIATELY, + /** + * 执行计划 + */ + val executionPlan: BackupExecutionPlan = BackupExecutionPlan(), + /** + * 是否进行存储目录大小检测 + */ + val spaceCapCheck: Boolean = true, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupContent.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupContent.kt new file mode 100644 index 0000000000..bf599ffa6d --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupContent.kt @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.task + +data class BackupContent( + var projects: MutableList? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTask.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTask.kt new file mode 100644 index 0000000000..180917a76e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTask.kt @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.task + +import com.tencent.bkrepo.job.backup.model.TBackupTask +import com.tencent.bkrepo.job.backup.pojo.setting.BackupSetting +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty +import java.time.LocalDateTime + +@ApiModel("数据备份任务") +data class BackupTask( + @ApiModelProperty("ID") + val id: String? = null, + @ApiModelProperty("创建人") + val createdBy: String, + @ApiModelProperty("创建时间") + val createdDate: LocalDateTime, + @ApiModelProperty("最后修改人") + val lastModifiedBy: String, + @ApiModelProperty("最后修改时间") + val lastModifiedDate: LocalDateTime, + @ApiModelProperty("任务开始执行的时间") + val startDate: LocalDateTime? = null, + @ApiModelProperty("任务结束执行的时间") + val endDate: LocalDateTime? = null, + @ApiModelProperty("任务状态") + val state: String, + @ApiModelProperty("任务内容") + val content: BackupContent?, + @ApiModelProperty("存储路径") + val storeLocation: String, + @ApiModelProperty("任务配置") + val backupSetting: BackupSetting, + @ApiModelProperty("任务类型") + val type: String, +) { + companion object { + fun TBackupTask.toDto() = BackupTask( + id = id, + createdBy = createdBy, + createdDate = createdDate, + lastModifiedBy = lastModifiedBy, + lastModifiedDate = lastModifiedDate, + startDate = startDate, + endDate = endDate, + state = state, + content = content, + storeLocation = storeLocation, + backupSetting = backupSetting, + type = type + ) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTaskRequest.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTaskRequest.kt new file mode 100644 index 0000000000..87ea4af65d --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTaskRequest.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.task + +import com.tencent.bkrepo.job.DATA_RECORDS_BACKUP +import com.tencent.bkrepo.job.backup.pojo.setting.BackupSetting + +data class BackupTaskRequest( + val name: String, + val storeLocation: String, + val type: String = DATA_RECORDS_BACKUP, + var content: BackupContent? = null, + val backupSetting: BackupSetting = BackupSetting(), +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/ProjectContentInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/ProjectContentInfo.kt new file mode 100644 index 0000000000..c0493e6cb7 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/ProjectContentInfo.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.task + +/** + * 项目备份配置 + */ +data class ProjectContentInfo( + val projectId: String? = null, + val projectRegex: String? = null, + val repoList: List? = null, + val repoRegex: String? = null, + val excludeProjects: List? = null, + val excludeRepos: List? = null, +) + diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataBackupService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataBackupService.kt new file mode 100644 index 0000000000..cb35c5d579 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataBackupService.kt @@ -0,0 +1,15 @@ +package com.tencent.bkrepo.job.backup.service + +import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.job.backup.pojo.task.BackupTask +import com.tencent.bkrepo.job.backup.pojo.task.BackupTaskRequest +import org.springframework.data.domain.PageRequest + +interface DataBackupService { + + fun createTask(taskRequest: BackupTaskRequest): String + + fun executeTask(taskId: String) + + fun findTasks(state: String?, pageRequest: PageRequest): Page +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsBackupService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsBackupService.kt new file mode 100644 index 0000000000..26cf850c1f --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsBackupService.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service + +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext + +interface DataRecordsBackupService { + + fun projectDataBackup(context: BackupContext) + +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsRestoreService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsRestoreService.kt new file mode 100644 index 0000000000..596bdba77e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsRestoreService.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service + +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext + +interface DataRecordsRestoreService { + + fun projectDataRestore(context: BackupContext) + +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/BaseService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/BaseService.kt new file mode 100644 index 0000000000..edb9f3adb7 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/BaseService.kt @@ -0,0 +1,34 @@ +package com.tencent.bkrepo.job.backup.service.impl + +import com.tencent.bkrepo.common.storage.core.locator.HashFileLocator +import org.slf4j.LoggerFactory +import java.nio.file.Path +import java.nio.file.Paths + +open class BaseService { + + /** + * 生成随机文件路径 + * */ + fun generateRandomPath(sha256: String): String { + val fileLocator = HashFileLocator() + return fileLocator.locate(sha256) + } + + + /** + * 生成随机文件路径 + * */ + fun generateRandomPath(root: Path, sha256: String): Path { + return Paths.get(root.toFile().path, generateRandomPath(sha256)) + } + + companion object { + private val logger = LoggerFactory.getLogger(BaseService::class.java) + const val PROJECT_FILE_NAME = "project.json" + const val REPOSITORY_FILE_NAME = "repository.json" + const val NODE_FILE_NAME = "node.json" + const val ZIP_FILE_SUFFRIX = ".zip" + val FILE_LIST = listOf(PROJECT_FILE_NAME, REPOSITORY_FILE_NAME, NODE_FILE_NAME) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt new file mode 100644 index 0000000000..ca8e8d25f1 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt @@ -0,0 +1,104 @@ +package com.tencent.bkrepo.job.backup.service.impl + +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.mongo.util.Pages +import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.job.DATA_RECORDS_BACKUP +import com.tencent.bkrepo.job.DATA_RECORDS_RESTORE +import com.tencent.bkrepo.job.backup.dao.BackupTaskDao +import com.tencent.bkrepo.job.backup.model.TBackupTask +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.task.BackupTask +import com.tencent.bkrepo.job.backup.pojo.task.BackupTask.Companion.toDto +import com.tencent.bkrepo.job.backup.pojo.task.BackupTaskRequest +import com.tencent.bkrepo.job.backup.service.DataBackupService +import com.tencent.bkrepo.job.backup.service.DataRecordsBackupService +import com.tencent.bkrepo.job.backup.service.DataRecordsRestoreService +import org.slf4j.LoggerFactory +import org.springframework.data.domain.PageRequest +import org.springframework.stereotype.Service +import java.io.FileNotFoundException +import java.nio.file.Paths +import java.time.LocalDateTime + +@Service +class DataBackupServiceImpl( + private val backupTaskDao: BackupTaskDao, + private val dataRecordsBackupService: DataRecordsBackupService, + private val dataRecordsRestoreService: DataRecordsRestoreService +) : DataBackupService { + override fun createTask(taskRequest: BackupTaskRequest): String { + contentCheck(taskRequest) + val task = buildBackupTask(taskRequest) + return backupTaskDao.save(task).id!! + } + + override fun executeTask(taskId: String) { + val records = backupTaskDao.findTasksById(taskId) + val task = records.firstOrNull() ?: throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "taskId") + if (task.state != BackupTaskState.PENDING.name) + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "state") + if (task.type == DATA_RECORDS_BACKUP) { + val context = BackupContext(task = task) + dataRecordsBackupService.projectDataBackup(context) + } else { + val context = BackupContext(task = task) + dataRecordsRestoreService.projectDataRestore(context) + } + } + + override fun findTasks(state: String?, pageRequest: PageRequest): Page { + val count = backupTaskDao.count(state) + val records = backupTaskDao.find(state, pageRequest).map { it.toDto() } + return Pages.ofResponse(pageRequest, count, records) + } + + private fun contentCheck(request: BackupTaskRequest) { + with(request) { + try { + val targetFile = Paths.get(storeLocation).toFile() + if (!targetFile.exists()) throw FileNotFoundException(storeLocation) + } catch (e: Exception) { + logger.warn("backup store location [$storeLocation] is illegal!") + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, BackupTaskRequest::storeLocation.name) + } + when (type) { + DATA_RECORDS_BACKUP -> { + if (content == null || content!!.projects.isNullOrEmpty()) { + logger.warn("backup content [$content] is illegal!") + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, BackupTaskRequest::content.name) + } + return + } + DATA_RECORDS_RESTORE -> { + return + } + else -> { + logger.warn("task type [$type] is illegal!") + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, BackupTaskRequest::type.name) + } + } + } + } + + private fun buildBackupTask(request: BackupTaskRequest): TBackupTask { + return TBackupTask( + name = request.name, + storeLocation = request.storeLocation, + type = request.type, + content = request.content, + createdBy = SecurityUtils.getUserId(), + createdDate = LocalDateTime.now(), + lastModifiedDate = LocalDateTime.now(), + lastModifiedBy = SecurityUtils.getUserId(), + backupSetting = request.backupSetting + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(DataBackupServiceImpl::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsBackupServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsBackupServiceImpl.kt new file mode 100644 index 0000000000..42263e37be --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsBackupServiceImpl.kt @@ -0,0 +1,323 @@ +package com.tencent.bkrepo.job.backup.service.impl + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.util.EscapeUtils +import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.mongo.constant.ID_IDX +import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.filesystem.FileSystemClient +import com.tencent.bkrepo.common.storage.message.StorageErrorException +import com.tencent.bkrepo.common.storage.message.StorageMessageCode +import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 +import com.tencent.bkrepo.job.BATCH_SIZE +import com.tencent.bkrepo.job.DELETED_DATE +import com.tencent.bkrepo.job.NAME +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.PROJECT_COLLECTION_NAME +import com.tencent.bkrepo.job.REPO_COLLECTION_NAME +import com.tencent.bkrepo.job.STORAGE_CREDENTIALS_COLLECTION_NAME +import com.tencent.bkrepo.job.backup.dao.BackupTaskDao +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.job.backup.pojo.query.BackupNodeInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupProjectInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupRepositoryInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupStorageCredentials +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.task.ProjectContentInfo +import com.tencent.bkrepo.job.backup.service.DataRecordsBackupService +import com.tencent.bkrepo.job.backup.util.ZipFileUtil +import com.tencent.bkrepo.job.separation.util.SeparationUtils +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import org.bson.types.ObjectId +import org.slf4j.LoggerFactory +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Service +import java.nio.file.Files +import java.nio.file.Paths +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +@Service +class DataRecordsBackupServiceImpl( + private val mongoTemplate: MongoTemplate, + private val storageManager: StorageManager, + private val backupTaskDao: BackupTaskDao, +) : DataRecordsBackupService, BaseService() { + override fun projectDataBackup(context: BackupContext) { + with(context) { + startDate = LocalDateTime.now() + backupTaskDao.updateState(taskId, BackupTaskState.RUNNING, startDate) + // TODO 需要进行磁盘判断 + // TODO 需要进行仓库用量判断 + if (task.content == null || task.content!!.projects.isNullOrEmpty()) return + initStorage(context) + for (projectFilterInfo in task.content!!.projects!!) { + if (!checkProjectParams(projectFilterInfo)) continue + val criteria = buildProjectCriteria(projectFilterInfo) + context.currrentProjectInfo = projectFilterInfo + queryResult(criteria, BackupProjectInfo::class.java, PROJECT_COLLECTION_NAME, context) + } + // TODO 最后需要进行压缩 + backupTaskDao.updateState(taskId, BackupTaskState.FINISHED, endDate = LocalDateTime.now()) + ZipFileUtil.compressDirectory(targertPath.toString(), buildZipFileName(context)) + } + } + + private fun initStorage(context: BackupContext) { + val currentDateStr = context.startDate.format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")) + val path = Paths.get(context.task.storeLocation, context.task.name, currentDateStr) + if (!Files.exists(path)) { + Files.createDirectories(path) + } + context.targertPath = path + context.tempClient = FileSystemClient(path) + } + + private fun buildZipFileName(context: BackupContext): String { + val currentDateStr = context.startDate.format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")) + val path = context.task.name + StringPool.DASH + currentDateStr + ZIP_FILE_SUFFRIX + return Paths.get(context.task.storeLocation, context.task.name, path).toString() + } + + private fun repoDataBackup(context: BackupContext) { + val criteria = buildRepositoryCriteria(context.currentProjectId!!, context.currrentProjectInfo!!) + queryResult(criteria, BackupRepositoryInfo::class.java, REPO_COLLECTION_NAME, context) + } + + private fun nodeDataBackup(context: BackupContext) { + val criteria = buildNodeCriteria(context.currentProjectId!!, context.currentRepoName!!) + val collectionName = SeparationUtils.getNodeCollectionName(context.currentProjectId!!) + queryResult(criteria, BackupNodeInfo::class.java, collectionName, context) + } + + private inline fun queryResult( + criteria: Criteria, + clazz: Class, + collectionName: String, + context: BackupContext + ) { + val pageSize = BATCH_SIZE + var querySize: Int + var lastId = ObjectId(MIN_OBJECT_ID) + do { + val query = Query(criteria) + .addCriteria(Criteria.where(ID).gt(lastId)) + .limit(BATCH_SIZE) + .with(Sort.by(ID).ascending()) + val data = mongoTemplate.find(query, clazz, collectionName) + if (data.isEmpty()) { + break + } + val dataLastId = processData(data, context) + querySize = data.size + lastId = ObjectId(dataLastId) + } while (querySize == pageSize) + } + + private inline fun processData(data: List, context: BackupContext): String { + when (T::class) { + BackupProjectInfo::class -> { + data.forEach { + val record = it as BackupProjectInfo + context.currentProjectId = record.name + context.currentRepoName = null + storeData(record, context) + repoDataBackup(context) + } + return (data.last() as BackupProjectInfo).id!! + } + BackupRepositoryInfo::class -> { + data.forEach { + val record = it as BackupRepositoryInfo + context.currentRepoName = record.name + context.currentStorageCredentials = findStorageCredentials(record.credentialsKey) + storeData(record, context) + nodeDataBackup(context) + } + return (data.last() as BackupRepositoryInfo).id!! + } + BackupNodeInfo::class -> { + data.forEach { + val record = it as BackupNodeInfo + context.currentNode = record + storeData(record, context) + storeRealFile(context) + } + return (data.last() as BackupNodeInfo).id!! + } + else -> return StringPool.EMPTY + } + } + + // TODO 全存储在一个文件中，当数据过多会导致内容过大 + private inline fun storeData(data: T, context: BackupContext) { + val fileName = when (T::class) { + BackupProjectInfo::class -> PROJECT_FILE_NAME + BackupRepositoryInfo::class -> REPOSITORY_FILE_NAME + BackupNodeInfo::class -> NODE_FILE_NAME + else -> return + } + try { + context.tempClient.touch(StringPool.EMPTY, fileName) + logger.info("Success to create file [$fileName]") + val dataStr = data!!.toJsonString().replace(System.lineSeparator(), "") + val inputStream = dataStr.byteInputStream() + val size = dataStr.length.toLong() + context.tempClient.append(StringPool.EMPTY, fileName, inputStream, size) + val lineEndStr = "\n" + context.tempClient.append( + StringPool.EMPTY, + fileName, + lineEndStr.byteInputStream(), + lineEndStr.length.toLong() + ) + logger.info("Success to append file [$fileName]") + } catch (exception: Exception) { + logger.error("Failed to create file", exception) + throw StorageErrorException(StorageMessageCode.STORE_ERROR) + // TODO 异常该如何处理 + } + } + + private fun findStorageCredentials(currentCredentialsKey: String?): StorageCredentials? { + val backupStorageCredentials = currentCredentialsKey?.let { + mongoTemplate.findOne( + Query(Criteria.where(ID_IDX).isEqualTo(it)), + BackupStorageCredentials::class.java, + STORAGE_CREDENTIALS_COLLECTION_NAME + ) + } + return backupStorageCredentials?.let { convert(backupStorageCredentials) } + } + + private fun storeRealFile(context: BackupContext) { + with(context) { + if (currentNode!!.folder || currentNode!!.sha256 == FAKE_SHA256 || currentNode!!.sha256.isNullOrEmpty()) return + val nodeDetail = convertToDetail(currentNode) + val dir = generateRandomPath(currentNode!!.sha256!!) + if (tempClient.exist(dir, currentNode!!.sha256!!)) { + logger.info("real file already exist [${currentNode!!.sha256}]") + return + } + // 存储异常如何处理 + storageManager.loadFullArtifactInputStream(nodeDetail, currentStorageCredentials)?.use { + try { + tempClient.store(dir, currentNode!!.sha256!!, it, currentNode!!.size) + logger.info("Success to store real file [${currentNode!!.sha256}]") + } catch (exception: Exception) { + logger.error("Failed to store real file", exception) + throw StorageErrorException(StorageMessageCode.STORE_ERROR) + } + } + } + } + + + private fun checkProjectParams(project: ProjectContentInfo?): Boolean { + return !(project != null && + project.projectRegex.isNullOrEmpty() && + project.projectId.isNullOrEmpty() && + project.excludeProjects.isNullOrEmpty()) + } + + private fun buildProjectCriteria(project: ProjectContentInfo): Criteria { + with(project) { + val criteria = Criteria() + if (projectId.isNullOrEmpty()) { + if (projectRegex.isNullOrEmpty()) { + criteria.and(NAME).nin(excludeProjects!!) + } else { + criteria.and(NAME).regex(".*${EscapeUtils.escapeRegex(projectRegex)}.*") + } + } else { + criteria.and(NAME).isEqualTo(projectId) + } + return criteria + } + } + + private fun buildRepositoryCriteria(projectId: String, project: ProjectContentInfo): Criteria { + val criteria = Criteria.where(PROJECT).isEqualTo(projectId) + if (project.repoList.isNullOrEmpty()) { + if (project.repoRegex.isNullOrEmpty()) { + if (!project.excludeRepos.isNullOrEmpty()) { + criteria.and(NAME).nin(project.excludeRepos) + } + } else { + criteria.and(NAME).regex(".*${EscapeUtils.escapeRegex(project.repoRegex)}.*") + } + } else { + criteria.and(NAME).`in`(project.repoList) + } + return criteria + } + + + private fun buildNodeCriteria(projectId: String, repoName: String): Criteria { + return Criteria.where(PROJECT).isEqualTo(projectId) + .and(REPO_NAME).isEqualTo(repoName) + .and(DELETED_DATE).isEqualTo(null) + } + + private fun convert(tNode: BackupNodeInfo?): NodeInfo? { + return tNode?.let { + val metadata = toMap(it.metadata) + NodeInfo( + id = it.id, + createdBy = it.createdBy, + createdDate = it.createdDate.format(DateTimeFormatter.ISO_DATE_TIME), + lastModifiedBy = it.lastModifiedBy, + lastModifiedDate = it.lastModifiedDate.format(DateTimeFormatter.ISO_DATE_TIME), + projectId = it.projectId, + repoName = it.repoName, + folder = it.folder, + path = it.path, + name = it.name, + fullPath = it.fullPath, + size = if (it.size < 0L) 0L else it.size, + nodeNum = it.nodeNum?.let { nodeNum -> + if (nodeNum < 0L) 0L else nodeNum + }, + sha256 = it.sha256, + md5 = it.md5, + metadata = metadata, + nodeMetadata = it.metadata, + copyFromCredentialsKey = it.copyFromCredentialsKey, + copyIntoCredentialsKey = it.copyIntoCredentialsKey, + deleted = it.deleted?.format(DateTimeFormatter.ISO_DATE_TIME), + lastAccessDate = it.lastAccessDate?.format(DateTimeFormatter.ISO_DATE_TIME), + clusterNames = it.clusterNames, + archived = it.archived, + compressed = it.compressed, + ) + } + } + + fun convertToDetail(tNode: BackupNodeInfo?): NodeDetail? { + return convert(tNode)?.let { NodeDetail(it) } + } + + fun toMap(metadataList: List?): Map { + return metadataList?.associate { it.key to it.value }.orEmpty() + } + + + private fun convert(credentials: BackupStorageCredentials): StorageCredentials { + return credentials.credentials.readJsonString().apply { this.key = credentials.id } + } + + companion object { + private val logger = LoggerFactory.getLogger(DataRecordsBackupServiceImpl::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsRestoreServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsRestoreServiceImpl.kt new file mode 100644 index 0000000000..b2893806d3 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsRestoreServiceImpl.kt @@ -0,0 +1,333 @@ +package com.tencent.bkrepo.job.backup.service.impl + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.util.JsonUtils +import com.tencent.bkrepo.common.artifact.api.toArtifactFile +import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.common.storage.filesystem.FileSystemClient +import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 +import com.tencent.bkrepo.job.DELETED_DATE +import com.tencent.bkrepo.job.NAME +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.PROJECT_COLLECTION_NAME +import com.tencent.bkrepo.job.REPO_COLLECTION_NAME +import com.tencent.bkrepo.job.backup.dao.BackupTaskDao +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.job.backup.pojo.query.BackupFileReferenceInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupNodeInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupProjectInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupRepositoryInfo +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.DataRecordsRestoreService +import com.tencent.bkrepo.job.backup.util.ZipFileUtil +import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils +import com.tencent.bkrepo.job.separation.pojo.query.NodeDetailInfo +import com.tencent.bkrepo.job.separation.util.SeparationUtils +import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import org.slf4j.LoggerFactory +import org.springframework.dao.DuplicateKeyException +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Service +import java.io.File +import java.io.FileNotFoundException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import kotlin.io.path.name +import kotlin.streams.toList + +@Service +class DataRecordsRestoreServiceImpl( + private val mongoTemplate: MongoTemplate, + private val storageService: StorageService, + private val backupTaskDao: BackupTaskDao, +) : DataRecordsRestoreService, BaseService() { + override fun projectDataRestore(context: BackupContext) { + with(context) { + startDate = LocalDateTime.now() + backupTaskDao.updateState(taskId, BackupTaskState.RUNNING, startDate) + val unZipTempFolder = buildTargetFolder(context) + ZipFileUtil.decompressFile(task.storeLocation, unZipTempFolder.toString()) + initStorage(unZipTempFolder, context) + processFiles(context) + backupTaskDao.updateState(taskId, BackupTaskState.FINISHED, endDate = LocalDateTime.now()) + } + } + + private fun buildTargetFolder(context: BackupContext): Path { + val currentDateStr = context.startDate.format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")) + val path = Paths.get(context.task.storeLocation) + if (!Files.exists(path)) { + throw FileNotFoundException(context.task.storeLocation) + } + val tempFolder = path.name.removeSuffix(ZIP_FILE_SUFFRIX) + StringPool.DASH + currentDateStr + return Paths.get(path.parent.toString(), tempFolder) + } + + private fun initStorage(unZipTempFolder: Path, context: BackupContext) { + if (!Files.exists(unZipTempFolder)) { + throw FileNotFoundException(unZipTempFolder.toString()) + } + val subdirectories = Files.list(unZipTempFolder) + .filter { Files.isDirectory(it) } + .toList() + val targetFolder = subdirectories.firstOrNull() ?: throw FileNotFoundException(unZipTempFolder.toString()) + context.targertPath = targetFolder + context.tempClient = FileSystemClient(targetFolder) + } + + private fun processFiles(context: BackupContext) { + for (file in FILE_LIST) { + when (file) { + PROJECT_FILE_NAME -> { + context.currentFile = Paths.get(context.targertPath.toString(), PROJECT_FILE_NAME).toString() + loadAndStoreRecord(BackupProjectInfo::class.java, context) + } + REPOSITORY_FILE_NAME -> { + context.currentFile = Paths.get(context.targertPath.toString(), REPOSITORY_FILE_NAME).toString() + loadAndStoreRecord(BackupRepositoryInfo::class.java, context) + } + NODE_FILE_NAME -> { + context.currentFile = Paths.get(context.targertPath.toString(), NODE_FILE_NAME).toString() + loadAndStoreRecord(BackupNodeInfo::class.java, context) + } + else -> continue + } + } + } + + private inline fun loadAndStoreRecord(clazz: Class, context: BackupContext) { + with(context) { + if (!Files.exists(Paths.get(currentFile))) { + logger.error("$currentFile not exist!") + return + } + val file = File(currentFile) + file.forEachLine { line -> + val record = JsonUtils.objectMapper.readValue(line, clazz) + processData(record, context) + } + } + } + + private inline fun processData(data: T, context: BackupContext) { + when (T::class) { + BackupProjectInfo::class -> { + val record = data as BackupProjectInfo + val existRecord = findExistProject(record) + storeData(existRecord, record, context) + } + BackupRepositoryInfo::class -> { + val record = data as BackupRepositoryInfo + val existRecord = findExistRepository(record) + storeData(existRecord, record, context) + } + BackupNodeInfo::class -> { + val record = data as BackupNodeInfo + val existRecord = findExistNode(record) + storeData(existRecord, record, context) + } + else -> return + } + } + + //TODO 如果项目仓库恢复失败, 节点恢复的时候需要禁止 + private inline fun storeData(existedData: T?, newData: T, context: BackupContext) { + if (existedData != null && context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + try { + when (T::class) { + BackupProjectInfo::class -> { + val record = newData as BackupProjectInfo + if (existedData != null) { + updateExistProject(record) + } else { + record.id = null + mongoTemplate.save(record, PROJECT_COLLECTION_NAME) + logger.info("Create project ${record.name} success!") + } + } + BackupRepositoryInfo::class -> { + val record = newData as BackupRepositoryInfo + if (existedData != null) { + // TODO 仓库涉及存储, 不能简单更新 + updateExistRepo(record) + } else { + record.id = null + mongoTemplate.save(record, REPO_COLLECTION_NAME) + logger.info("Create repo ${record.projectId}|${record.name} success!") + } + } + BackupNodeInfo::class -> { + val record = newData as BackupNodeInfo + val existRecord = existedData as BackupNodeInfo? + val collectionName = SeparationUtils.getNodeCollectionName(record.projectId) + if (existRecord != null) { + removeExistNode(existRecord, collectionName) + } else { + record.id = null + mongoTemplate.save(record, collectionName) + logger.info("Create node ${record.fullPath} in ${record.projectId}|${record.repoName} success!") + } + uploadFile(record, context) + } + else -> return + } + } catch (e: DuplicateKeyException) { + logger.warn("insert data occurred DuplicateKeyException") + } + + } + + private fun updateExistProject(projectInfo: BackupProjectInfo) { + val projectQuery = Query(Criteria.where(NAME).isEqualTo(projectInfo.name)) + // 逻辑删除， 同时删除索引 + val update = Update() + .set(BackupProjectInfo::lastModifiedBy.name, projectInfo.lastModifiedBy) + .set(BackupProjectInfo::createdBy.name, projectInfo.createdBy) + .set(BackupProjectInfo::createdDate.name, projectInfo.createdDate) + .set(BackupProjectInfo::lastModifiedDate.name, projectInfo.lastModifiedDate) + .set(BackupProjectInfo::displayName.name, projectInfo.displayName) + .set(BackupProjectInfo::description.name, projectInfo.description) + .set(BackupProjectInfo::metadata.name, projectInfo.metadata) + .set(BackupProjectInfo::credentialsKey.name, projectInfo.credentialsKey) + + val updateResult = mongoTemplate.updateFirst(projectQuery, update, PROJECT_COLLECTION_NAME) + if (updateResult.modifiedCount != 1L) { + logger.error("update exist project failed with name ${projectInfo.name} ") + } else { + logger.info("update exist project success with name ${projectInfo.name}") + } + } + + private fun updateExistRepo(repoInfo: BackupRepositoryInfo) { + val repoQuery = Query( + Criteria.where(NAME).isEqualTo(repoInfo.name) + .and(PROJECT).isEqualTo(repoInfo.projectId) + .and(DELETED_DATE).isEqualTo(null) + ) + // 逻辑删除， 同时删除索引 + val update = Update() + .set(BackupRepositoryInfo::lastModifiedBy.name, repoInfo.lastModifiedBy) + .set(BackupRepositoryInfo::createdBy.name, repoInfo.createdBy) + .set(BackupRepositoryInfo::createdDate.name, repoInfo.createdDate) + .set(BackupRepositoryInfo::lastModifiedDate.name, repoInfo.lastModifiedDate) + .set(BackupRepositoryInfo::type.name, repoInfo.type) + .set(BackupRepositoryInfo::description.name, repoInfo.description) + .set(BackupRepositoryInfo::category.name, repoInfo.category) + .set(BackupRepositoryInfo::public.name, repoInfo.public) + .set(BackupRepositoryInfo::configuration.name, repoInfo.configuration) + .set(BackupRepositoryInfo::oldCredentialsKey.name, repoInfo.oldCredentialsKey) + .set(BackupRepositoryInfo::display.name, repoInfo.display) + .set(BackupRepositoryInfo::clusterNames.name, repoInfo.clusterNames) + .set(BackupRepositoryInfo::credentialsKey.name, repoInfo.credentialsKey) + + // TODO quote和used需要进行事实计算更新 + + val updateResult = mongoTemplate.updateFirst(repoQuery, update, REPO_COLLECTION_NAME) + + if (updateResult.modifiedCount != 1L) { + logger.error("update exist repo failed with name ${repoInfo.projectId}|${repoInfo.name}") + } else { + logger.info("update exist repo success with name ${repoInfo.projectId}|${repoInfo.name}") + } + } + + private fun removeExistNode( + nodeInfo: BackupNodeInfo, + nodeCollectionName: String + ) { + val nodeQuery = Query(Criteria.where(ID).isEqualTo(nodeInfo.id)) + // 逻辑删除， 同时删除索引 + val update = Update() + .set(NodeDetailInfo::lastModifiedBy.name, SYSTEM_USER) + .set(NodeDetailInfo::deleted.name, LocalDateTime.now()) + val updateResult = mongoTemplate.updateFirst(nodeQuery, update, nodeCollectionName) + if (updateResult.modifiedCount != 1L) { + logger.error( + "delete exist node failed with id ${nodeInfo.id} " + + "and fullPath ${nodeInfo.fullPath} in ${nodeInfo.projectId}|${nodeInfo.repoName}" + ) + } else { + logger.info( + "delete exist node success with id ${nodeInfo.id} " + + "and fullPath ${nodeInfo.fullPath} in ${nodeInfo.projectId}|${nodeInfo.repoName}" + ) + } + } + + private fun findExistProject(record: BackupProjectInfo): BackupProjectInfo? { + val existProjectQuery = Query( + Criteria.where(BackupProjectInfo::name.name).isEqualTo(record.name) + ) + return mongoTemplate.findOne(existProjectQuery, BackupProjectInfo::class.java, PROJECT_COLLECTION_NAME) + } + + private fun findExistRepository(record: BackupRepositoryInfo): BackupRepositoryInfo? { + val existRepoQuery = Query( + Criteria.where(BackupRepositoryInfo::name.name).isEqualTo(record.name) + .and(BackupRepositoryInfo::projectId.name).isEqualTo(record.projectId) + ) + return mongoTemplate.findOne(existRepoQuery, BackupRepositoryInfo::class.java, REPO_COLLECTION_NAME) + } + + private fun findExistNode(record: BackupNodeInfo): BackupNodeInfo? { + val existNodeQuery = Query( + Criteria.where(BackupNodeInfo::repoName.name).isEqualTo(record.repoName) + .and(BackupNodeInfo::projectId.name).isEqualTo(record.projectId) + .and(BackupNodeInfo::fullPath.name).isEqualTo(record.fullPath) + .and(BackupNodeInfo::deleted.name).isEqualTo(null) + ) + val collectionName = SeparationUtils.getNodeCollectionName(record.projectId) + return mongoTemplate.findOne(existNodeQuery, BackupNodeInfo::class.java, collectionName) + } + + fun uploadFile(record: BackupNodeInfo, context: BackupContext) { + if (!sha256Check(record.folder, record.sha256)) return + val repo = RepositoryCommonUtils.getRepositoryDetail(record.projectId, record.repoName) + val filePath = generateRandomPath(context.targertPath, record.sha256!!) + val artifactFile = filePath.toFile().toArtifactFile() + // TODO 增加重试已经异常捕获 + storageService.store(record.sha256!!, artifactFile, repo.storageCredentials) + // 只有新增的时候才去尽显文件索引新增 + increment(record.sha256!!, repo.storageCredentials?.key) + } + + + fun sha256Check(folder: Boolean, sha256: String?): Boolean { + // 增加对应cold node 的文件引用 + if (folder || sha256.isNullOrBlank() || sha256 == FAKE_SHA256) { + return false + } + return true + } + + + fun increment(sha256: String, credentialsKey: String?) { + val collectionName = SeparationUtils.getFileReferenceCollectionName(sha256) + val criteria = Criteria.where(BackupFileReferenceInfo::sha256.name).`is`(sha256) + .and(BackupFileReferenceInfo::credentialsKey.name).`is`(credentialsKey) + val query = Query(criteria) + val update = Update().inc(BackupFileReferenceInfo::count.name, 1) + try { + mongoTemplate.upsert(query, update, collectionName) + } catch (exception: DuplicateKeyException) { + // retry because upsert operation is not atomic + mongoTemplate.upsert(query, update, collectionName) + } + logger.info("Increment node reference of file [$sha256] on credentialsKey [$credentialsKey].") + } + + companion object { + private val logger = LoggerFactory.getLogger(DataRecordsRestoreServiceImpl::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/util/ZipFileUtil.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/util/ZipFileUtil.kt new file mode 100644 index 0000000000..88345ed2d4 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/util/ZipFileUtil.kt @@ -0,0 +1,62 @@ +package com.tencent.bkrepo.job.backup.util + +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream + +object ZipFileUtil { + + fun compressDirectory(directoryPath: String, zipFilePath: String) { + val directory = File(directoryPath) + val zipFile = File(zipFilePath) + + val zipOutputStream = ZipOutputStream(FileOutputStream(zipFile)) + + compress(directory, directory.name, zipOutputStream) + + zipOutputStream.close() + } + + fun decompressFile(zipFilePath: String, destinationFolder: String) { + val zipFile = ZipFile(zipFilePath) + val entries = zipFile.entries() + while (entries.hasMoreElements()) { + val entry = entries.nextElement() + val entryDestination = File(destinationFolder, entry.name) + if (entry.isDirectory) { + entryDestination.mkdirs() + } else { + entryDestination.parentFile.mkdirs() + val inputStream = zipFile.getInputStream(entry) + val outputStream = FileOutputStream(entryDestination) + inputStream.copyTo(outputStream) + inputStream.close() + outputStream.close() + } + } + } + + private fun compress(file: File, fileName: String, zipOutputStream: ZipOutputStream) { + if (file.isDirectory) { + val files = file.listFiles() + for (childFile in files) { + compress(childFile, fileName + File.separator + childFile.name, zipOutputStream) + } + } else { + val buffer = ByteArray(1024) + val fileInputStream = FileInputStream(file) + + zipOutputStream.putNextEntry(ZipEntry(fileName)) + + var length: Int + while (fileInputStream.read(buffer).also { length = it } > 0) { + zipOutputStream.write(buffer, 0, length) + } + + fileInputStream.close() + } + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt index b0edcd330d..c2d8c6c85a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.config import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.job.backup.config.DataBackupConfig import com.tencent.bkrepo.job.executor.BlockThreadPoolTaskExecutorDecorator import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.separation.config.DataSeparationConfig @@ -48,6 +49,7 @@ import java.util.function.Consumer JobProperties::class, MigrateRepoStorageProperties::class, DataSeparationConfig::class, + DataBackupConfig::class ) class JobConfig { @Bean diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/UserBackupController.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/UserBackupController.kt new file mode 100644 index 0000000000..8855f94b5b --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/UserBackupController.kt @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.controller.user + +import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER +import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE +import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.mongo.util.Pages +import com.tencent.bkrepo.common.security.permission.Principal +import com.tencent.bkrepo.common.security.permission.PrincipalType +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.job.backup.pojo.task.BackupTask +import com.tencent.bkrepo.job.backup.pojo.task.BackupTaskRequest +import com.tencent.bkrepo.job.backup.service.DataBackupService +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/job/backup") +@Principal(type = PrincipalType.ADMIN) +class UserBackupController( + private val dataBackupService: DataBackupService, +) { + @PostMapping + fun createTask(@RequestBody request: BackupTaskRequest): Response { + return ResponseBuilder.success(dataBackupService.createTask(request)) + } + + @PostMapping("/execute/{taskId}") + fun executeTask(@PathVariable("taskId") taskId: String): Response { + dataBackupService.executeTask(taskId) + return ResponseBuilder.success() + } + + @GetMapping("/tasks") + fun tasks( + @RequestParam(required = false) state: String? = null, + @RequestParam(required = false, defaultValue = "$DEFAULT_PAGE_NUMBER") pageNumber: Int = DEFAULT_PAGE_NUMBER, + @RequestParam(required = false, defaultValue = "$DEFAULT_PAGE_SIZE") pageSize: Int = DEFAULT_PAGE_SIZE, + ): Response> { + val page = dataBackupService.findTasks(state, Pages.ofRequest(pageNumber, pageSize)) + return ResponseBuilder.success(page) + } +}