From a883e5299f6a9d9a590c06e7fec402a883910459 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:07:17 +0800 Subject: [PATCH 01/51] Create entry.sh-old --- entry.sh-old | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 entry.sh-old diff --git a/entry.sh-old b/entry.sh-old new file mode 100644 index 0000000..9af1e7e --- /dev/null +++ b/entry.sh-old @@ -0,0 +1,46 @@ +#!/bin/sh + +echo "Starting container ..." + +if [ -n "${NFS_TARGET}" ]; then + echo "Mounting NFS based on NFS_TARGET: ${NFS_TARGET}" + mount -o nolock -v ${NFS_TARGET} /mnt/restic +fi + +restic snapshots ${RESTIC_INIT_ARGS} &>/dev/null +status=$? +echo "Check Repo status $status" + +if [ $status != 0 ]; then + echo "Restic repository '${RESTIC_REPOSITORY}' does not exists. Running restic init." + restic init ${RESTIC_INIT_ARGS} + + init_status=$? + echo "Repo init status $init_status" + + if [ $init_status != 0 ]; then + echo "Failed to init the repository: '${RESTIC_REPOSITORY}'" + exit 1 + fi +fi + + + +echo "Setup backup cron job with cron expression BACKUP_CRON: ${BACKUP_CRON}" +echo "${BACKUP_CRON} /usr/bin/flock -n /var/run/backup.lock /bin/backup >> /var/log/cron.log 2>&1" > /var/spool/cron/crontabs/root + +# If CHECK_CRON is set we will enable automatic backup checking +if [ -n "${CHECK_CRON}" ]; then + echo "Setup check cron job with cron expression CHECK_CRON: ${CHECK_CRON}" + echo "${CHECK_CRON} /usr/bin/flock -n /var/run/backup.lock /bin/check >> /var/log/cron.log 2>&1" >> /var/spool/cron/crontabs/root +fi + +# Make sure the file exists before we start tail +touch /var/log/cron.log + +# start the cron deamon +crond + +echo "Container started." + +exec "$@" From 04c50ab20d4f6ea415cc87735ec45b87cf85a7d9 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:32:17 +0800 Subject: [PATCH 02/51] Create package.json --- package.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 package.json diff --git a/package.json b/package.json new file mode 100644 index 0000000..9c506e8 --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "npmscript", + "version": "1.0.0", + "description": "A Node.js application that connects to Datebase", + "main": "dump.js", + "scripts": { + "start": "node dump.js", + "dump": "node dump.js" + }, + "author": "emengweb", + "license": "MIT", + "dependencies": { + "mongodb": "^4.10.0", + "mysql": "^2.18.1", + "pg": "^8.8.0", + "moment": "^2.29.4" + } +} From 0224f69b1a812f5f70a77ec4c268057620a36948 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:04:01 +0800 Subject: [PATCH 03/51] Update package.json --- package.json | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 9c506e8..ca8ce24 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,20 @@ { - "name": "npmscript", - "version": "1.0.0", - "description": "A Node.js application that connects to Datebase", - "main": "dump.js", - "scripts": { - "start": "node dump.js", - "dump": "node dump.js" - }, - "author": "emengweb", - "license": "MIT", - "dependencies": { - "mongodb": "^4.10.0", - "mysql": "^2.18.1", - "pg": "^8.8.0", - "moment": "^2.29.4" - } + "name": "npmscript", + "version": "1.0.0", + "description": "A Node.js application that connects to Datebase", + "main": "dump.js", + "scripts": { + "start": "node dump.js", + "dump": "node dump.js" + }, + "author": "emengweb", + "license": "MIT", + "dependencies": { + "moment": "^2.29.4", + "mongodb": "^4.10.0", + "mysql": "^2.18.1", + "pg": "^8.8.0", + "pg-copy-streams": "^6.0.6", + "util": "^0.12.5" + } } From d53979d40f51388780fdaebde9e9b56bfcef1d36 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:04:23 +0800 Subject: [PATCH 04/51] Create dump.js --- dump.js | 507 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 507 insertions(+) create mode 100644 dump.js diff --git a/dump.js b/dump.js new file mode 100644 index 0000000..e92c7f3 --- /dev/null +++ b/dump.js @@ -0,0 +1,507 @@ +const MongoClient = require('mongodb').MongoClient; +const mysql = require('mysql'); +const { Pool } = require('pg'); +const { copyToStream } = require('pg-copy-streams'); + +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); +const moment = require('moment'); +const util = require('util'); + + +/** + * 导出所有MongoDB数据库的数据 + * + * 此函数会连接MongoDB服务器，获取所有非系统数据库， + * 并将每个数据库中的所有集合数据导出为JSON文件。 + * + * 环境变量要求: + * - MONGO_ROOT_USERNAME: MongoDB用户名 + * - MONGO_ROOT_PASSWORD: MongoDB密码 + * - MONGO_HOST: MongoDB主机地址 + * - MONGO_PORT: MongoDB端口号 + * - MONGO_DATABASE: MongoDB数据库名称 + * + * 导出文件将保存在 ./dump 目录下,按数据库名和集合名组织: + * ./dump/ + * ├── database1/ + * │ ├── collection1.json + * │ └── collection2.json + * └── database2/ + * └── collection1.json + * + * @async + * @throws {Error} 当数据库连接失败或导出过程出错时抛出异常 + */ +async function dumpMongoAllDatabases() { + const uri = `mongodb://${process.env.DATABASE_USER || process.env.MONGO_ROOT_USERNAME}:${process.env.DATABASE_PASSWORD || process.env.MONGO_ROOT_PASSWORD}@${process.env.DATABASE_HOST || process.env.MONGO_HOST}:${process.env.DATABASE_PORT || process.env.MONGO_PORT || '27017'}/`; + // const dbName = process.env.DATABASE_NAME || process.env.MONGO_DATEBASE; // 替换成你的数据库名称 + try { + const client = new MongoClient(uri, { useNewUrlParser: true, useUnifiedTopology: true }); + await client.connect(); + + // 获取所有数据库名称 + const databases = await client.db().admin().listDatabases(); + const databaseNames = databases.databases.map(db => db.name); + + // 创建一个文件夹来存储 dump 文件 + const dumpDir = './dump'; + if (!fs.existsSync(dumpDir)) { + fs.mkdirSync(dumpDir); + } + + // 循环遍历每个数据库并 dump + for (const dbName of databaseNames) { + // 判断数据库是否为系统数据库 + if (dbName === 'admin' || dbName === 'local' || dbName === 'config') { + console.log(`Skipping system database: ${dbName}`); + continue; + } + const db = client.db(dbName); + + // 获取所有集合 + const collections = await db.collections(); + + // 循环遍历每个集合并 dump + for (const collection of collections) { + const collectionName = collection.collectionName; + + // 创建一文件来存储 dump 数据 + const dumpFile = `${dumpDir}/${dbName}/${collectionName}.json`; + + // 创建文件夹 + const collectionDir = `${dumpDir}/${dbName}`; + if (!fs.existsSync(collectionDir)) { + fs.mkdirSync(collectionDir); + } + + // 读取集合数据 + const documents = await collection.find().toArray(); + + // 将数据写入文件 + fs.writeFileSync(dumpFile, JSON.stringify(documents, null, 2)); + + console.log(`Dumped ${collectionName} to ${dumpFile}`); + } + + console.log(`Database ${dbName} dump completed!`); + } + + console.log('All databases dump completed!'); + await client.close(); + } catch (err) { + console.error('Error during database dump:', err); + } +} + +/** + * MongoDB数据库备份工具函数 + * + * @description 该函数用于备份MongoDB数据库中的所有集合数据 + * 将每个集合的数据以JSON格式保存到本地文件中 + * + * @param {string} uri - MongoDB连接字符串 + * @param {string[]} databaseNames - 需要备份的数据库名称列表 + * + * @example + * const uri = 'mongodb://localhost:27017'; + * const databaseNames = ['mydb1', 'mydb2']; + * await dumpMongoDatabases(uri, databaseNames); + * + * @returns {Promise} + * @throws {Error} 当数据库连接或备份过程出错时抛出异常 + */ +async function dumpMongoDatabase() { + const uri = `mongodb://${process.env.DATABASE_USER || process.env.MONGO_ROOT_USERNAME}:${process.env.DATABASE_PASSWORD || process.env.MONGO_ROOT_PASSWORD}@${process.env.DATABASE_HOST || process.env.MONGO_HOST}:${process.env.DATABASE_PORT || process.env.MONGO_PORT || '27017'}/`; + const dbName = process.env.DATABASE_NAME || process.env.MONGO_DATABASE; // 替换成你的数据库名称 + try { + if(!dbName){ + console.error('DATABASE_NAME or MONGO_DATEBASE is not set'); + return; + } + const client = new MongoClient(uri, { useNewUrlParser: true, useUnifiedTopology: true }); + await client.connect(); + const db = client.db(dbName); + + const collections = await db.collections(); + + // 创建一个文件夹来存储 dump 文件 + const dumpDir = './dump'; + if (!fs.existsSync(dumpDir)) { + fs.mkdirSync(dumpDir); + } + + // 循环遍历每个集合并 dump + for (const collection of collections) { + const collectionName = collection.collectionName; + + // 创建一个文件来存储 dump 数据 + const dumpFile = `${dumpDir}/${collectionName}.json`; + + // 读取集合数据 + const documents = await collection.find().toArray(); + + // 将数���写入文件 + fs.writeFileSync(dumpFile, JSON.stringify(documents, null, 2)); + + console.log(`Dumped ${collectionName} to ${dumpFile}`); + } + + console.log('Database dump completed!'); + await client.close(); + } catch (err) { + console.error('Error during database dump:', err); + } +} + + +/** + * 备份MySQL数据库中的指定表数据到本地文件 + * + * @param {Object} config - MySQL数据库连接配置 + * @param {string} config.host - 数据库主机地址 + * @param {string} config.user - 数据库用户名 + * @param {string} config.password - 数据库密码 + * @param {string} config.database - 数据库名称 + * @param {string} backupDir - 备份文件保存目录 + * + * @example + * const config = { + * host: 'localhost', + * user: 'root', + * password: '123456', + * database: 'mydb' + * }; + * const backupDir = './backup'; + * await backupMysqlDatabase(config, backupDir); + * + * @returns {Promise} + * @throws {Error} 当数据库连接或备份过程出错时抛出异常 + */ +async function backupMysqlAllDatabase() { + const config = { + host: process.env.DATABASE_HOST || process.env.MYSQL_HOST, + user: process.env.DATABASE_USER || process.env.MYSQL_USER, + password: process.env.DATABASE_PASSWORD || process.env.MYSQL_PASSWORD, + database: process.env.DATABASE_NAME || process.env.MYSQL_DATABASE, + port: parseInt(process.env.DATABASE_PORT || process.env.MYSQL_PORT || 3306) + }; + const connection = mysql.createConnection(config); + const backupDir = './dump'; + + try { + if (!fs.existsSync(backupDir)) { + fs.mkdirSync(backupDir); + } + + await connection.connect(); + + // 获取所有表名 + const [tables] = await util.promisify(connection.query) + .call(connection, ` + SELECT TABLE_NAME + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = ? + AND TABLE_NAME NOT LIKE 'sys_%' + AND TABLE_NAME NOT LIKE 'performance_%' + AND TABLE_NAME NOT LIKE 'innodb_%' + `, [config.database]); + + const timestamp = moment().format('YYYYMMDDHHmmss'); + + // 为每个表创建备份 + for (const table of tables) { + const tableName = table.TABLE_NAME; + const tableBackupFile = path.join(backupDir, `${tableName}-${timestamp}.csv`); + + const writeStream = fs.createWriteStream(tableBackupFile); + const query = `SELECT * FROM ${tableName}`; + const stream = connection.query(query); + + await new Promise((resolve, reject) => { + stream.on('error', reject); + stream.on('result', (row) => { + const rowString = Object.values(row).join(',') + '\n'; + writeStream.write(rowString); + }); + stream.on('end', () => { + writeStream.end(); + console.log(`表 ${tableName} 已备份到 ${tableBackupFile}`); + resolve(); + }); + }); + } + + console.log('所有数据表备份完成！'); + } catch (err) { + console.error('数据库备份过程中出错:', err); + } finally { + if (connection) { + connection.end(); + } + } +} + +/** + * 备份MySQL数据库中的指定表数据到本地文件 + * + * @param {Object} config - MySQL数据库连接配置 + * @param {string} config.host - 数据库主机地址 + * @param {string} config.user - 数据库用户名 + * @param {string} config.password - 数据库密码 + * @param {string} config.database - 数据库名称 + * @param {string} backupDir - 备份文件保存目录 + * + * @example + * const config = { + * host: 'localhost', + * user: 'root', + * password: '123456', + * database: 'mydb' + * }; + * const backupDir = './backup'; + * await backupMysqlDatabase(config, backupDir); + * + * @returns {Promise} + * @throws {Error} 当数据库连接或备份过程出错时抛出异常 + */ +async function backupMysqlDatabase() { + const config = { + host: process.env.DATABASE_HOST || process.env.MYSQL_HOST, + user: process.env.DATABASE_USER || process.env.MYSQL_USER, + password: process.env.DATABASE_PASSWORD || process.env.MYSQL_PASSWORD, + database: process.env.DATABASE_NAME || process.env.MYSQL_DATABASE, + port: parseInt(process.env.DATABASE_PORT || process.env.MYSQL_PORT || 3306) + }; + const connection = mysql.createConnection(config); + const backupDir = './dump'; + + try { + // 创建备份目录 + if (!fs.existsSync(backupDir)) { + fs.mkdirSync(backupDir); + } + + // 连接数据库 + await connection.connect(); + + // 获取当前时间作为备份文件名 + const timestamp = moment().format('YYYYMMDDHHmmss'); + const backupFile = path.join(backupDir, `backup-${timestamp}.sql`); + + // 执行备份命令 + const query = `SELECT * FROM ${process.env.DATABASE_NAME || process.env.MYSQL_DATABASE}`; // 替换为你要备份的表名 + const stream = connection.query(query); + + // 写入备份文件 + const writeStream = fs.createWriteStream(backupFile); + stream.on('rows', (rows) => { + // 处理结果集，并写入备份文件 + rows.forEach((row) => { + // 将数据转换为字符串，并写入文件 + const rowString = Object.values(row).join(','); + writeStream.write(rowString + '\n'); + }); + }); + stream.on('end', () => { + console.log(`Database backup completed successfully! Backup file: ${backupFile}`); + }); + + } catch (err) { + console.error('Error connecting to database:', err); + } finally { + // 关闭数据库连接 + if (connection) { + connection.end(); + } + } +} + + +/** + * 备份PostgreSQL数据库中的所有表数据到本地文件 + * + * @param {Object} config - PostgreSQL数据库连接配置 + * @param {string} config.user - 数据库用户名 + * @param {string} config.host - 数据库主机地址 + * @param {string} config.database - 数据库名称 + * @param {string} config.password - 数据库密码 + * @param {number} config.port - 数据库端口号 + * @param {string} backupDir - 备份文件保存目录 + */ +async function backupPostgresAllDatabase() { + const config = { + host: process.env.DATABASE_HOST || process.env.PG_HOST, + user: process.env.DATABASE_USER || process.env.PG_USER, + password: process.env.DATABASE_PASSWORD || process.env.PG_PASSWORD, + database: process.env.DATABASE_NAME || process.env.PG_DATABASE, + port: parseInt(process.env.DATABASE_PORT || process.env.PG_PORT || 5432), + }; + const pool = new Pool(config); + const backupDir = './dump'; + + try { + // 创建备份目录 + if (!fs.existsSync(backupDir)) { + fs.mkdirSync(backupDir); + } + // 连接数据库 + await pool.connect(); + + // 获取当前时间作为备份文件名 + const timestamp = moment().format('YYYYMMDDHHmmss'); + const backupFile = path.join(backupDir, `backup-${timestamp}.sql`); + + // 首先获取所有表名 + const tablesQuery = ` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + `; + const { rows: tables } = await pool.query(tablesQuery); + + // 为每个表创建一个备份文件 + for (const table of tables) { + // 跳过系统表 + if (table.table_name.startsWith('pg_') || table.table_name.startsWith('sql_')) { + console.log(`Skipping system table: ${table.table_name}`); + continue; + } + const tableName = table.table_name; + const tableBackupFile = path.join(backupDir, `${tableName}-${timestamp}.csv`); + + // 使用COPY命令备份单个表 + const copyQuery = `COPY ${tableName} TO STDOUT WITH (FORMAT CSV, HEADER)`; + const writeStream = fs.createWriteStream(tableBackupFile); + + const copyStream = await pool.query(copyToStream(copyQuery)); + copyStream.pipe(writeStream); + + console.log(`Table ${tableName} backed up to ${tableBackupFile}`); + } + console.log('Database backup completed successfully!'); + } catch (err) { + console.error('Error connecting to database:', err); + } finally { + // 关闭数据库连接 + if (pool) { + pool.end(); + } + } +} + +/** + * 备份PostgreSQL数据库中的指定的表数据到本地文件 + * + * @param {Object} config - PostgreSQL数据库连接配置 + * @param {string} config.user - 数据库用户名 + * @param {string} config.host - 数据库主机地址 + * @param {string} config.database - 数据库名称 + * @param {string} config.password - 数据库密码 + * @param {number} config.port - 数据库端口号 + * @param {string} backupDir - 备份文件保存目录 + */ +async function backupPostgresDatabase() { + const config = { + host: process.env.DATABASE_HOST || process.env.PG_HOST, + user: process.env.DATABASE_USER || process.env.PG_USER, + password: process.env.DATABASE_PASSWORD || process.env.PG_PASSWORD, + database: process.env.DATABASE_NAME || process.env.PG_DATABASE, + port: parseInt(process.env.DATABASE_PORT || process.env.PG_PORT || 5432), + }; + const pool = new Pool(config); + const backupDir = './dump'; + const tableName = process.env.TABLE_NAME; // 新增：从环境变量获取表名 + + try { + if (!tableName) { + throw new Error('TABLE_NAME environment variable is not set'); + } + + // 创建备份目录 + if (!fs.existsSync(backupDir)) { + fs.mkdirSync(backupDir); + } + // 连接数据库 + await pool.connect(); + + // 获取当前时间作为备份文件名 + const timestamp = moment().format('YYYYMMDDHHmmss'); + const backupFile = path.join(backupDir, `backup-${timestamp}.sql`); + + // 直接备份指定的表 + const tableBackupFile = path.join(backupDir, `${tableName}-${timestamp}.csv`); + + // 验证表是否存在 + const tableExistsQuery = ` + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = $1 + )`; + const { rows: [{ exists }] } = await pool.query(tableExistsQuery, [tableName]); + + if (!exists) { + throw new Error(`Table "${tableName}" does not exist`); + } + + // 使用COPY命令备份指定表 + const copyQuery = `COPY ${tableName} TO STDOUT WITH (FORMAT CSV, HEADER)`; + const writeStream = fs.createWriteStream(tableBackupFile); + + const copyStream = await pool.query(copyToStream(copyQuery)); + await new Promise((resolve, reject) => { + copyStream.on('error', reject); + writeStream.on('error', reject); + writeStream.on('finish', resolve); + copyStream.pipe(writeStream); + }); + + console.log(`Table ${tableName} backed up to ${tableBackupFile}`); + console.log('Database backup completed successfully!'); + } catch (err) { + console.error('Error connecting to database:', err); + } finally { + // 关闭数据库连接 + if (pool) { + pool.end(); + } + } +} + +// 根据 DATABASE_TYPE 全局变判断要执行的备份类型 +if(process.env.DATABASE_TYPE){ + let type = process.env.DATABASE_TYPE.toLowerCase() + switch(type){ + case "mongo": + case "mongodb": + if(process.env.DATABASE_NAME){ + await dumpMongoDatabase() + }else{ + await dumpMongoAllDatabases() + } + break; + case "mysql": + if(process.env.DATABASE_NAME){ + await backupMysqlDatabase() + }else{ + await backupMysqlAllDatabase() + } + break; + case "pg": + case "postgres": + case "postgresql": + if(process.env.TABLE_NAME){ + await backupPostgresDatabase() + }else{ + await backupPostgresAllDatabase() + } + break; + default: + console.error('不支持的数据库类型:', type); + break; + } +} From d2ed0334c2dd46d537e74f33769c43752167de04 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:05:50 +0800 Subject: [PATCH 05/51] Update entry.sh --- entry.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/entry.sh b/entry.sh index de51760..c192863 100755 --- a/entry.sh +++ b/entry.sh @@ -2,6 +2,15 @@ echo "Starting container ..." +# Copy Custem Hooks Script File +if [ -d "/custem" ]; then + mkdir /hooks + cp -L /custem/* /hooks + chmod u+x /hooks/* + # Run npm install + npm install +fi + if [ -n "${NFS_TARGET}" ]; then echo "Mounting NFS based on NFS_TARGET: ${NFS_TARGET}" mount -o nolock -v ${NFS_TARGET} /mnt/restic @@ -43,4 +52,4 @@ crond echo "Container started." -exec "$@" \ No newline at end of file +exec "$@" From 401dea8beade2cebdf3b1253b228f939f3eac4a4 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:12:38 +0800 Subject: [PATCH 06/51] Update Dockerfile --- Dockerfile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9cf220b..2680fc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN unzip rclone-current-linux-amd64.zip && mv rclone-*-linux-amd64/rclone /bin/ FROM restic/restic:0.16.0 -RUN apk add --update --no-cache curl mailx +RUN apk add --update --no-cache curl mailx nodejs npm COPY --from=rclone /bin/rclone /bin/rclone @@ -36,6 +36,13 @@ ENV OS_REGION_NAME="" ENV OS_INTERFACE="" ENV OS_IDENTITY_API_VERSION=3 +ENV DATABASE_TYPE="" +ENV DATABASE_HOST="" +ENV DATABASE_PORT="" +ENV DATABASE_USER="" +ENV DATABASE_PASSWORD="" +ENV DATABASE_NAME="" + # openshift fix RUN mkdir /.cache && \ chgrp -R 0 /.cache && \ @@ -54,5 +61,14 @@ COPY backup.sh /bin/backup COPY check.sh /bin/check COPY entry.sh /entry.sh +RUN mkdir /script && \ + chgrp -R 0 /script && \ + chmod -R g=u /script +COPY package.json /script/package.json +COPY package.json /script/dump.js +RUN chmod u+x /script/* +RUN cd /script && \ + npm install + ENTRYPOINT ["/entry.sh"] CMD ["tail","-fn0","/var/log/cron.log"] From ab5b08298b86802788f7db816bc3dd69fb7d271e Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:16:59 +0800 Subject: [PATCH 07/51] Update Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 2680fc4..dcccab2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,8 @@ ENV DATABASE_USER="" ENV DATABASE_PASSWORD="" ENV DATABASE_NAME="" +ENV TZ="Asia/Shanghai" + # openshift fix RUN mkdir /.cache && \ chgrp -R 0 /.cache && \ From ff84d0fbb07e82bef0f158c5f485441f7e84e5b5 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:20:03 +0800 Subject: [PATCH 08/51] Update backup.sh --- backup.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/backup.sh b/backup.sh index 6391ebc..a1ae4a6 100755 --- a/backup.sh +++ b/backup.sh @@ -12,6 +12,34 @@ logLast() { echo "$1" >> ${lastLogfile} } +backupDatebase(){ + echo "### Start MongoDB Dump ###" + echo "Backup Datebase: ${DATABASE_TYPE}" + # 检查 dump 目录是否存在，如果存在则删除 + if [ -d "/script/dump" ]; then + rm -rf /script/dump + fi + mkdir /script/dump + # 运行 dump 脚本 + npm run dump + # 检查 /script/dump 目录下是否为空，不为空则复制 dump 数据到 + if [ "$(ls -A /script/dump)" ]; then + # 检查 /data/dump 目录存在，自动删除旧备份；如果不存在则创建 dump 目录 + if [ -d "/data/dump" ]; then + rm -rf /data/dump/* + else + mkdir /data/dump + fi + # 复制最新的备份 + cp -r /script/dump /data/ + echo "\n MongoDB Dump List:" + ls -l /data/dump + else + echo "./dump Folder Empty, MongoDB Dump Fail." + fi + echo "### End ${DATABASE_TYPE} Dump ###" +} + if [ -f "/hooks/pre-backup.sh" ]; then echo "Starting pre-backup script ..." /hooks/pre-backup.sh @@ -19,6 +47,19 @@ else echo "Pre-backup script not found ..." fi +# Dump Datebase +if [ -n "${DATABASE_TYPE}" ]; then + # 判断时间是否在晚上12点到凌晨5点之间 + current_hour=$(date +%H) + if [[ $current_hour -ge 0 && $current_hour -le 23 ]]; then + echo "Current within the Backup Time Period (AM 0~5)" + backupDatebase + else + echo "Current not within the Backup Time Period (AM 0~5)" + echo "Skip ${DATABASE_TYPE} Dump" + fi +fi + start=`date +%s` rm -f ${lastLogfile} ${lastMailLogfile} echo "Starting Backup at $(date +"%Y-%m-%d %H:%M:%S")" From 79f0e3e17bd81c38af410dee742ef90df401199d Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:28:26 +0800 Subject: [PATCH 09/51] Update Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index dcccab2..151aa9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,8 @@ ENV DATABASE_PORT="" ENV DATABASE_USER="" ENV DATABASE_PASSWORD="" ENV DATABASE_NAME="" +ENV DATABASE_BACKUP_START=0 +ENV DATABASE_BACKUP_END=23 ENV TZ="Asia/Shanghai" From 18d51d9dce92c5a8775bb51a01a1ee7e81b3a25b Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:29:26 +0800 Subject: [PATCH 10/51] Update backup.sh --- backup.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backup.sh b/backup.sh index a1ae4a6..b699944 100755 --- a/backup.sh +++ b/backup.sh @@ -49,13 +49,14 @@ fi # Dump Datebase if [ -n "${DATABASE_TYPE}" ]; then - # 判断时间是否在晚上12点到凌晨5点之间 + # 判断时间是否在指定时间段内(建议凌晨1点到凌晨5点之间) current_hour=$(date +%H) - if [[ $current_hour -ge 0 && $current_hour -le 23 ]]; then - echo "Current within the Backup Time Period (AM 0~5)" + # if [[ $current_hour -ge 0 && $current_hour -le 23 ]]; then + if [[ $current_hour -ge ${DATABASE_BACKUP_START} && $current_hour -le ${DATABASE_BACKUP_END} ]]; then + echo "Current within the Backup Time Period (${DATABASE_BACKUP_START}~${DATABASE_BACKUP_END})" backupDatebase else - echo "Current not within the Backup Time Period (AM 0~5)" + echo "Current not within the Backup Time Period (${DATABASE_BACKUP_START}~${DATABASE_BACKUP_END})" echo "Skip ${DATABASE_TYPE} Dump" fi fi From d4feddfd27604897b2b8b87f96d5c54427cb186f Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:40:56 +0800 Subject: [PATCH 11/51] Update backup.sh --- backup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/backup.sh b/backup.sh index b699944..99a8bc9 100755 --- a/backup.sh +++ b/backup.sh @@ -21,6 +21,7 @@ backupDatebase(){ fi mkdir /script/dump # 运行 dump 脚本 + cd /script npm run dump # 检查 /script/dump 目录下是否为空，不为空则复制 dump 数据到 if [ "$(ls -A /script/dump)" ]; then From a021c0dae960d23139e59c093bdeb402e422ff5a Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:46:42 +0800 Subject: [PATCH 12/51] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 151aa9f..bac4a71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,7 +69,7 @@ RUN mkdir /script && \ chgrp -R 0 /script && \ chmod -R g=u /script COPY package.json /script/package.json -COPY package.json /script/dump.js +COPY dump.js /script/dump.js RUN chmod u+x /script/* RUN cd /script && \ npm install From bb61f59a5d57dffbf3787a229acea5f450750cf2 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:49:25 +0800 Subject: [PATCH 13/51] Update dump.js --- dump.js | 72 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/dump.js b/dump.js index e92c7f3..77ab74d 100644 --- a/dump.js +++ b/dump.js @@ -142,7 +142,7 @@ async function dumpMongoDatabase() { // 读取集合数据 const documents = await collection.find().toArray(); - // 将数���写入文件 + // 将数据写入文件 fs.writeFileSync(dumpFile, JSON.stringify(documents, null, 2)); console.log(`Dumped ${collectionName} to ${dumpFile}`); @@ -424,7 +424,7 @@ async function backupPostgresDatabase() { if (!fs.existsSync(backupDir)) { fs.mkdirSync(backupDir); } - // 连接数据库 + // ��接数据库 await pool.connect(); // 获取当前时间作为备份文件名 @@ -472,36 +472,42 @@ async function backupPostgresDatabase() { } } -// 根据 DATABASE_TYPE 全局变判断要执行的备份类型 -if(process.env.DATABASE_TYPE){ - let type = process.env.DATABASE_TYPE.toLowerCase() - switch(type){ - case "mongo": - case "mongodb": - if(process.env.DATABASE_NAME){ - await dumpMongoDatabase() - }else{ - await dumpMongoAllDatabases() - } - break; - case "mysql": - if(process.env.DATABASE_NAME){ - await backupMysqlDatabase() - }else{ - await backupMysqlAllDatabase() - } - break; - case "pg": - case "postgres": - case "postgresql": - if(process.env.TABLE_NAME){ - await backupPostgresDatabase() - }else{ - await backupPostgresAllDatabase() - } - break; - default: - console.error('不支持的数据库类型:', type); - break; +// 创建一个主函数来处理数据库备份 +async function main() { + // 根据 DATABASE_TYPE 全局变判断要执行的备份类型 + if(process.env.DATABASE_TYPE){ + let type = process.env.DATABASE_TYPE.toLowerCase() + switch(type){ + case "mongo": + case "mongodb": + if(process.env.DATABASE_NAME){ + await dumpMongoDatabase() + }else{ + await dumpMongoAllDatabases() + } + break; + case "mysql": + if(process.env.DATABASE_NAME){ + await backupMysqlDatabase() + }else{ + await backupMysqlAllDatabase() + } + break; + case "pg": + case "postgres": + case "postgresql": + if(process.env.TABLE_NAME){ + await backupPostgresDatabase() + }else{ + await backupPostgresAllDatabase() + } + break; + default: + console.error('不支持的数据库类型:', type); + break; + } } } + +// 执行主函数 +main().catch(console.error); From 3e5b1c38ec001bef222b92c2ee1f90927c85dd00 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:07:09 +0800 Subject: [PATCH 14/51] Update backup.sh --- backup.sh | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/backup.sh b/backup.sh index 99a8bc9..eda7085 100755 --- a/backup.sh +++ b/backup.sh @@ -13,7 +13,7 @@ logLast() { } backupDatebase(){ - echo "### Start MongoDB Dump ###" + echo "### Start ${DATABASE_TYPE} Dump ###" echo "Backup Datebase: ${DATABASE_TYPE}" # 检查 dump 目录是否存在，如果存在则删除 if [ -d "/script/dump" ]; then @@ -50,16 +50,7 @@ fi # Dump Datebase if [ -n "${DATABASE_TYPE}" ]; then - # 判断时间是否在指定时间段内(建议凌晨1点到凌晨5点之间) - current_hour=$(date +%H) - # if [[ $current_hour -ge 0 && $current_hour -le 23 ]]; then - if [[ $current_hour -ge ${DATABASE_BACKUP_START} && $current_hour -le ${DATABASE_BACKUP_END} ]]; then - echo "Current within the Backup Time Period (${DATABASE_BACKUP_START}~${DATABASE_BACKUP_END})" - backupDatebase - else - echo "Current not within the Backup Time Period (${DATABASE_BACKUP_START}~${DATABASE_BACKUP_END})" - echo "Skip ${DATABASE_TYPE} Dump" - fi + backupDatebase fi start=`date +%s` From 2b0f6c8868d249477b94ab60b3f378dd04669eb8 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:08:00 +0800 Subject: [PATCH 15/51] Update dump.js DATABASE_BACKUP_TIME --- dump.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dump.js b/dump.js index 77ab74d..685113f 100644 --- a/dump.js +++ b/dump.js @@ -474,6 +474,24 @@ async function backupPostgresDatabase() { // 创建一个主函数来处理数据库备份 async function main() { + // 通过TIME_RANGE环境变量指定时间范围，格式为"HH-HH"，如"0-23" + const timeRange = process.env.DATABASE_BACKUP_TIME || "0-23"; + const [startStr, endStr] = timeRange.split("-"); + let start = parseInt(startStr); + let end = parseInt(endStr); + const now = moment().hour(); + // 验证时间范围的有效性 + if (isNaN(start) || isNaN(end) || start < 0 || start > 23 || end < 0 || end > 23) { + console.error('无效的时间范围格式，已使用默认值"0-23"'); + // return; + start = 0; + end = 23; + } + console.log(`当前时间: ${now}, 指定时间范围: ${start}-${end}`); + if (now < start || now > end) { + console.log('当前时间不在指定时间段内，备份任务已取消'); + return; + } // 根据 DATABASE_TYPE 全局变判断要执行的备份类型 if(process.env.DATABASE_TYPE){ let type = process.env.DATABASE_TYPE.toLowerCase() From f4fc9eca047c2dbc7dbf4c42a06c6797cea4ec17 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:08:27 +0800 Subject: [PATCH 16/51] Update Dockerfile DATABASE_BACKUP_TIME --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bac4a71..bdc2025 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,8 +42,7 @@ ENV DATABASE_PORT="" ENV DATABASE_USER="" ENV DATABASE_PASSWORD="" ENV DATABASE_NAME="" -ENV DATABASE_BACKUP_START=0 -ENV DATABASE_BACKUP_END=23 +ENV DATABASE_BACKUP_TIME="0-23" ENV TZ="Asia/Shanghai" From 4ee53e570d7b561f48fa3791ce9a1de7c392d43b Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:10:44 +0800 Subject: [PATCH 17/51] Update backup.sh --- backup.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backup.sh b/backup.sh index eda7085..342863a 100755 --- a/backup.sh +++ b/backup.sh @@ -13,6 +13,7 @@ logLast() { } backupDatebase(){ + echo "" echo "### Start ${DATABASE_TYPE} Dump ###" echo "Backup Datebase: ${DATABASE_TYPE}" # 检查 dump 目录是否存在，如果存在则删除 @@ -33,12 +34,14 @@ backupDatebase(){ fi # 复制最新的备份 cp -r /script/dump /data/ - echo "\n MongoDB Dump List:" + echo "" + echo "MongoDB Dump List:" ls -l /data/dump else echo "./dump Folder Empty, MongoDB Dump Fail." fi echo "### End ${DATABASE_TYPE} Dump ###" + echo "" } if [ -f "/hooks/pre-backup.sh" ]; then From 09d3a6deb9304aa9712a1eb870bedcb2a929a4ba Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:12:48 +0800 Subject: [PATCH 18/51] Update dump.js --- dump.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dump.js b/dump.js index 685113f..60b5208 100644 --- a/dump.js +++ b/dump.js @@ -482,14 +482,14 @@ async function main() { const now = moment().hour(); // 验证时间范围的有效性 if (isNaN(start) || isNaN(end) || start < 0 || start > 23 || end < 0 || end > 23) { - console.error('无效的时间范围格式，已使用默认值"0-23"'); + console.error('Invalid time range format, using default value "0-23"'); // return; start = 0; end = 23; } - console.log(`当前时间: ${now}, 指定时间范围: ${start}-${end}`); + console.log(`Current time: ${now}, Specified time range: ${start}-${end}`); if (now < start || now > end) { - console.log('当前时间不在指定时间段内，备份任务已取消'); + console.log(`Now not within the specified time period ${process.env.DATABASE_BACKUP_TIME}, Database Dump Task has been cancelled.`); return; } // 根据 DATABASE_TYPE 全局变判断要执行的备份类型 From c7ff78fc2112bcf4f7fa1e2bd20ee7d5fbc82d80 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:41:16 +0800 Subject: [PATCH 19/51] Update README.md --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index a75fcd6..1c3bdc4 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,15 @@ The container is set up by setting [environment variables](https://docs.docker.c * `OS_REGION_NAME` - Optional. When using restic with OpenStack Swift container. * `OS_INTERFACE` - Optional. When using restic with OpenStack Swift container. * `OS_IDENTITY_API_VERSION` - Optional. When using restic with OpenStack Swift container. + +NEW Env For Database Dump (Only MongoDB has passed the test, other types databases have not been tested yet) +* DATABASE_TYPE - Optional. Specify the database type (mongo/mongodb/mysql/pg/postgres/postgresql). Specifying this option will enable the database backup function. +* DATABASE_BACKUP_TIME - Optional. Database backup is automatically enabled within the specified time range (default 0-23) +* DATABASE_HOST - Optional. Database host address +* DATABASE_PORT - Optional. Database host port (if not specified, it will automatically follow the default port value of the database type) +* DATABASE_USER - Optional. Database Username +* DATABASE_PASSWORD - Optional. Database password +* DATABASE_NAME - Optional. Database name. If not specified, all database tables (except system tables) are used by default. ## Volumes From cde9fe7697baf7368577cfdd05cda13a0e540b00 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:15:17 +0800 Subject: [PATCH 20/51] Update README.md --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1c3bdc4..17c40b1 100644 --- a/README.md +++ b/README.md @@ -144,13 +144,13 @@ The container is set up by setting [environment variables](https://docs.docker.c * `OS_IDENTITY_API_VERSION` - Optional. When using restic with OpenStack Swift container. NEW Env For Database Dump (Only MongoDB has passed the test, other types databases have not been tested yet) -* DATABASE_TYPE - Optional. Specify the database type (mongo/mongodb/mysql/pg/postgres/postgresql). Specifying this option will enable the database backup function. -* DATABASE_BACKUP_TIME - Optional. Database backup is automatically enabled within the specified time range (default 0-23) -* DATABASE_HOST - Optional. Database host address -* DATABASE_PORT - Optional. Database host port (if not specified, it will automatically follow the default port value of the database type) -* DATABASE_USER - Optional. Database Username -* DATABASE_PASSWORD - Optional. Database password -* DATABASE_NAME - Optional. Database name. If not specified, all database tables (except system tables) are used by default. +* `DATABASE_TYPE` - Optional. Specify the database type (mongo/mongodb/mysql/pg/postgres/postgresql). Specifying this option will enable the database backup function. +* `DATABASE_BACKUP_TIME` - Optional. Database backup is automatically enabled within the specified time range (default 0-23) +* `DATABASE_HOST` - Optional. Database host address +* `DATABASE_PORT` - Optional. Database host port (if not specified, it will automatically follow the default port value of the database type) +* `DATABASE_USER` - Optional. Database Username +* `DATABASE_PASSWORD` - Optional. Database password +* `DATABASE_NAME` - Optional. Database name. If not specified, all database tables (except system tables) are used by default. ## Volumes From 8e18cb1e95eaaa999c5982714da5767fa7459ea6 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:23:28 +0800 Subject: [PATCH 21/51] Update dump.js --- dump.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dump.js b/dump.js index 60b5208..d85ecf7 100644 --- a/dump.js +++ b/dump.js @@ -1,5 +1,5 @@ const MongoClient = require('mongodb').MongoClient; -const mysql = require('mysql'); +const mysql = require('mysql2'); const { Pool } = require('pg'); const { copyToStream } = require('pg-copy-streams'); @@ -187,7 +187,7 @@ async function backupMysqlAllDatabase() { database: process.env.DATABASE_NAME || process.env.MYSQL_DATABASE, port: parseInt(process.env.DATABASE_PORT || process.env.MYSQL_PORT || 3306) }; - const connection = mysql.createConnection(config); + const connection = mysql.createConnection(config).promise(); const backupDir = './dump'; try { @@ -274,7 +274,7 @@ async function backupMysqlDatabase() { database: process.env.DATABASE_NAME || process.env.MYSQL_DATABASE, port: parseInt(process.env.DATABASE_PORT || process.env.MYSQL_PORT || 3306) }; - const connection = mysql.createConnection(config); + const connection = mysql.createConnection(config).promise(); const backupDir = './dump'; try { @@ -424,7 +424,7 @@ async function backupPostgresDatabase() { if (!fs.existsSync(backupDir)) { fs.mkdirSync(backupDir); } - // ��接数据库 + // 连接数据库 await pool.connect(); // 获取当前时间作为备份文件名 From b4826397d918edb04cd359a259a07f0ff847ce4f Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:23:45 +0800 Subject: [PATCH 22/51] Update package.json --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index ca8ce24..e33a621 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "moment": "^2.29.4", "mongodb": "^4.10.0", "mysql": "^2.18.1", + "mysql2": "^3.11.5", "pg": "^8.8.0", "pg-copy-streams": "^6.0.6", "util": "^0.12.5" From d88deada54e1b52d3fd0505ca650db11ad8ce7dc Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:33:41 +0800 Subject: [PATCH 23/51] Update dump.js --- dump.js | 326 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 211 insertions(+), 115 deletions(-) diff --git a/dump.js b/dump.js index d85ecf7..9b10d1c 100644 --- a/dump.js +++ b/dump.js @@ -198,53 +198,73 @@ async function backupMysqlAllDatabase() { await connection.connect(); // 获取所有表名 - const [tables] = await util.promisify(connection.query) - .call(connection, ` - SELECT TABLE_NAME - FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = ? - AND TABLE_NAME NOT LIKE 'sys_%' - AND TABLE_NAME NOT LIKE 'performance_%' - AND TABLE_NAME NOT LIKE 'innodb_%' - `, [config.database]); + const [tables] = await connection.query(` + SELECT TABLE_NAME + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = ? + AND TABLE_NAME NOT LIKE 'sys_%' + AND TABLE_NAME NOT LIKE 'performance_%' + AND TABLE_NAME NOT LIKE 'innodb_%' + `, [config.database]); const timestamp = moment().format('YYYYMMDDHHmmss'); + const backupFile = path.join(backupDir, `mysql-backup-${timestamp}.sql`); + const writeStream = fs.createWriteStream(backupFile); + // 写入文件头部信息 + writeStream.write(`-- MySQL dump\n`); + writeStream.write(`-- Created at: ${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); + writeStream.write(`SET FOREIGN_KEY_CHECKS=0;\n\n`); + // 为每个表创建备份 for (const table of tables) { const tableName = table.TABLE_NAME; - const tableBackupFile = path.join(backupDir, `${tableName}-${timestamp}.csv`); - const writeStream = fs.createWriteStream(tableBackupFile); - const query = `SELECT * FROM ${tableName}`; - const stream = connection.query(query); + // 获取表结构 + const [tableStructure] = await connection.query(`SHOW CREATE TABLE \`${tableName}\``); + writeStream.write(`-- Table structure: ${tableName}\n`); + writeStream.write(`DROP TABLE IF EXISTS \`${tableName}\`;\n`); + writeStream.write(`${tableStructure[0]['Create Table']};\n\n`); + + // 获取表数据 + const [rows] = await connection.query(`SELECT * FROM \`${tableName}\``); - await new Promise((resolve, reject) => { - stream.on('error', reject); - stream.on('result', (row) => { - const rowString = Object.values(row).join(',') + '\n'; - writeStream.write(rowString); - }); - stream.on('end', () => { - writeStream.end(); - console.log(`表 ${tableName} 已备份到 ${tableBackupFile}`); - resolve(); - }); - }); + if (rows.length > 0) { + writeStream.write(`-- Table data: ${tableName}\n`); + const columns = Object.keys(rows[0]); + + for (const row of rows) { + const values = columns.map(column => { + const value = row[column]; + if (value === null) return 'NULL'; + if (typeof value === 'number') return value; + return `'${value.toString().replace(/'/g, "''")}'`; + }); + + writeStream.write( + `INSERT INTO \`${tableName}\` (${columns.map(c => '`'+c+'`').join(', ')}) ` + + `VALUES (${values.join(', ')});\n` + ); + } + writeStream.write('\n'); + } } - console.log('所有数据表备份完成！'); + writeStream.write(`SET FOREIGN_KEY_CHECKS=1;\n`); + writeStream.end(); + console.log(`Database backup completed! Backup file: ${backupFile}`); + } catch (err) { - console.error('数据库备份过程中出错:', err); + console.error('Error during database backup:', err); } finally { if (connection) { - connection.end(); + await connection.end(); } } } /** - * 备份MySQL数据库中的指定表数据到本地文件 + * 备份MySQL数据库中的指定表数据到��地文件 * * @param {Object} config - MySQL数据库连接配置 * @param {string} config.host - 数据库主机地址 @@ -278,42 +298,57 @@ async function backupMysqlDatabase() { const backupDir = './dump'; try { - // 创建备份目录 if (!fs.existsSync(backupDir)) { fs.mkdirSync(backupDir); } - // 连接数据库 await connection.connect(); - - // 获取当前时间作为备份文件名 + const timestamp = moment().format('YYYYMMDDHHmmss'); const backupFile = path.join(backupDir, `backup-${timestamp}.sql`); + const writeStream = fs.createWriteStream(backupFile); - // 执行备份命令 - const query = `SELECT * FROM ${process.env.DATABASE_NAME || process.env.MYSQL_DATABASE}`; // 替换为你要备份的表名 - const stream = connection.query(query); + // 获取表结构 + const [tableStructure] = await connection.query( + `SHOW CREATE TABLE \`${config.database}\`` + ); + + // 写入表结构 + writeStream.write(`-- Database table structure\n`); + writeStream.write(`DROP TABLE IF EXISTS \`${config.database}\`;\n`); + writeStream.write(`${tableStructure[0]['Create Table']};\n\n`); + + // 获取表数据 + const [rows] = await connection.query(`SELECT * FROM \`${config.database}\``); + + if (rows.length > 0) { + // 写入数据插入语句 + writeStream.write(`-- Data records\n`); + const columns = Object.keys(rows[0]); + + for (const row of rows) { + const values = columns.map(column => { + const value = row[column]; + if (value === null) return 'NULL'; + if (typeof value === 'number') return value; + return `'${value.toString().replace(/'/g, "''")}'`; + }); + + writeStream.write( + `INSERT INTO \`${config.database}\` (${columns.map(c => '`'+c+'`').join(', ')}) ` + + `VALUES (${values.join(', ')});\n` + ); + } + } - // 写入备份文件 - const writeStream = fs.createWriteStream(backupFile); - stream.on('rows', (rows) => { - // 处理结果集，并写入备份文件 - rows.forEach((row) => { - // 将数据转换为字符串，并写入文件 - const rowString = Object.values(row).join(','); - writeStream.write(rowString + '\n'); - }); - }); - stream.on('end', () => { - console.log(`Database backup completed successfully! Backup file: ${backupFile}`); - }); + writeStream.end(); + console.log(`Database backup completed! Backup file: ${backupFile}`); } catch (err) { - console.error('Error connecting to database:', err); + console.error('Error during database backup:', err); } finally { - // 关闭数据库连接 if (connection) { - connection.end(); + await connection.end(); } } } @@ -342,52 +377,90 @@ async function backupPostgresAllDatabase() { const backupDir = './dump'; try { - // 创建备份目录 if (!fs.existsSync(backupDir)) { fs.mkdirSync(backupDir); } - // 连接数据库 - await pool.connect(); - - // 获取当前时间作为备份文件名 + const timestamp = moment().format('YYYYMMDDHHmmss'); - const backupFile = path.join(backupDir, `backup-${timestamp}.sql`); + const backupFile = path.join(backupDir, `postgres-backup-${timestamp}.sql`); + const writeStream = fs.createWriteStream(backupFile); - // 首先获取所有表名 - const tablesQuery = ` + // 写入文件头部信息 + writeStream.write(`-- PostgreSQL dump\n`); + writeStream.write(`-- 创建时间：${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); + + // 获取所有表名 + const { rows: tables } = await pool.query(` SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE' - `; - const { rows: tables } = await pool.query(tablesQuery); + AND table_name NOT LIKE 'pg_%' + AND table_name NOT LIKE 'sql_%' + `); - // 为每个表创建一个备份文件 for (const table of tables) { - // 跳过系统表 - if (table.table_name.startsWith('pg_') || table.table_name.startsWith('sql_')) { - console.log(`Skipping system table: ${table.table_name}`); - continue; - } const tableName = table.table_name; - const tableBackupFile = path.join(backupDir, `${tableName}-${timestamp}.csv`); - // 使用COPY命令备份单个表 - const copyQuery = `COPY ${tableName} TO STDOUT WITH (FORMAT CSV, HEADER)`; - const writeStream = fs.createWriteStream(tableBackupFile); + // 获取表结构 + const { rows: columns } = await pool.query(` + SELECT column_name, data_type, character_maximum_length, + is_nullable, column_default + FROM information_schema.columns + WHERE table_name = $1 + ORDER BY ordinal_position + `, [tableName]); + + // 写入表结构 + writeStream.write(`-- 表结构: ${tableName}\n`); + writeStream.write(`DROP TABLE IF EXISTS "${tableName}";\n`); + writeStream.write(`CREATE TABLE "${tableName}" (\n`); - const copyStream = await pool.query(copyToStream(copyQuery)); - copyStream.pipe(writeStream); + const columnDefs = columns.map(col => { + let def = ` "${col.column_name}" ${col.data_type}`; + if (col.character_maximum_length) { + def += `(${col.character_maximum_length})`; + } + if (col.is_nullable === 'NO') { + def += ' NOT NULL'; + } + if (col.column_default) { + def += ` DEFAULT ${col.column_default}`; + } + return def; + }); - console.log(`Table ${tableName} backed up to ${tableBackupFile}`); + writeStream.write(columnDefs.join(',\n')); + writeStream.write('\n);\n\n'); + + // 获取并写入表数据 + const { rows: data } = await pool.query(`SELECT * FROM "${tableName}"`); + if (data.length > 0) { + writeStream.write(`-- 表数据: ${tableName}\n`); + for (const row of data) { + const values = Object.values(row).map(val => { + if (val === null) return 'NULL'; + if (typeof val === 'number') return val; + return `'${val.toString().replace(/'/g, "''")}'`; + }); + + writeStream.write( + `INSERT INTO "${tableName}" (${Object.keys(row).map(k => `"${k}"`).join(', ')}) ` + + `VALUES (${values.join(', ')});\n` + ); + } + writeStream.write('\n'); + } } - console.log('Database backup completed successfully!'); + + writeStream.end(); + console.log(`数据库备份完成！备份文件: ${backupFile}`); + } catch (err) { - console.error('Error connecting to database:', err); + console.error('数据库备份过程中出错:', err); } finally { - // 关闭数据库连接 if (pool) { - pool.end(); + await pool.end(); } } } @@ -420,54 +493,77 @@ async function backupPostgresDatabase() { throw new Error('TABLE_NAME environment variable is not set'); } - // 创建备份目录 if (!fs.existsSync(backupDir)) { fs.mkdirSync(backupDir); } - // 连接数据库 + await pool.connect(); - - // 获取当前时间作为备份文件名 const timestamp = moment().format('YYYYMMDDHHmmss'); - const backupFile = path.join(backupDir, `backup-${timestamp}.sql`); + const backupFile = path.join(backupDir, `${tableName}-${timestamp}.sql`); + const writeStream = fs.createWriteStream(backupFile); - // 直接备份指定的表 - const tableBackupFile = path.join(backupDir, `${tableName}-${timestamp}.csv`); - - // 验证表是否存在 - const tableExistsQuery = ` - SELECT EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = 'public' - AND table_name = $1 - )`; - const { rows: [{ exists }] } = await pool.query(tableExistsQuery, [tableName]); - - if (!exists) { - throw new Error(`Table "${tableName}" does not exist`); - } + // 写入文件头部信息 + writeStream.write(`-- PostgreSQL dump of table ${tableName}\n`); + writeStream.write(`-- 创建时间：${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); + + // 获取表结构 + const { rows: columns } = await pool.query(` + SELECT column_name, data_type, character_maximum_length, + is_nullable, column_default + FROM information_schema.columns + WHERE table_name = $1 + ORDER BY ordinal_position + `, [tableName]); + + // 写入表结构 + writeStream.write(`-- 表结构\n`); + writeStream.write(`DROP TABLE IF EXISTS "${tableName}";\n`); + writeStream.write(`CREATE TABLE "${tableName}" (\n`); - // 使用COPY命令备份指定表 - const copyQuery = `COPY ${tableName} TO STDOUT WITH (FORMAT CSV, HEADER)`; - const writeStream = fs.createWriteStream(tableBackupFile); - - const copyStream = await pool.query(copyToStream(copyQuery)); - await new Promise((resolve, reject) => { - copyStream.on('error', reject); - writeStream.on('error', reject); - writeStream.on('finish', resolve); - copyStream.pipe(writeStream); + const columnDefs = columns.map(col => { + let def = ` "${col.column_name}" ${col.data_type}`; + if (col.character_maximum_length) { + def += `(${col.character_maximum_length})`; + } + if (col.is_nullable === 'NO') { + def += ' NOT NULL'; + } + if (col.column_default) { + def += ` DEFAULT ${col.column_default}`; + } + return def; }); - console.log(`Table ${tableName} backed up to ${tableBackupFile}`); - console.log('Database backup completed successfully!'); + writeStream.write(columnDefs.join(',\n')); + writeStream.write('\n);\n\n'); + + // 获取并写入表数据 + const { rows: data } = await pool.query(`SELECT * FROM "${tableName}"`); + if (data.length > 0) { + writeStream.write(`-- 表数据\n`); + for (const row of data) { + const values = Object.values(row).map(val => { + if (val === null) return 'NULL'; + if (typeof val === 'number') return val; + return `'${val.toString().replace(/'/g, "''")}'`; + }); + + writeStream.write( + `INSERT INTO "${tableName}" (${Object.keys(row).map(k => `"${k}"`).join(', ')}) ` + + `VALUES (${values.join(', ')});\n` + ); + } + } + + writeStream.end(); + console.log(`表 ${tableName} 已备份到 ${backupFile}`); + console.log('数据库备份完成！'); + } catch (err) { - console.error('Error connecting to database:', err); + console.error('数据库备份过程中出错:', err); } finally { - // 关闭数据库连接 if (pool) { - pool.end(); + await pool.end(); } } } From 1869359d3d6e3b4871e33009e4977a98da3f0dcf Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:36:15 +0800 Subject: [PATCH 24/51] Update dump.js --- dump.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dump.js b/dump.js index 9b10d1c..1bc82d1 100644 --- a/dump.js +++ b/dump.js @@ -387,7 +387,7 @@ async function backupPostgresAllDatabase() { // 写入文件头部信息 writeStream.write(`-- PostgreSQL dump\n`); - writeStream.write(`-- 创建时间：${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); + writeStream.write(`-- Created at: ${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); // 获取所有表名 const { rows: tables } = await pool.query(` @@ -412,7 +412,7 @@ async function backupPostgresAllDatabase() { `, [tableName]); // 写入表结构 - writeStream.write(`-- 表结构: ${tableName}\n`); + writeStream.write(`-- Table structure: ${tableName}\n`); writeStream.write(`DROP TABLE IF EXISTS "${tableName}";\n`); writeStream.write(`CREATE TABLE "${tableName}" (\n`); @@ -436,7 +436,7 @@ async function backupPostgresAllDatabase() { // 获取并写入表数据 const { rows: data } = await pool.query(`SELECT * FROM "${tableName}"`); if (data.length > 0) { - writeStream.write(`-- 表数据: ${tableName}\n`); + writeStream.write(`-- Table data: ${tableName}\n`); for (const row of data) { const values = Object.values(row).map(val => { if (val === null) return 'NULL'; @@ -454,10 +454,10 @@ async function backupPostgresAllDatabase() { } writeStream.end(); - console.log(`数据库备份完成！备份文件: ${backupFile}`); + console.log(`Database backup completed! Backup file: ${backupFile}`); } catch (err) { - console.error('数据库备份过程中出错:', err); + console.error('Error during database backup:', err); } finally { if (pool) { await pool.end(); @@ -504,7 +504,7 @@ async function backupPostgresDatabase() { // 写入文件头部信息 writeStream.write(`-- PostgreSQL dump of table ${tableName}\n`); - writeStream.write(`-- 创建时间：${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); + writeStream.write(`-- Created at: ${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); // 获取表结构 const { rows: columns } = await pool.query(` @@ -516,7 +516,7 @@ async function backupPostgresDatabase() { `, [tableName]); // 写入表结构 - writeStream.write(`-- 表结构\n`); + writeStream.write(`-- Table structure\n`); writeStream.write(`DROP TABLE IF EXISTS "${tableName}";\n`); writeStream.write(`CREATE TABLE "${tableName}" (\n`); @@ -540,7 +540,7 @@ async function backupPostgresDatabase() { // 获取并写入表数据 const { rows: data } = await pool.query(`SELECT * FROM "${tableName}"`); if (data.length > 0) { - writeStream.write(`-- 表数据\n`); + writeStream.write(`-- Table data\n`); for (const row of data) { const values = Object.values(row).map(val => { if (val === null) return 'NULL'; @@ -556,11 +556,11 @@ async function backupPostgresDatabase() { } writeStream.end(); - console.log(`表 ${tableName} 已备份到 ${backupFile}`); - console.log('数据库备份完成！'); + console.log(`Table ${tableName} has been backed up to ${backupFile}`); + console.log('Database backup completed!'); } catch (err) { - console.error('数据库备份过程中出错:', err); + console.error('Error during database backup:', err); } finally { if (pool) { await pool.end(); From e33997dd4e509ab02721e388d6364241eb41be1d Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:42:11 +0800 Subject: [PATCH 25/51] Update dump.js --- dump.js | 68 +++++++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/dump.js b/dump.js index 1bc82d1..adb23c3 100644 --- a/dump.js +++ b/dump.js @@ -264,7 +264,7 @@ async function backupMysqlAllDatabase() { } /** - * 备份MySQL数据库中的指定表数据到��地文件 + * 备份MySQL数据库中的指定表数据���本地文件 * * @param {Object} config - MySQL数据库连接配置 * @param {string} config.host - 数据库主机地址 @@ -304,40 +304,50 @@ async function backupMysqlDatabase() { await connection.connect(); + // 获取数据库中的所有表 + const [tables] = await connection.query(` + SELECT TABLE_NAME + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = ? + `, [config.database]); + const timestamp = moment().format('YYYYMMDDHHmmss'); const backupFile = path.join(backupDir, `backup-${timestamp}.sql`); const writeStream = fs.createWriteStream(backupFile); - // 获取表结构 - const [tableStructure] = await connection.query( - `SHOW CREATE TABLE \`${config.database}\`` - ); - - // 写入表结构 - writeStream.write(`-- Database table structure\n`); - writeStream.write(`DROP TABLE IF EXISTS \`${config.database}\`;\n`); - writeStream.write(`${tableStructure[0]['Create Table']};\n\n`); - - // 获取表数据 - const [rows] = await connection.query(`SELECT * FROM \`${config.database}\``); - - if (rows.length > 0) { - // 写入数据插入语句 - writeStream.write(`-- Data records\n`); - const columns = Object.keys(rows[0]); + // 遍历每个表并备份 + for (const table of tables) { + const tableName = table.TABLE_NAME; - for (const row of rows) { - const values = columns.map(column => { - const value = row[column]; - if (value === null) return 'NULL'; - if (typeof value === 'number') return value; - return `'${value.toString().replace(/'/g, "''")}'`; - }); + // 获取表结构 + const [tableStructure] = await connection.query(`SHOW CREATE TABLE \`${tableName}\``); + + // 写入表结构 + writeStream.write(`-- Table structure for ${tableName}\n`); + writeStream.write(`DROP TABLE IF EXISTS \`${tableName}\`;\n`); + writeStream.write(`${tableStructure[0]['Create Table']};\n\n`); + + // 获取表数据 + const [rows] = await connection.query(`SELECT * FROM \`${tableName}\``); + + if (rows.length > 0) { + writeStream.write(`-- Data for table ${tableName}\n`); + const columns = Object.keys(rows[0]); - writeStream.write( - `INSERT INTO \`${config.database}\` (${columns.map(c => '`'+c+'`').join(', ')}) ` + - `VALUES (${values.join(', ')});\n` - ); + for (const row of rows) { + const values = columns.map(column => { + const value = row[column]; + if (value === null) return 'NULL'; + if (typeof value === 'number') return value; + return `'${value.toString().replace(/'/g, "''")}'`; + }); + + writeStream.write( + `INSERT INTO \`${tableName}\` (${columns.map(c => '`'+c+'`').join(', ')}) ` + + `VALUES (${values.join(', ')});\n` + ); + } + writeStream.write('\n'); } } From 96326fc8dec7ba2047963fa82ec82592cb965371 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:48:26 +0800 Subject: [PATCH 26/51] Update dump.js --- dump.js | 77 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/dump.js b/dump.js index adb23c3..efb1c25 100644 --- a/dump.js +++ b/dump.js @@ -197,54 +197,76 @@ async function backupMysqlAllDatabase() { await connection.connect(); - // 获取所有表名 + // 修改获取表名的查询，排除系统表 const [tables] = await connection.query(` SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? - AND TABLE_NAME NOT LIKE 'sys_%' - AND TABLE_NAME NOT LIKE 'performance_%' - AND TABLE_NAME NOT LIKE 'innodb_%' + AND TABLE_TYPE = 'BASE TABLE' `, [config.database]); const timestamp = moment().format('YYYYMMDDHHmmss'); - const backupFile = path.join(backupDir, `mysql-backup-${timestamp}.sql`); + // const backupFile = path.join(backupDir, `mysql-backup-${timestamp}.sql`); + const backupFile = path.join(backupDir, `mysql-backup.sql`); const writeStream = fs.createWriteStream(backupFile); // 写入文件头部信息 - writeStream.write(`-- MySQL dump\n`); + writeStream.write(`-- MySQL dump for database ${config.database}\n`); writeStream.write(`-- Created at: ${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); + writeStream.write(`SET NAMES utf8mb4;\n`); writeStream.write(`SET FOREIGN_KEY_CHECKS=0;\n\n`); // 为每个表创建备份 for (const table of tables) { const tableName = table.TABLE_NAME; - // 获取表结构 - const [tableStructure] = await connection.query(`SHOW CREATE TABLE \`${tableName}\``); - writeStream.write(`-- Table structure: ${tableName}\n`); + // 获取建表语句 + const [createTable] = await connection.query(`SHOW CREATE TABLE \`${tableName}\``); + const createTableSql = createTable[0]['Create Table']; + + writeStream.write(`--\n-- Table structure for \`${tableName}\`\n--\n\n`); writeStream.write(`DROP TABLE IF EXISTS \`${tableName}\`;\n`); - writeStream.write(`${tableStructure[0]['Create Table']};\n\n`); + writeStream.write(`${createTableSql};\n\n`); // 获取表数据 const [rows] = await connection.query(`SELECT * FROM \`${tableName}\``); if (rows.length > 0) { - writeStream.write(`-- Table data: ${tableName}\n`); - const columns = Object.keys(rows[0]); + writeStream.write(`--\n-- Dumping data for table \`${tableName}\`\n--\n\n`); - for (const row of rows) { - const values = columns.map(column => { - const value = row[column]; - if (value === null) return 'NULL'; - if (typeof value === 'number') return value; - return `'${value.toString().replace(/'/g, "''")}'`; + // 批量处理插入语句，提高效率 + const batchSize = 100; + for (let i = 0; i < rows.length; i += batchSize) { + const batch = rows.slice(i, i + batchSize); + const values = batch.map(row => { + const rowValues = Object.values(row).map(value => { + if (value === null) return 'NULL'; + if (typeof value === 'boolean') return value ? 1 : 0; + if (typeof value === 'number') return value; + if (value instanceof Date) return `'${moment(value).format('YYYY-MM-DD HH:mm:ss')}'`; + if (Buffer.isBuffer(value)) return `0x${value.toString('hex')}`; + return `'${value.toString().replace(/[\0\x08\x09\x1a\n\r"'\\\%]/g, char => { + switch (char) { + case "\0": return "\\0"; + case "\x08": return "\\b"; + case "\x09": return "\\t"; + case "\x1a": return "\\z"; + case "\n": return "\\n"; + case "\r": return "\\r"; + case "\"": + case "'": + case "\\": + case "%": return "\\"+char; + default: return char; + } + })}'`; + }); + return `(${rowValues.join(', ')})`; }); - writeStream.write( - `INSERT INTO \`${tableName}\` (${columns.map(c => '`'+c+'`').join(', ')}) ` + - `VALUES (${values.join(', ')});\n` - ); + const columns = Object.keys(rows[0]).map(key => `\`${key}\``).join(', '); + writeStream.write(`INSERT INTO \`${tableName}\` (${columns}) VALUES\n`); + writeStream.write(`${values.join(',\n')};\n`); } writeStream.write('\n'); } @@ -264,7 +286,7 @@ async function backupMysqlAllDatabase() { } /** - * 备份MySQL数据库中的指定表数据���本地文件 + * 备份MySQL数据库中的指定表数据到本地文件 * * @param {Object} config - MySQL数据库连接配置 * @param {string} config.host - 数据库主机地址 @@ -312,7 +334,8 @@ async function backupMysqlDatabase() { `, [config.database]); const timestamp = moment().format('YYYYMMDDHHmmss'); - const backupFile = path.join(backupDir, `backup-${timestamp}.sql`); + // const backupFile = path.join(backupDir, `mysql-backup-${timestamp}.sql`); + const backupFile = path.join(backupDir, `mysql-backup.sql`); const writeStream = fs.createWriteStream(backupFile); // 遍历每个表并备份 @@ -392,7 +415,8 @@ async function backupPostgresAllDatabase() { } const timestamp = moment().format('YYYYMMDDHHmmss'); - const backupFile = path.join(backupDir, `postgres-backup-${timestamp}.sql`); + // const backupFile = path.join(backupDir, `postgres-backup-${timestamp}.sql`); + const backupFile = path.join(backupDir, `postgres-backup.sql`); const writeStream = fs.createWriteStream(backupFile); // 写入文件头部信息 @@ -509,7 +533,8 @@ async function backupPostgresDatabase() { await pool.connect(); const timestamp = moment().format('YYYYMMDDHHmmss'); - const backupFile = path.join(backupDir, `${tableName}-${timestamp}.sql`); + // const backupFile = path.join(backupDir, `postgres-${tableName}-backup-${timestamp}.sql`); + const backupFile = path.join(backupDir, `postgres-${tableName}-backup.sql`); const writeStream = fs.createWriteStream(backupFile); // 写入文件头部信息 From 15bd93978ebc083eb76d6db7fd5a71859b02725c Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:51:48 +0800 Subject: [PATCH 27/51] Update dump.js --- dump.js | 1 + 1 file changed, 1 insertion(+) diff --git a/dump.js b/dump.js index efb1c25..c42795d 100644 --- a/dump.js +++ b/dump.js @@ -204,6 +204,7 @@ async function backupMysqlAllDatabase() { WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' `, [config.database]); + console.log('tables:',tables) const timestamp = moment().format('YYYYMMDDHHmmss'); // const backupFile = path.join(backupDir, `mysql-backup-${timestamp}.sql`); From b6c63140e6bfd71eeeb3510c0638db80d06cb0e5 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:04:46 +0800 Subject: [PATCH 28/51] Update dump.js --- dump.js | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/dump.js b/dump.js index c42795d..f7e1f83 100644 --- a/dump.js +++ b/dump.js @@ -157,7 +157,7 @@ async function dumpMongoDatabase() { /** - * 备份MySQL数据库中的指定表数据到本地文件 + * 备份MySQL数据库中的所有的数据到本地文件 * * @param {Object} config - MySQL数据库连接配置 * @param {string} config.host - 数据库主机地址 @@ -174,7 +174,7 @@ async function dumpMongoDatabase() { * database: 'mydb' * }; * const backupDir = './backup'; - * await backupMysqlDatabase(config, backupDir); + * await backupMysqlAllDatabase(config, backupDir); * * @returns {Promise} * @throws {Error} 当数据库连接或备份过程出错时抛出异常 @@ -197,14 +197,19 @@ async function backupMysqlAllDatabase() { await connection.connect(); - // 修改获取表名的查询，排除系统表 + // 查询所有表 + // const [tables] = await connection.query(` + // SELECT TABLE_NAME + // FROM INFORMATION_SCHEMA.TABLES + // WHERE TABLE_SCHEMA = ? + // AND TABLE_TYPE = 'BASE TABLE' + // `, [config.database]); const [tables] = await connection.query(` - SELECT TABLE_NAME - FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = ? - AND TABLE_TYPE = 'BASE TABLE' + SHOW FULL TABLES + WHERE Table_type = 'BASE TABLE'; `, [config.database]); - console.log('tables:',tables) + + console.log('tables:', tables); const timestamp = moment().format('YYYYMMDDHHmmss'); // const backupFile = path.join(backupDir, `mysql-backup-${timestamp}.sql`); @@ -624,7 +629,7 @@ async function main() { console.log(`Now not within the specified time period ${process.env.DATABASE_BACKUP_TIME}, Database Dump Task has been cancelled.`); return; } - // 根据 DATABASE_TYPE 全局变判断要执行的备份类型 + // ���据 DATABASE_TYPE 全局变判断要执行的备份类型 if(process.env.DATABASE_TYPE){ let type = process.env.DATABASE_TYPE.toLowerCase() switch(type){ @@ -640,7 +645,8 @@ async function main() { if(process.env.DATABASE_NAME){ await backupMysqlDatabase() }else{ - await backupMysqlAllDatabase() + console.error('DATABASE_NAME env must be specified'); + // await backupMysqlAllDatabase() } break; case "pg": @@ -653,7 +659,7 @@ async function main() { } break; default: - console.error('不支持的数据库类型:', type); + console.error('Unsupported database type:', type); break; } } From ccc9e04ff0bbb7c5a1e51e85e382501ccbd46ca4 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:06:54 +0800 Subject: [PATCH 29/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 17c40b1..8f17de2 100644 --- a/README.md +++ b/README.md @@ -148,9 +148,9 @@ NEW Env For Database Dump (Only MongoDB has passed the test, other types databas * `DATABASE_BACKUP_TIME` - Optional. Database backup is automatically enabled within the specified time range (default 0-23) * `DATABASE_HOST` - Optional. Database host address * `DATABASE_PORT` - Optional. Database host port (if not specified, it will automatically follow the default port value of the database type) +* `DATABASE_NAME` - Optional. Database name, if not specified all database tables (except system tables) are used by default. Database type **mysql** MUST set `DATABASE_NAME`. * `DATABASE_USER` - Optional. Database Username * `DATABASE_PASSWORD` - Optional. Database password -* `DATABASE_NAME` - Optional. Database name. If not specified, all database tables (except system tables) are used by default. ## Volumes From 84c64c72422b0c3ffd7bd46fc1b2a45709f56007 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:11:36 +0800 Subject: [PATCH 30/51] Update dump.js --- dump.js | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/dump.js b/dump.js index f7e1f83..e2b9bfd 100644 --- a/dump.js +++ b/dump.js @@ -314,12 +314,12 @@ async function backupMysqlAllDatabase() { * @returns {Promise} * @throws {Error} 当数据库连接或备份过程出错时抛出异常 */ -async function backupMysqlDatabase() { +async function backupMysqlDatabase(databaseName) { const config = { host: process.env.DATABASE_HOST || process.env.MYSQL_HOST, user: process.env.DATABASE_USER || process.env.MYSQL_USER, password: process.env.DATABASE_PASSWORD || process.env.MYSQL_PASSWORD, - database: process.env.DATABASE_NAME || process.env.MYSQL_DATABASE, + database: databaseName, port: parseInt(process.env.DATABASE_PORT || process.env.MYSQL_PORT || 3306) }; const connection = mysql.createConnection(config).promise(); @@ -341,7 +341,7 @@ async function backupMysqlDatabase() { const timestamp = moment().format('YYYYMMDDHHmmss'); // const backupFile = path.join(backupDir, `mysql-backup-${timestamp}.sql`); - const backupFile = path.join(backupDir, `mysql-backup.sql`); + const backupFile = path.join(backupDir, `mysql-${databaseName}-backup.sql`); const writeStream = fs.createWriteStream(backupFile); // 遍历每个表并备份 @@ -516,17 +516,17 @@ async function backupPostgresAllDatabase() { * @param {number} config.port - 数据库端口号 * @param {string} backupDir - 备份文件保存目录 */ -async function backupPostgresDatabase() { +async function backupPostgresDatabase(tableName) { const config = { host: process.env.DATABASE_HOST || process.env.PG_HOST, user: process.env.DATABASE_USER || process.env.PG_USER, password: process.env.DATABASE_PASSWORD || process.env.PG_PASSWORD, - database: process.env.DATABASE_NAME || process.env.PG_DATABASE, + database: databaseName, port: parseInt(process.env.DATABASE_PORT || process.env.PG_PORT || 5432), }; const pool = new Pool(config); const backupDir = './dump'; - const tableName = process.env.TABLE_NAME; // 新增：从环境变量获取表名 + // const tableName = process.env.TABLE_NAME; // 新增：从环境变量获取表名 try { if (!tableName) { @@ -643,7 +643,13 @@ async function main() { break; case "mysql": if(process.env.DATABASE_NAME){ - await backupMysqlDatabase() + // 备份数据库可能为多个，使用,进行分割 + const databaseNames = process.env.DATABASE_NAME.split(','); + for(const databaseName of databaseNames){ + console.log(`Backuping database: ${databaseName}`); + await backupMysqlDatabase(databaseName) + console.log(`Backuping database: ${databaseName} completed!`); + } }else{ console.error('DATABASE_NAME env must be specified'); // await backupMysqlAllDatabase() @@ -653,7 +659,12 @@ async function main() { case "postgres": case "postgresql": if(process.env.TABLE_NAME){ - await backupPostgresDatabase() + const databaseNames = process.env.DATABASE_NAME.split(','); + for(const databaseName of databaseNames){ + console.log(`Backuping database: ${databaseName}`); + await backupPostgresDatabase(databaseName) + console.log(`Backuping database: ${databaseName} completed!`); + } }else{ await backupPostgresAllDatabase() } From 8755711d378025d2c15d5b21854f7b60bca97205 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:14:24 +0800 Subject: [PATCH 31/51] Update dump.js --- dump.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/dump.js b/dump.js index e2b9bfd..42a9c8c 100644 --- a/dump.js +++ b/dump.js @@ -112,9 +112,9 @@ async function dumpMongoAllDatabases() { * @returns {Promise} * @throws {Error} 当数据库连接或备份过程出错时抛出异常 */ -async function dumpMongoDatabase() { +async function dumpMongoDatabase(dbName) { const uri = `mongodb://${process.env.DATABASE_USER || process.env.MONGO_ROOT_USERNAME}:${process.env.DATABASE_PASSWORD || process.env.MONGO_ROOT_PASSWORD}@${process.env.DATABASE_HOST || process.env.MONGO_HOST}:${process.env.DATABASE_PORT || process.env.MONGO_PORT || '27017'}/`; - const dbName = process.env.DATABASE_NAME || process.env.MONGO_DATABASE; // 替换成你的数据库名称 + // const dbName = process.env.DATABASE_NAME || process.env.MONGO_DATABASE; // 替换成你的数据库名称 try { if(!dbName){ console.error('DATABASE_NAME or MONGO_DATEBASE is not set'); @@ -137,7 +137,13 @@ async function dumpMongoDatabase() { const collectionName = collection.collectionName; // 创建一个文件来存储 dump 数据 - const dumpFile = `${dumpDir}/${collectionName}.json`; + const dumpFile = `${dumpDir}/${dbName}/${collectionName}.json`; + + // 创建文件夹 + const collectionDir = `${dumpDir}/${dbName}`; + if (!fs.existsSync(collectionDir)) { + fs.mkdirSync(collectionDir); + } // 读取集合数据 const documents = await collection.find().toArray(); @@ -636,7 +642,13 @@ async function main() { case "mongo": case "mongodb": if(process.env.DATABASE_NAME){ - await dumpMongoDatabase() + // 备份数据库可能为多个，使用,进行分割 + const databaseNames = process.env.DATABASE_NAME.split(','); + for(const databaseName of databaseNames){ + console.log(`Backuping database: ${databaseName}`); + await dumpMongoDatabase(databaseName) + console.log(`Backuping database: ${databaseName} completed!`); + } }else{ await dumpMongoAllDatabases() } From 980656596787add4547770c3d89b1056d8b8dadf Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:17:10 +0800 Subject: [PATCH 32/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8f17de2..8999a7e 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ NEW Env For Database Dump (Only MongoDB has passed the test, other types databas * `DATABASE_BACKUP_TIME` - Optional. Database backup is automatically enabled within the specified time range (default 0-23) * `DATABASE_HOST` - Optional. Database host address * `DATABASE_PORT` - Optional. Database host port (if not specified, it will automatically follow the default port value of the database type) -* `DATABASE_NAME` - Optional. Database name, if not specified all database tables (except system tables) are used by default. Database type **mysql** MUST set `DATABASE_NAME`. +* `DATABASE_NAME` - Optional. Database name, Supports specifying multiple database names separated by ','. If not specified all database tables (except system tables) are used by default. Database type **mysql** MUST set `DATABASE_NAME`. * `DATABASE_USER` - Optional. Database Username * `DATABASE_PASSWORD` - Optional. Database password From 1cfc36bb056731f2fbdf7594ae14e1c6827a8050 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:54:03 +0800 Subject: [PATCH 33/51] Update dump.js --- dump.js | 113 +++----------------------------------------------------- 1 file changed, 5 insertions(+), 108 deletions(-) diff --git a/dump.js b/dump.js index 42a9c8c..ac26a87 100644 --- a/dump.js +++ b/dump.js @@ -410,12 +410,12 @@ async function backupMysqlDatabase(databaseName) { * @param {number} config.port - 数据库端口号 * @param {string} backupDir - 备份文件保存目录 */ -async function backupPostgresAllDatabase() { +async function backupPostgresDatabase(databaseName) { const config = { host: process.env.DATABASE_HOST || process.env.PG_HOST, user: process.env.DATABASE_USER || process.env.PG_USER, password: process.env.DATABASE_PASSWORD || process.env.PG_PASSWORD, - database: process.env.DATABASE_NAME || process.env.PG_DATABASE, + database: databaseName, port: parseInt(process.env.DATABASE_PORT || process.env.PG_PORT || 5432), }; const pool = new Pool(config); @@ -428,7 +428,7 @@ async function backupPostgresAllDatabase() { const timestamp = moment().format('YYYYMMDDHHmmss'); // const backupFile = path.join(backupDir, `postgres-backup-${timestamp}.sql`); - const backupFile = path.join(backupDir, `postgres-backup.sql`); + const backupFile = path.join(backupDir, `postgres-${databaseName}-backup.sql`); const writeStream = fs.createWriteStream(backupFile); // 写入文件头部信息 @@ -511,109 +511,6 @@ async function backupPostgresAllDatabase() { } } -/** - * 备份PostgreSQL数据库中的指定的表数据到本地文件 - * - * @param {Object} config - PostgreSQL数据库连接配置 - * @param {string} config.user - 数据库用户名 - * @param {string} config.host - 数据库主机地址 - * @param {string} config.database - 数据库名称 - * @param {string} config.password - 数据库密码 - * @param {number} config.port - 数据库端口号 - * @param {string} backupDir - 备份文件保存目录 - */ -async function backupPostgresDatabase(tableName) { - const config = { - host: process.env.DATABASE_HOST || process.env.PG_HOST, - user: process.env.DATABASE_USER || process.env.PG_USER, - password: process.env.DATABASE_PASSWORD || process.env.PG_PASSWORD, - database: databaseName, - port: parseInt(process.env.DATABASE_PORT || process.env.PG_PORT || 5432), - }; - const pool = new Pool(config); - const backupDir = './dump'; - // const tableName = process.env.TABLE_NAME; // 新增：从环境变量获取表名 - - try { - if (!tableName) { - throw new Error('TABLE_NAME environment variable is not set'); - } - - if (!fs.existsSync(backupDir)) { - fs.mkdirSync(backupDir); - } - - await pool.connect(); - const timestamp = moment().format('YYYYMMDDHHmmss'); - // const backupFile = path.join(backupDir, `postgres-${tableName}-backup-${timestamp}.sql`); - const backupFile = path.join(backupDir, `postgres-${tableName}-backup.sql`); - const writeStream = fs.createWriteStream(backupFile); - - // 写入文件头部信息 - writeStream.write(`-- PostgreSQL dump of table ${tableName}\n`); - writeStream.write(`-- Created at: ${moment().format('YYYY-MM-DD HH:mm:ss')}\n\n`); - - // 获取表结构 - const { rows: columns } = await pool.query(` - SELECT column_name, data_type, character_maximum_length, - is_nullable, column_default - FROM information_schema.columns - WHERE table_name = $1 - ORDER BY ordinal_position - `, [tableName]); - - // 写入表结构 - writeStream.write(`-- Table structure\n`); - writeStream.write(`DROP TABLE IF EXISTS "${tableName}";\n`); - writeStream.write(`CREATE TABLE "${tableName}" (\n`); - - const columnDefs = columns.map(col => { - let def = ` "${col.column_name}" ${col.data_type}`; - if (col.character_maximum_length) { - def += `(${col.character_maximum_length})`; - } - if (col.is_nullable === 'NO') { - def += ' NOT NULL'; - } - if (col.column_default) { - def += ` DEFAULT ${col.column_default}`; - } - return def; - }); - - writeStream.write(columnDefs.join(',\n')); - writeStream.write('\n);\n\n'); - - // 获取并写入表数据 - const { rows: data } = await pool.query(`SELECT * FROM "${tableName}"`); - if (data.length > 0) { - writeStream.write(`-- Table data\n`); - for (const row of data) { - const values = Object.values(row).map(val => { - if (val === null) return 'NULL'; - if (typeof val === 'number') return val; - return `'${val.toString().replace(/'/g, "''")}'`; - }); - - writeStream.write( - `INSERT INTO "${tableName}" (${Object.keys(row).map(k => `"${k}"`).join(', ')}) ` + - `VALUES (${values.join(', ')});\n` - ); - } - } - - writeStream.end(); - console.log(`Table ${tableName} has been backed up to ${backupFile}`); - console.log('Database backup completed!'); - - } catch (err) { - console.error('Error during database backup:', err); - } finally { - if (pool) { - await pool.end(); - } - } -} // 创建一个主函数来处理数据库备份 async function main() { @@ -670,7 +567,7 @@ async function main() { case "pg": case "postgres": case "postgresql": - if(process.env.TABLE_NAME){ + if(process.env.DATABASE_NAME){ const databaseNames = process.env.DATABASE_NAME.split(','); for(const databaseName of databaseNames){ console.log(`Backuping database: ${databaseName}`); @@ -678,7 +575,7 @@ async function main() { console.log(`Backuping database: ${databaseName} completed!`); } }else{ - await backupPostgresAllDatabase() + console.error('DATABASE_NAME env must be specified'); } break; default: From 95651a7b066a4c6e38fd0368f0683823e0f47116 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:59:31 +0800 Subject: [PATCH 34/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8999a7e..e46ec7d 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ NEW Env For Database Dump (Only MongoDB has passed the test, other types databas * `DATABASE_BACKUP_TIME` - Optional. Database backup is automatically enabled within the specified time range (default 0-23) * `DATABASE_HOST` - Optional. Database host address * `DATABASE_PORT` - Optional. Database host port (if not specified, it will automatically follow the default port value of the database type) -* `DATABASE_NAME` - Optional. Database name, Supports specifying multiple database names separated by ','. If not specified all database tables (except system tables) are used by default. Database type **mysql** MUST set `DATABASE_NAME`. +* `DATABASE_NAME` - Optional. Database name, Supports specifying multiple database names separated by ','. If not specified all database tables (except system tables) are used by default. Database type **mysql** AND **postgresql** MUST set. * `DATABASE_USER` - Optional. Database Username * `DATABASE_PASSWORD` - Optional. Database password From 5fcd25e675dbc8e01c4fff0eb3f69b4537d73b25 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:32:58 +0800 Subject: [PATCH 35/51] Update backup.sh --- backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.sh b/backup.sh index 342863a..c19f80b 100755 --- a/backup.sh +++ b/backup.sh @@ -38,7 +38,7 @@ backupDatebase(){ echo "MongoDB Dump List:" ls -l /data/dump else - echo "./dump Folder Empty, MongoDB Dump Fail." + echo "./dump Folder Empty, ${DATABASE_TYPE} Dump Fail." fi echo "### End ${DATABASE_TYPE} Dump ###" echo "" From fbb04da9833d09939666c077b7cf220df15f177c Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:33:58 +0800 Subject: [PATCH 36/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e46ec7d..103b964 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ The container is set up by setting [environment variables](https://docs.docker.c * `OS_INTERFACE` - Optional. When using restic with OpenStack Swift container. * `OS_IDENTITY_API_VERSION` - Optional. When using restic with OpenStack Swift container. -NEW Env For Database Dump (Only MongoDB has passed the test, other types databases have not been tested yet) +NEW Env For Database Dump * `DATABASE_TYPE` - Optional. Specify the database type (mongo/mongodb/mysql/pg/postgres/postgresql). Specifying this option will enable the database backup function. * `DATABASE_BACKUP_TIME` - Optional. Database backup is automatically enabled within the specified time range (default 0-23) * `DATABASE_HOST` - Optional. Database host address From 166cea7c077d9261fd3bee5d7033d557d57a8876 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:18:25 +0800 Subject: [PATCH 37/51] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index bdc2025..e120cc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,7 @@ ENV RESTIC_TAG="" ENV NFS_TARGET="" ENV BACKUP_CRON="0 */6 * * *" ENV CHECK_CRON="" +ENV PRUNE_CRON="" ENV RESTIC_INIT_ARGS="" ENV RESTIC_FORGET_ARGS="" ENV RESTIC_JOB_ARGS="" From 20e3d2688d472386ddd956a4dd520495ed7ce3cd Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:19:30 +0800 Subject: [PATCH 38/51] Update entry.sh --- entry.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/entry.sh b/entry.sh index c192863..128605d 100755 --- a/entry.sh +++ b/entry.sh @@ -44,6 +44,12 @@ if [ -n "${CHECK_CRON}" ]; then echo "${CHECK_CRON} /usr/bin/flock -n /var/run/backup.lock /bin/check >> /var/log/cron.log 2>&1" >> /var/spool/cron/crontabs/root fi +# If PRUNE_CRON is set we will enable automatic backup checking +if [ -n "${PRUNE_CRON}" ]; then + echo "Setup check cron job with cron expression PRUNE_CRON: ${PRUNE_CRON}" + echo "${PRUNE_CRON} /usr/bin/flock -n /var/run/backup.lock /bin/prune >> /var/log/cron.log 2>&1" >> /var/spool/cron/crontabs/root +fi + # Make sure the file exists before we start tail touch /var/log/cron.log From 253a104b8e88c1ddf381ddca2ea6cd8d515a40ba Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:20:16 +0800 Subject: [PATCH 39/51] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e120cc8..3298389 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,6 +63,7 @@ VOLUME /data COPY backup.sh /bin/backup COPY check.sh /bin/check +COPY prune.sh /bin/prune COPY entry.sh /entry.sh RUN mkdir /script && \ From d2d60b16794ec1e8ce7a4ab28e1b79c5428bcc82 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:24:07 +0800 Subject: [PATCH 40/51] Create prune.sh --- prune.sh | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 prune.sh diff --git a/prune.sh b/prune.sh new file mode 100644 index 0000000..1349d76 --- /dev/null +++ b/prune.sh @@ -0,0 +1,72 @@ +#!/bin/sh + +lastLogfile="/var/log/prune-last.log" +lastMailLogfile="/var/log/prune-mail-last.log" +lastMicrosoftTeamsLogfile="/var/log/prune-microsoft-teams-last.log" + +copyErrorLog() { + cp ${lastLogfile} /var/log/prune-error-last.log +} + +logLast() { + echo "$1" >> ${lastLogfile} +} + +if [ -f "/hooks/pre-prune.sh" ]; then + echo "Starting pre-prune script ..." + /hooks/pre-prune.sh +else + echo "Pre-prune script not found ..." +fi + +start=`date +%s` +rm -f ${lastLogfile} ${lastMailLogfile} +echo "Starting Prune at $(date +"%Y-%m-%d %H:%M:%S")" +echo "Starting Prune at $(date)" >> ${lastLogfile} +logLast "PRUNE_CRON: ${PRUNE_CRON}" +logLast "RESTIC_DATA_SUBSET: ${RESTIC_DATA_SUBSET}" +logLast "RESTIC_REPOSITORY: ${RESTIC_REPOSITORY}" +logLast "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}" + +# Do not save full prune log to logfile but to prune-last.log +restic prune >> ${lastLogfile} 2>&1 +pruneRC=$? +logLast "Finished prune at $(date)" +if [[ $pruneRC == 0 ]]; then + echo "Prune Successful" +else + echo "Prune Failed with Status ${pruneRC}" + restic unlock + copyErrorLog +fi + +end=`date +%s` +echo "Finished Prune at $(date +"%Y-%m-%d %H:%M:%S") after $((end-start)) seconds" + +if [ -n "${TEAMS_WEBHOOK_URL}" ]; then + teamsTitle="Restic Last Prune Log" + teamsMessage=$( cat ${lastLogfile} | sed 's/"/\"/g' | sed "s/'/\'/g" | sed ':a;N;$!ba;s/\n/\n\n/g' ) + teamsReqBody="{\"title\": \"${teamsTitle}\", \"text\": \"${teamsMessage}\" }" + sh -c "curl -H 'Content-Type: application/json' -d '${teamsReqBody}' '${TEAMS_WEBHOOK_URL}' > ${lastMicrosoftTeamsLogfile} 2>&1" + if [ $? == 0 ]; then + echo "Microsoft Teams notification successfully sent." + else + echo "Sending Microsoft Teams notification FAILED. Prune ${lastMicrosoftTeamsLogfile} for further information." + fi +fi + +if [ -n "${MAILX_ARGS}" ]; then + sh -c "mail -v -S sendwait ${MAILX_ARGS} < ${lastLogfile} > ${lastMailLogfile} 2>&1" + if [ $? == 0 ]; then + echo "Mail notification successfully sent." + else + echo "Sending mail notification FAILED. Prune ${lastMailLogfile} for further information." + fi +fi + +if [ -f "/hooks/post-prune.sh" ]; then + echo "Starting post-prune script ..." + /hooks/post-prune.sh $pruneRC +else + echo "Post-prune script not found ..." +fi From 8f7bf8d06d57cd24a099450ed52fd858d8b4341c Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:36:58 +0800 Subject: [PATCH 41/51] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 103b964..44f5574 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ The container is set up by setting [environment variables](https://docs.docker.c * `NFS_TARGET` - Optional. If set, the given NFS is mounted, i.e. `mount -o nolock -v ${NFS_TARGET} /mnt/restic`. `RESTIC_REPOSITORY` must remain its default value! * `BACKUP_CRON` - A cron expression to run the backup. Note: The cron daemon uses UTC time zone. Default: `0 */6 * * *` aka every 6 hours. * `CHECK_CRON` - Optional. A cron expression to run data integrity check (`restic check`). If left unset, data will not be checked. Note: The cron daemon uses UTC time zone. Example: `0 23 * * 3` to run 11PM every Tuesday. +* `PRUNE_CRON` - Optional. A cron expression to Remove unneeded data from the repository (`restic prune`). If left unset, data will not be prune. * `RESTIC_FORGET_ARGS` - Optional. Only if specified, `restic forget` is run with the given arguments after each backup. Example value: `-e "RESTIC_FORGET_ARGS=--prune --keep-last 10 --keep-hourly 24 --keep-daily 7 --keep-weekly 52 --keep-monthly 120 --keep-yearly 100"` * `RESTIC_INIT_ARGS` - Optional. Allows specifying extra arguments to `restic init` such as a password file with `--password-file`. * `RESTIC_JOB_ARGS` - Optional. Allows specifying extra arguments to the backup job such as limiting bandwith with `--limit-upload` or excluding file masks with `--exclude`. @@ -217,6 +218,8 @@ services: volumes: - /volume1/Backup:/data/Backup:ro # Backup /volume1/Backup from host - /home/user:/data/home:ro # Backup /home/user from host + - ./post-backup.sh:/custem/post-backup.sh:ro # For k8s file in custem folder will auto copy to hooks. Run script post-backup.sh after every backup + - ./post-check.sh:/custem/post-check.sh:ro # For k8s file in custem folder will auto copy to hooks.Run script post-check.sh after every check - ./post-backup.sh:/hooks/post-backup.sh:ro # Run script post-backup.sh after every backup - ./post-check.sh:/hooks/post-check.sh:ro # Run script post-check.sh after every check - ./ssh:/root/.ssh # SSH keys and config so we can login to "storageserver" without password From ffed5b6ce8e60d58c57d2d6e4e3e0875d2ff28f4 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:39:03 +0800 Subject: [PATCH 42/51] Update Dockerfile --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 3298389..8b1d2c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,8 +62,11 @@ RUN mkdir /.cache && \ VOLUME /data COPY backup.sh /bin/backup +RUN chmod u+x /bin/backup COPY check.sh /bin/check +RUN chmod u+x /bin/check COPY prune.sh /bin/prune +RUN chmod u+x /bin/prune COPY entry.sh /entry.sh RUN mkdir /script && \ From 2db4a04285b5121a5d5a14358f15c9cf5586ceff Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:06:12 +0800 Subject: [PATCH 43/51] Update prune.sh --- prune.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/prune.sh b/prune.sh index 1349d76..4c167bc 100644 --- a/prune.sh +++ b/prune.sh @@ -29,7 +29,29 @@ logLast "RESTIC_REPOSITORY: ${RESTIC_REPOSITORY}" logLast "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}" # Do not save full prune log to logfile but to prune-last.log -restic prune >> ${lastLogfile} 2>&1 + +if [ -n "${RESTIC_FORGET_ARGS}" ]; then + echo "Prune about old snapshots based on RESTIC_FORGET_ARGS = ${RESTIC_FORGET_ARGS}" + restic forget --prune ${RESTIC_FORGET_ARGS} >> ${lastLogfile} 2>&1 + rc=$? + logLast "Finished forget at $(date)" + if [[ $rc == 0 ]]; then + echo "Prune Successful" + else + echo "Prune Failed with Status ${rc}" + restic unlock + copyErrorLog + fi +else + restic prune >> ${lastLogfile} 2>&1 + if [[ $rc == 0 ]]; then + echo "Prune Successful" + else + echo "Prune Failed with Status ${rc}" + restic unlock + copyErrorLog + fi +fi pruneRC=$? logLast "Finished prune at $(date)" if [[ $pruneRC == 0 ]]; then From 1811c935c75e52429311ab9554c539a51464b699 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:41:42 +0800 Subject: [PATCH 44/51] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 8b1d2c4..bb70887 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ ENV NFS_TARGET="" ENV BACKUP_CRON="0 */6 * * *" ENV CHECK_CRON="" ENV PRUNE_CRON="" +ENV PRUNE_ALL_HOST="" ENV RESTIC_INIT_ARGS="" ENV RESTIC_FORGET_ARGS="" ENV RESTIC_JOB_ARGS="" From ce1a1fa8a4c084671ff73d5bf6f4f151963d5395 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:42:51 +0800 Subject: [PATCH 45/51] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 44f5574..7d4d5b1 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ The container is set up by setting [environment variables](https://docs.docker.c * `BACKUP_CRON` - A cron expression to run the backup. Note: The cron daemon uses UTC time zone. Default: `0 */6 * * *` aka every 6 hours. * `CHECK_CRON` - Optional. A cron expression to run data integrity check (`restic check`). If left unset, data will not be checked. Note: The cron daemon uses UTC time zone. Example: `0 23 * * 3` to run 11PM every Tuesday. * `PRUNE_CRON` - Optional. A cron expression to Remove unneeded data from the repository (`restic prune`). If left unset, data will not be prune. +* `PRUNE_ALL_HOST` - Optional. set 'true' will prune all Host name on snapshots. * `RESTIC_FORGET_ARGS` - Optional. Only if specified, `restic forget` is run with the given arguments after each backup. Example value: `-e "RESTIC_FORGET_ARGS=--prune --keep-last 10 --keep-hourly 24 --keep-daily 7 --keep-weekly 52 --keep-monthly 120 --keep-yearly 100"` * `RESTIC_INIT_ARGS` - Optional. Allows specifying extra arguments to `restic init` such as a password file with `--password-file`. * `RESTIC_JOB_ARGS` - Optional. Allows specifying extra arguments to the backup job such as limiting bandwith with `--limit-upload` or excluding file masks with `--exclude`. From 19f3c74648ffe332aa270db5f603ef1aaffa4941 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:45:41 +0800 Subject: [PATCH 46/51] Update prune.sh --- prune.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/prune.sh b/prune.sh index 4c167bc..746245a 100644 --- a/prune.sh +++ b/prune.sh @@ -32,6 +32,10 @@ logLast "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}" if [ -n "${RESTIC_FORGET_ARGS}" ]; then echo "Prune about old snapshots based on RESTIC_FORGET_ARGS = ${RESTIC_FORGET_ARGS}" + # use keep-tag $(hostname) for forget all snapshots that name not with $(hostname) + if [ "${PRUNE_ALL_HOST}" = "true" ]; then + restic forget --prune ${RESTIC_FORGET_ARGS} --keep-tag $(hostname) >> ${lastLogfile} 2>&1 + fi restic forget --prune ${RESTIC_FORGET_ARGS} >> ${lastLogfile} 2>&1 rc=$? logLast "Finished forget at $(date)" From f26f0d1c2404ccdb7be43f39c1514bebc3536976 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 18:20:49 +0800 Subject: [PATCH 47/51] Update Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bb70887..8b1d2c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,6 @@ ENV NFS_TARGET="" ENV BACKUP_CRON="0 */6 * * *" ENV CHECK_CRON="" ENV PRUNE_CRON="" -ENV PRUNE_ALL_HOST="" ENV RESTIC_INIT_ARGS="" ENV RESTIC_FORGET_ARGS="" ENV RESTIC_JOB_ARGS="" From 5620916d39f74e8f6715b1cbbc308692ddee644e Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 18:22:10 +0800 Subject: [PATCH 48/51] Update prune.sh --- prune.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/prune.sh b/prune.sh index 746245a..4c167bc 100644 --- a/prune.sh +++ b/prune.sh @@ -32,10 +32,6 @@ logLast "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}" if [ -n "${RESTIC_FORGET_ARGS}" ]; then echo "Prune about old snapshots based on RESTIC_FORGET_ARGS = ${RESTIC_FORGET_ARGS}" - # use keep-tag $(hostname) for forget all snapshots that name not with $(hostname) - if [ "${PRUNE_ALL_HOST}" = "true" ]; then - restic forget --prune ${RESTIC_FORGET_ARGS} --keep-tag $(hostname) >> ${lastLogfile} 2>&1 - fi restic forget --prune ${RESTIC_FORGET_ARGS} >> ${lastLogfile} 2>&1 rc=$? logLast "Finished forget at $(date)" From d3d541730f37d022e6af736b73a7d297788b6f3b Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Fri, 20 Dec 2024 18:22:33 +0800 Subject: [PATCH 49/51] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7d4d5b1..44f5574 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,6 @@ The container is set up by setting [environment variables](https://docs.docker.c * `BACKUP_CRON` - A cron expression to run the backup. Note: The cron daemon uses UTC time zone. Default: `0 */6 * * *` aka every 6 hours. * `CHECK_CRON` - Optional. A cron expression to run data integrity check (`restic check`). If left unset, data will not be checked. Note: The cron daemon uses UTC time zone. Example: `0 23 * * 3` to run 11PM every Tuesday. * `PRUNE_CRON` - Optional. A cron expression to Remove unneeded data from the repository (`restic prune`). If left unset, data will not be prune. -* `PRUNE_ALL_HOST` - Optional. set 'true' will prune all Host name on snapshots. * `RESTIC_FORGET_ARGS` - Optional. Only if specified, `restic forget` is run with the given arguments after each backup. Example value: `-e "RESTIC_FORGET_ARGS=--prune --keep-last 10 --keep-hourly 24 --keep-daily 7 --keep-weekly 52 --keep-monthly 120 --keep-yearly 100"` * `RESTIC_INIT_ARGS` - Optional. Allows specifying extra arguments to `restic init` such as a password file with `--password-file`. * `RESTIC_JOB_ARGS` - Optional. Allows specifying extra arguments to the backup job such as limiting bandwith with `--limit-upload` or excluding file masks with `--exclude`. From 017c8140659610122e7eba645b33e9c03e59cfc7 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Tue, 24 Dec 2024 12:33:48 +0800 Subject: [PATCH 50/51] Update entry.sh --- entry.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/entry.sh b/entry.sh index 128605d..c8b1725 100755 --- a/entry.sh +++ b/entry.sh @@ -2,10 +2,10 @@ echo "Starting container ..." -# Copy Custem Hooks Script File -if [ -d "/custem" ]; then +# Copy Custom Hooks Script File +if [ -d "/custom" ]; then mkdir /hooks - cp -L /custem/* /hooks + cp -L /custom/* /hooks chmod u+x /hooks/* # Run npm install npm install From 2655553440b243c546db349fb003207c29df64a8 Mon Sep 17 00:00:00 2001 From: emengweb <31469739+emengweb@users.noreply.github.com> Date: Tue, 24 Dec 2024 12:34:14 +0800 Subject: [PATCH 51/51] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 44f5574..f9425b7 100644 --- a/README.md +++ b/README.md @@ -218,8 +218,8 @@ services: volumes: - /volume1/Backup:/data/Backup:ro # Backup /volume1/Backup from host - /home/user:/data/home:ro # Backup /home/user from host - - ./post-backup.sh:/custem/post-backup.sh:ro # For k8s file in custem folder will auto copy to hooks. Run script post-backup.sh after every backup - - ./post-check.sh:/custem/post-check.sh:ro # For k8s file in custem folder will auto copy to hooks.Run script post-check.sh after every check + - ./post-backup.sh:/custom/post-backup.sh:ro # For k8s file in custom folder will auto copy to hooks. Run script post-backup.sh after every backup + - ./post-check.sh:/custom/post-check.sh:ro # For k8s file in custom folder will auto copy to hooks.Run script post-check.sh after every check - ./post-backup.sh:/hooks/post-backup.sh:ro # Run script post-backup.sh after every backup - ./post-check.sh:/hooks/post-check.sh:ro # Run script post-check.sh after every check - ./ssh:/root/.ssh # SSH keys and config so we can login to "storageserver" without password