From ecb8a3ff9a4ecfd90ad7a5f641643ba199505255 Mon Sep 17 00:00:00 2001 From: DJKnaeckebrot Date: Wed, 3 Apr 2024 14:07:47 +0200 Subject: [PATCH 1/2] MongoDB Implementation Due to issues with MircoORM I had to comment out the the backup and restore features/functions in the database and APi but as this is MongoDB explicit this should not matter --- .env.example | 6 +- .../migrations/Migration20240403115346.js | 11 ++ package-lock.json | 18 +++ package.json | 1 + src/api/controllers/database.ts | 78 ++++----- src/commands/Admin/prefix.ts | 2 +- src/configs/database.ts | 8 +- src/entities/BaseEntity.ts | 8 +- src/entities/Guild.ts | 8 +- src/entities/Image.ts | 3 - src/entities/Pastebin.ts | 10 +- src/entities/Stat.ts | 8 +- src/entities/User.ts | 8 +- src/services/Database.ts | 148 +++++++++--------- src/services/Stats.ts | 53 +++---- src/utils/functions/synchronizer.ts | 12 +- src/utils/types/environment.ts | 2 +- 17 files changed, 208 insertions(+), 176 deletions(-) create mode 100644 database/migrations/Migration20240403115346.js diff --git a/.env.example b/.env.example index f044d4c2..bd0f1731 100644 --- a/.env.example +++ b/.env.example @@ -4,11 +4,7 @@ TEST_GUILD_ID="TEST_GUILD_ID" BOT_OWNER_ID="YOUR_DISCORD_ID" # database -# DATABASE_HOST="database" # if you use docker-compose, it should be the name of the service hosting the database -# DATABASE_PORT=5432 -# DATABASE_NAME="tscord_bot" -# DATABASE_USER="tscord" -# DATABASE_PASSWORD="tscord123" +DATABASE_HOST="mongodb+srv://:@/?retryWrites=true&w=majority" # api # API_PORT=4000 diff --git a/database/migrations/Migration20240403115346.js b/database/migrations/Migration20240403115346.js new file mode 100644 index 00000000..f4627334 --- /dev/null +++ b/database/migrations/Migration20240403115346.js @@ -0,0 +1,11 @@ +'use strict'; +Object.defineProperty(exports, '__esModule', { value: true }); +const { Migration } = require('@mikro-orm/migrations-mongodb'); + +class Migration20240403115346 extends Migration { + + async up() { + } + +} +exports.Migration20240403115346 = Migration20240403115346; diff --git a/package-lock.json b/package-lock.json index 29991e99..bc7bfc2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@mikro-orm/core": "~5.6.16", "@mikro-orm/mariadb": "~5.6.16", "@mikro-orm/migrations": "~5.6.16", + "@mikro-orm/migrations-mongodb": "~5.6.16", "@mikro-orm/mongodb": "~5.6.16", "@mikro-orm/mysql": "~5.6.16", "@mikro-orm/postgresql": "~5.6.16", @@ -2071,6 +2072,23 @@ "@mikro-orm/core": "^5.0.0" } }, + "node_modules/@mikro-orm/migrations-mongodb": { + "version": "5.6.16", + "resolved": "https://registry.npmjs.org/@mikro-orm/migrations-mongodb/-/migrations-mongodb-5.6.16.tgz", + "integrity": "sha512-c1/sKAvX2WZH/Wiq+CAfuB4Fv0A1rw9vslkPEB043/Kpvb3fn4lbV3/2O/hlaH4juzvfDC9kF+o72zQ69uu05A==", + "dependencies": { + "@mikro-orm/mongodb": "~5.6.16", + "fs-extra": "11.1.1", + "mongodb": "4.13.0", + "umzug": "3.2.1" + }, + "engines": { + "node": ">= 14.0.0" + }, + "peerDependencies": { + "@mikro-orm/core": "^5.0.0" + } + }, "node_modules/@mikro-orm/mongodb": { "version": "5.6.16", "resolved": "https://registry.npmjs.org/@mikro-orm/mongodb/-/mongodb-5.6.16.tgz", diff --git a/package.json b/package.json index c6da7446..06e89d2e 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@mikro-orm/core": "~5.6.16", "@mikro-orm/mariadb": "~5.6.16", "@mikro-orm/migrations": "~5.6.16", + "@mikro-orm/migrations-mongodb": "~5.6.16", "@mikro-orm/mongodb": "~5.6.16", "@mikro-orm/mysql": "~5.6.16", "@mikro-orm/postgresql": "~5.6.16", diff --git a/src/api/controllers/database.ts b/src/api/controllers/database.ts index d3c31643..f9296452 100644 --- a/src/api/controllers/database.ts +++ b/src/api/controllers/database.ts @@ -26,50 +26,52 @@ export class DatabaseController extends BaseController { }) } - @Post('/backup') - async generateBackup() { - const snapshotName = `snapshot-${formatDate(new Date(), 'onlyDateFileName')}-manual-${Date.now()}` - const success = await this.db.backup(snapshotName) + // Commented due to Type Issues from MicroORM - if (success) { - return { - message: 'Backup generated', - data: { - snapshotName: `${snapshotName}.txt`, - }, - } - } else { - throw new InternalServerError('Couldn\'t generate backup, see the logs for more information') - } - } + // @Post('/backup') + // async generateBackup() { + // const snapshotName = `snapshot-${formatDate(new Date(), 'onlyDateFileName')}-manual-${Date.now()}` + // const success = await this.db.backup(snapshotName) - @Post('/restore') - async restoreBackup( - @Required() @BodyParams('snapshotName') snapshotName: string - ) { - const success = await this.db.restore(snapshotName) + // if (success) { + // return { + // message: 'Backup generated', + // data: { + // snapshotName: `${snapshotName}.txt`, + // }, + // } + // } else { + // throw new InternalServerError('Couldn\'t generate backup, see the logs for more information') + // } + // } - if (success) - return { message: 'Backup restored' } - else throw new InternalServerError('Couldn\'t restore backup, see the logs for more information') - } + // @Post('/restore') + // async restoreBackup( + // @Required() @BodyParams('snapshotName') snapshotName: string + // ) { + // const success = await this.db.restore(snapshotName) - @Get('/backups') - async getBackups() { - const backupPath = databaseConfig.backup.path - if (!backupPath) - throw new InternalServerError('Backup path not set, couldn\'t find backups') + // if (success) + // return { message: 'Backup restored' } + // else throw new InternalServerError('Couldn\'t restore backup, see the logs for more information') + // } - const backupList = this.db.getBackupList() + // @Get('/backups') + // async getBackups() { + // const backupPath = databaseConfig.backup.path + // if (!backupPath) + // throw new InternalServerError('Backup path not set, couldn\'t find backups') - if (backupList) - return backupList - else throw new InternalServerError('Couldn\'t get backup list, see the logs for more information') - } + // const backupList = this.db.getBackupList() - @Get('/size') - async size() { - return await this.db.getSize() - } + // if (backupList) + // return backupList + // else throw new InternalServerError('Couldn\'t get backup list, see the logs for more information') + // } + + // @Get('/size') + // async size() { + // return await this.db.getSize() + // } } diff --git a/src/commands/Admin/prefix.ts b/src/commands/Admin/prefix.ts index 8f81547c..1abcea8b 100644 --- a/src/commands/Admin/prefix.ts +++ b/src/commands/Admin/prefix.ts @@ -34,7 +34,7 @@ export default class PrefixCommand { { localize }: InteractionData ) { const guild = resolveGuild(interaction) - const guildData = await this.db.get(Guild).findOne({ id: guild?.id || '' }) + const guildData = await this.db.get(Guild).findOne({ guildId: guild?.id || '' }) if (guildData) { guildData.prefix = prefix || null diff --git a/src/configs/database.ts b/src/configs/database.ts index dab4eb51..7ef92593 100644 --- a/src/configs/database.ts +++ b/src/configs/database.ts @@ -27,14 +27,14 @@ const envMikroORMConfig = { /** * SQLite */ - type: 'better-sqlite', // or 'sqlite' - dbName: `${databaseConfig.path}db.sqlite`, + //type: 'better-sqlite', // or 'sqlite' + //dbName: `${databaseConfig.path}db.sqlite`, /** * MongoDB */ - // type: 'mongo', - // clientUrl: env['DATABASE_HOST'], + type: 'mongo', + clientUrl: env['DATABASE_HOST'], /** * PostgreSQL diff --git a/src/entities/BaseEntity.ts b/src/entities/BaseEntity.ts index 73641471..b962c8dd 100644 --- a/src/entities/BaseEntity.ts +++ b/src/entities/BaseEntity.ts @@ -1,6 +1,12 @@ -import { Property } from '@mikro-orm/core' +import { PrimaryKey, Property, SerializedPrimaryKey } from "@mikro-orm/core" +import { ObjectId } from "@mikro-orm/mongodb"; export abstract class CustomBaseEntity { + @PrimaryKey() + _id: ObjectId; + + @SerializedPrimaryKey() + id!: string; // won't be saved in the database @Property() createdAt: Date = new Date() diff --git a/src/entities/Guild.ts b/src/entities/Guild.ts index 3188b27a..168e98df 100644 --- a/src/entities/Guild.ts +++ b/src/entities/Guild.ts @@ -1,4 +1,4 @@ -import { Entity, EntityRepositoryType, PrimaryKey, Property } from '@mikro-orm/core' +import { Entity, EntityRepositoryType, Property } from '@mikro-orm/core' import { EntityRepository } from '@mikro-orm/sqlite' import { CustomBaseEntity } from './BaseEntity' @@ -12,8 +12,8 @@ export class Guild extends CustomBaseEntity { [EntityRepositoryType]?: GuildRepository - @PrimaryKey({ autoincrement: false }) - id!: string + @Property() + guildId!: string @Property({ nullable: true, type: 'string' }) prefix: string | null @@ -33,7 +33,7 @@ export class Guild extends CustomBaseEntity { export class GuildRepository extends EntityRepository { async updateLastInteract(guildId?: string): Promise { - const guild = await this.findOne({ id: guildId }) + const guild = await this.findOne({ guildId }) if (guild) { guild.lastInteract = new Date() diff --git a/src/entities/Image.ts b/src/entities/Image.ts index 77bbdb8d..16ee672d 100644 --- a/src/entities/Image.ts +++ b/src/entities/Image.ts @@ -12,9 +12,6 @@ export class Image extends CustomBaseEntity { [EntityRepositoryType]?: ImageRepository - @PrimaryKey() - id: number - @Property() fileName: string diff --git a/src/entities/Pastebin.ts b/src/entities/Pastebin.ts index 9aa0b878..fb5a5212 100644 --- a/src/entities/Pastebin.ts +++ b/src/entities/Pastebin.ts @@ -1,4 +1,5 @@ -import { Entity, EntityRepositoryType, PrimaryKey, Property } from '@mikro-orm/core' +import { Entity, EntityRepositoryType, PrimaryKey, Property, SerializedPrimaryKey } from '@mikro-orm/core' +import { ObjectId } from '@mikro-orm/mongodb'; import { EntityRepository } from '@mikro-orm/sqlite' // =========================================== @@ -10,8 +11,11 @@ export class Pastebin { [EntityRepositoryType]?: PastebinRepository - @PrimaryKey({ autoincrement: false }) - id: string + @PrimaryKey() + _id: ObjectId; + + @SerializedPrimaryKey() + id!: string; // won't be saved in the database @Property() editCode: string diff --git a/src/entities/Stat.ts b/src/entities/Stat.ts index 5006502b..a4138fa2 100644 --- a/src/entities/Stat.ts +++ b/src/entities/Stat.ts @@ -1,4 +1,5 @@ -import { Entity, EntityRepositoryType, PrimaryKey, Property } from '@mikro-orm/core' +import { Entity, EntityRepositoryType, PrimaryKey, Property, SerializedPrimaryKey } from '@mikro-orm/core' +import { ObjectId } from '@mikro-orm/mongodb'; import { EntityRepository } from '@mikro-orm/sqlite' // =========================================== @@ -11,7 +12,10 @@ export class Stat { [EntityRepositoryType]?: StatRepository @PrimaryKey() - id: number + _id: ObjectId; + + @SerializedPrimaryKey() + id!: string; // won't be saved in the database @Property() type!: string diff --git a/src/entities/User.ts b/src/entities/User.ts index 5e3284e7..208ecd90 100644 --- a/src/entities/User.ts +++ b/src/entities/User.ts @@ -1,4 +1,4 @@ -import { Entity, EntityRepositoryType, PrimaryKey, Property } from '@mikro-orm/core' +import { Entity, EntityRepositoryType, Property } from '@mikro-orm/core' import { EntityRepository } from '@mikro-orm/sqlite' import { CustomBaseEntity } from './BaseEntity' @@ -12,8 +12,8 @@ export class User extends CustomBaseEntity { [EntityRepositoryType]?: UserRepository - @PrimaryKey({ autoincrement: false }) - id!: string + @Property() + userId!: string @Property() lastInteract: Date = new Date() @@ -27,7 +27,7 @@ export class User extends CustomBaseEntity { export class UserRepository extends EntityRepository { async updateLastInteract(userId?: string): Promise { - const user = await this.findOne({ id: userId }) + const user = await this.findOne({ userId }) if (user) { user.lastInteract = new Date() diff --git a/src/services/Database.ts b/src/services/Database.ts index 4de2446c..a80f7239 100644 --- a/src/services/Database.ts +++ b/src/services/Database.ts @@ -73,80 +73,86 @@ export class Database { /** * Create a snapshot of the database each day at 00:00 */ - @Schedule('0 0 * * *') - async backup(snapshotName?: string): Promise { - const { formatDate } = await import('@/utils/functions') - if (!databaseConfig.backup.enabled && !snapshotName) - return false - if (!this.isSQLiteDatabase()) { - this.logger.log('Database is not SQLite, couldn\'t backup') + // Commented due to Type Issues from MicroORM - return false - } + // @Schedule('0 0 * * *') + // async backup(snapshotName?: string): Promise { + // const { formatDate } = await import('@/utils/functions') - const backupPath = databaseConfig.backup.path - if (!backupPath) { - this.logger.log('Backup path not set, couldn\'t backup', 'error', true) + // if (!databaseConfig.backup.enabled && !snapshotName) + // return false + // if (!this.isSQLiteDatabase()) { + // this.logger.log('Database is not SQLite, couldn\'t backup') - return false - } + // return false + // } - if (!snapshotName) - snapshotName = `snapshot-${formatDate(new Date(), 'onlyDateFileName')}` - const objectsPath = `${backupPath}objects/` as `${string}/` + // const backupPath = databaseConfig.backup.path + // if (!backupPath) { + // this.logger.log('Backup path not set, couldn\'t backup', 'error', true) - try { - await backup( - mikroORMConfig[env.NODE_ENV]!.dbName!, - `${snapshotName}.txt`, - objectsPath - ) + // return false + // } - return true - } catch (e) { - const errorMessage = typeof e === 'string' ? e : e instanceof Error ? e.message : 'Unknown error' + // if (!snapshotName) + // snapshotName = `snapshot-${formatDate(new Date(), 'onlyDateFileName')}` + // const objectsPath = `${backupPath}objects/` as `${string}/` - this.logger.log(`Couldn't backup : ${errorMessage}`, 'error', true) + // try { + // await backup( + // mikroORMConfig[env.NODE_ENV]!.dbName!, + // `${snapshotName}.txt`, + // objectsPath + // ) - return false - } - } + // return true + // } catch (e) { + // const errorMessage = typeof e === 'string' ? e : e instanceof Error ? e.message : 'Unknown error' + + // this.logger.log(`Couldn't backup : ${errorMessage}`, 'error', true) + + // return false + // } + // } /** * Restore the SQLite database from a snapshot file. * @param snapshotName name of the snapshot to restore * @returns true if the snapshot has been restored, false otherwise */ - async restore(snapshotName: string): Promise { - if (!this.isSQLiteDatabase()) { - this.logger.log('Database is not SQLite, couldn\'t restore', 'error') - return false - } + // Commented due to Type Issues from MicroORM - const backupPath = databaseConfig.backup.path - if (!backupPath) - this.logger.log('Backup path not set, couldn\'t restore', 'error', true) + // async restore(snapshotName: string): Promise { + // if (!this.isSQLiteDatabase()) { + // this.logger.log('Database is not SQLite, couldn\'t restore', 'error') - try { - console.debug(mikroORMConfig[env.NODE_ENV]!.dbName!) - console.debug(`${backupPath}${snapshotName}`) - await restore( - mikroORMConfig[env.NODE_ENV]!.dbName!, - `${backupPath}${snapshotName}` - ) + // return false + // } - await this.refreshConnection() + // const backupPath = databaseConfig.backup.path + // if (!backupPath) + // this.logger.log('Backup path not set, couldn\'t restore', 'error', true) - return true - } catch (error) { - console.debug(error) - this.logger.log('Snapshot file not found, couldn\'t restore', 'error', true) + // try { + // console.debug(mikroORMConfig[env.NODE_ENV]!.dbName!) + // console.debug(`${backupPath}${snapshotName}`) + // await restore( + // mikroORMConfig[env.NODE_ENV]!.dbName!, + // `${backupPath}${snapshotName}` + // ) - return false - } - } + // await this.refreshConnection() + + // return true + // } catch (error) { + // console.debug(error) + // this.logger.log('Snapshot file not found, couldn\'t restore', 'error', true) + + // return false + // } + // } getBackupList(): string[] | null { const backupPath = databaseConfig.backup.path @@ -162,28 +168,30 @@ export class Database { return backupList } - getSize(): DatabaseSize { - const size: DatabaseSize = { - db: null, - backups: null, - } + // Commented due to Type Issues from MicroORM - if (this.isSQLiteDatabase()) { - const dbPath = mikroORMConfig[env.NODE_ENV]!.dbName! - const dbSize = fs.statSync(dbPath).size + // getSize(): DatabaseSize { + // const size: DatabaseSize = { + // db: null, + // backups: null, + // } - size.db = dbSize - } + // if (this.isSQLiteDatabase()) { + // const dbPath = mikroORMConfig[env.NODE_ENV]!.dbName! + // const dbSize = fs.statSync(dbPath).size - const backupPath = databaseConfig.backup.path - if (backupPath) { - const backupSize = fastFolderSizeSync(backupPath) + // size.db = dbSize + // } - size.backups = backupSize || null - } + // const backupPath = databaseConfig.backup.path + // if (backupPath) { + // const backupSize = fastFolderSizeSync(backupPath) - return size - } + // size.backups = backupSize || null + // } + + // return size + // } isSQLiteDatabase(): boolean { const type = mikroORMConfig[env.NODE_ENV]!.type diff --git a/src/services/Stats.ts b/src/services/Stats.ts index c403343e..3bcc6320 100644 --- a/src/services/Stats.ts +++ b/src/services/Stats.ts @@ -129,44 +129,29 @@ export class Stats { * Get commands sorted by total amount of uses in DESC order. */ async getTopCommands() { - if ('createQueryBuilder' in this.db.em) { - const qb = this.db.em.createQueryBuilder(Stat) - const query = qb - .select(['type', 'value as name', 'count(*) as count']) - .where(allInteractions) - .groupBy(['type', 'value']) - - const slashCommands = await query.execute() - - return slashCommands.sort((a: any, b: any) => b.count - a.count) - } else if ('aggregate' in this.db.em) { - // @ts-expect-error - aggregate is not in the types - const slashCommands = await this.db.em.aggregate(Stat, [ - { - $match: allInteractions, - }, - { - $group: { - _id: { type: '$type', value: '$value' }, - count: { $sum: 1 }, - }, + const slashCommands = await this.db.em.aggregate(Stat, [ + { + $match: allInteractions, + }, + { + $group: { + _id: { type: '$type', value: '$value' }, + count: { $sum: 1 }, }, - { - $replaceRoot: { - newRoot: { - $mergeObjects: [ - '$_id', - { count: '$count' }, - ], - }, + }, + { + $replaceRoot: { + newRoot: { + $mergeObjects: [ + '$_id', + { count: '$count' }, + ], }, }, - ]) + }, + ]) - return slashCommands.sort((a: any, b: any) => b.count - a.count) - } else { - return [] - } + return slashCommands.sort((a: any, b: any) => b.count - a.count) } /** diff --git a/src/utils/functions/synchronizer.ts b/src/utils/functions/synchronizer.ts index 387472f9..5dd3e145 100644 --- a/src/utils/functions/synchronizer.ts +++ b/src/utils/functions/synchronizer.ts @@ -15,13 +15,13 @@ export async function syncUser(user: DUser) { const userRepo = db.get(User) const userData = await userRepo.findOne({ - id: user.id, + userId: user.id, }) if (!userData) { // add user to the db const newUser = new User() - newUser.id = user.id + newUser.userId = user.id await userRepo.persistAndFlush(newUser) // record new user both in logs and stats @@ -39,13 +39,13 @@ export async function syncGuild(guildId: string, client: Client) { const [db, stats, logger] = await resolveDependencies([Database, Stats, Logger]) const guildRepo = db.get(Guild) - const guildData = await guildRepo.findOne({ id: guildId, deleted: false }) + const guildData = await guildRepo.findOne({ guildId, deleted: false }) const fetchedGuild = await client.guilds.fetch(guildId).catch(() => null) // check if this guild exists in the database, if not it creates it (or recovers it from the deleted ones) if (!guildData) { - const deletedGuildData = await guildRepo.findOne({ id: guildId, deleted: true }) + const deletedGuildData = await guildRepo.findOne({ guildId, deleted: true }) if (deletedGuildData) { // recover deleted guild @@ -59,7 +59,7 @@ export async function syncGuild(guildId: string, client: Client) { // create new guild const newGuild = new Guild() - newGuild.id = guildId + newGuild.guildId = guildId await guildRepo.persistAndFlush(newGuild) stats.register('NEW_GUILD', guildId) @@ -92,5 +92,5 @@ export async function syncAllGuilds(client: Client) { const guildRepo = db.get(Guild) const guildsData = await guildRepo.getActiveGuilds() for (const guildData of guildsData) - await syncGuild(guildData.id, client) + await syncGuild(guildData.guildId, client) } diff --git a/src/utils/types/environment.ts b/src/utils/types/environment.ts index 11eb0f84..690f5648 100644 --- a/src/utils/types/environment.ts +++ b/src/utils/types/environment.ts @@ -24,7 +24,7 @@ export const env = cleanEnv(process.env, { }) export function checkEnvironmentVariables() { - if (!['sqlite', 'better-sqlite'].includes(mikroORMConfig[env.NODE_ENV].type)) { + if (!['sqlite', 'better-sqlite', 'mongo'].includes(mikroORMConfig[env.NODE_ENV].type)) { cleanEnv(process.env, { DATABASE_HOST: str(), DATABASE_PORT: num(), From 546a2e089a7da3dfa7bcdda584c00e9eb2f949be Mon Sep 17 00:00:00 2001 From: DJKnaeckebrot Date: Wed, 3 Apr 2024 14:13:40 +0200 Subject: [PATCH 2/2] fix pushed migration --- database/migrations/Migration20240403115346.js | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 database/migrations/Migration20240403115346.js diff --git a/database/migrations/Migration20240403115346.js b/database/migrations/Migration20240403115346.js deleted file mode 100644 index f4627334..00000000 --- a/database/migrations/Migration20240403115346.js +++ /dev/null @@ -1,11 +0,0 @@ -'use strict'; -Object.defineProperty(exports, '__esModule', { value: true }); -const { Migration } = require('@mikro-orm/migrations-mongodb'); - -class Migration20240403115346 extends Migration { - - async up() { - } - -} -exports.Migration20240403115346 = Migration20240403115346;