Data management improvements

Signed-off-by: Hoang Pham <[email protected]>

.gitignore

@@ -7,6 +7,7 @@
 /css/
 /vendor/
 /node_modules/
+/backup/
 
 .php-cs-fixer.cache
 .phpunit.result.cache

websocket_server/ApiService.js

@@ -59,7 +59,13 @@ export default class ApiService {
 		console.log(`[${roomID}] Saving room data to server: ${roomData.length} elements, ${Object.keys(files).length} files`)
 
 		const url = `${this.NEXTCLOUD_URL}/index.php/apps/whiteboard/${roomID}`
-		const body = { data: { elements: roomData, files: this.cleanupFiles(roomData, files) } }
+		const body = {
+			data: {
+				elements: roomData,
+				files: this.cleanupFiles(roomData, files),
+				savedAt: Date.now(),
+			},
+		}
 		const options = this.fetchOptions('PUT', null, body, roomID, lastEditedUser)
 		return this.fetchData(url, options)
 	}

websocket_server/BackupManager.js

@@ -0,0 +1,233 @@
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+/* eslint-disable no-console */
+
+import fs from 'fs/promises'
+import path from 'path'
+import crypto from 'crypto'
+import zlib from 'zlib'
+import { promisify } from 'util'
+
+const gzip = promisify(zlib.gzip)
+const gunzip = promisify(zlib.gunzip)
+
+export default class BackupManager {
+
+	constructor(options = {}) {
+		const { backupDir = './backup', maxBackupsPerRoom = 5 } = options
+		this.backupDir = backupDir
+		this.maxBackupsPerRoom = maxBackupsPerRoom
+		this.locks = new Map()
+		this.lockTimeout = options.lockTimeout || 5000 // 5 seconds
+		this.lockRetryInterval = options.lockRetryInterval || 50 // 50ms
+		this.init()
+	}
+
+	async init() {
+		try {
+			await fs.mkdir(this.backupDir, { recursive: true })
+			await this.cleanupTemporaryFiles()
+		} catch (error) {
+			console.error('Failed to initialize BackupManager:', error)
+			throw error
+		}
+	}
+
+	async cleanupTemporaryFiles() {
+		try {
+			const files = await fs.readdir(this.backupDir)
+			const tmpFiles = files.filter((f) => f.endsWith('.tmp'))
+			await Promise.all(
+				tmpFiles.map((file) =>
+					fs
+						.unlink(path.join(this.backupDir, file))
+						.catch(console.error),
+				),
+			)
+		} catch (error) {
+			console.error('Failed to cleanup temporary files:', error)
+		}
+	}
+
+	async acquireLock(roomId) {
+		const startTime = Date.now()
+		while (this.locks.get(roomId)) {
+			if (Date.now() - startTime > this.lockTimeout) {
+				throw new Error(`Lock acquisition timeout for room ${roomId}`)
+			}
+			await new Promise((resolve) =>
+				setTimeout(resolve, this.lockRetryInterval),
+			)
+		}
+		this.locks.set(roomId, Date.now())
+	}
+
+	async releaseLock(roomId) {
+		this.locks.delete(roomId)
+	}
+
+	sanitizeRoomId(roomId) {
+		return roomId.replace(/[^a-zA-Z0-9-_]/g, '_')
+	}
+
+	calculateChecksum(data) {
+		return crypto
+			.createHash('sha256')
+			.update(typeof data === 'string' ? data : JSON.stringify(data))
+			.digest('hex')
+	}
+
+	async createBackup(roomId, data) {
+		if (!roomId || !data) {
+			throw new Error('Invalid backup parameters')
+		}
+
+		const sanitizedRoomId = this.sanitizeRoomId(roomId)
+
+		try {
+			await this.acquireLock(sanitizedRoomId)
+
+			const backupData = this.prepareBackupData(sanitizedRoomId, data)
+			await this.writeBackupFile(sanitizedRoomId, backupData)
+			await this.cleanupOldBackups(sanitizedRoomId)
+
+			return backupData.id
+		} finally {
+			await this.releaseLock(sanitizedRoomId)
+		}
+	}
+
+	prepareBackupData(roomId, data) {
+		return {
+			id: crypto.randomUUID(),
+			timestamp: Date.now(),
+			roomId,
+			checksum: this.calculateChecksum(data),
+			data,
+			savedAt: data.savedAt || Date.now(),
+		}
+	}
+
+	async writeBackupFile(roomId, backupData) {
+		const backupFile = path.join(
+			this.backupDir,
+			`${roomId}_${backupData.timestamp}.bak`,
+		)
+		const tempFile = `${backupFile}.tmp`
+
+		const compressed = await gzip(JSON.stringify(backupData))
+		await fs.writeFile(tempFile, compressed)
+		await fs.rename(tempFile, backupFile)
+	}
+
+	async getLatestBackup(roomId) {
+		const sanitizedRoomId = this.sanitizeRoomId(roomId)
+		const files = await fs.readdir(this.backupDir)
+		const roomBackups = files
+			.filter(
+				(f) =>
+					f.startsWith(`${sanitizedRoomId}_`) && f.endsWith('.bak'),
+			)
+			.sort()
+			.reverse()
+
+		if (roomBackups.length === 0) return null
+
+		try {
+			const compressed = await fs.readFile(
+				path.join(this.backupDir, roomBackups[0]),
+			)
+			const decompressed = await gunzip(compressed)
+			const backup = JSON.parse(decompressed.toString())
+
+			// Verify checksum
+			const calculatedChecksum = this.calculateChecksum(backup.data)
+			if (calculatedChecksum !== backup.checksum) {
+				throw new Error('Backup data corruption detected')
+			}
+
+			return backup
+		} catch (error) {
+			console.error(
+				`Failed to read latest backup for room ${sanitizedRoomId}:`,
+				error,
+			)
+			throw error
+		}
+	}
+
+	async cleanupOldBackups(roomId) {
+		const sanitizedRoomId = this.sanitizeRoomId(roomId)
+
+		try {
+			const files = await fs.readdir(this.backupDir)
+			const roomBackups = files
+				.filter(
+					(f) =>
+						f.startsWith(`${sanitizedRoomId}_`)
+						&& f.endsWith('.bak'),
+				)
+				.sort()
+				.reverse()
+
+			if (roomBackups.length <= this.maxBackupsPerRoom) {
+				return
+			}
+
+			const filesToDelete = roomBackups.slice(this.maxBackupsPerRoom)
+			await Promise.all(
+				filesToDelete.map((file) =>
+					fs
+						.unlink(path.join(this.backupDir, file))
+						.catch((error) => {
+							console.error(
+								`Failed to delete backup ${file}:`,
+								error,
+							)
+							// Don't throw to continue with other deletions
+						}),
+				),
+			)
+		} catch (error) {
+			console.error(`Failed to cleanup old backups for ${roomId}:`, error)
+			// Don't throw as this is a non-critical operation
+		}
+	}
+
+	async getAllBackups(roomId) {
+		const sanitizedRoomId = this.sanitizeRoomId(roomId)
+		const files = await fs.readdir(this.backupDir)
+		return files
+			.filter(
+				(f) =>
+					f.startsWith(`${sanitizedRoomId}_`) && f.endsWith('.bak'),
+			)
+			.sort()
+			.reverse()
+	}
+
+	async recoverFromBackup(roomId) {
+		const backup = await this.getLatestBackup(roomId)
+		if (!backup) {
+			console.log(`No backup found for room ${roomId}`)
+			return null
+		}
+		return backup.data
+	}
+
+	async isDataFresher(roomId, serverData) {
+		const latestBackup = await this.getLatestBackup(roomId)
+
+		if (!latestBackup) return true
+
+		// Simply compare savedAt timestamps
+		const serverTimestamp = serverData?.savedAt || 0
+		const backupTimestamp = latestBackup.savedAt || 0
+
+		return serverTimestamp >= backupTimestamp
+	}
+
+}

websocket_server/LRUCacheStrategy.js

@@ -20,6 +20,7 @@ export default class LRUCacheStrategy extends StorageStrategy {
 			ttlAutopurge: true,
 			dispose: async (value, key) => {
 				console.log(`[${key}] Disposing room`)
+
 				if (value?.data && value?.lastEditedUser) {
 					try {
 						await this.apiService.saveRoomDataToServer(
@@ -53,7 +54,9 @@ export default class LRUCacheStrategy extends StorageStrategy {
 	}
 
 	getRooms() {
-		const rooms = Array.from(this.cache.values()).filter((room) => room instanceof Room)
+		const rooms = Array.from(this.cache.values()).filter(
+			(room) => room instanceof Room,
+		)
 
 		return rooms
 	}