Add batching.

hpi-schul-cloud · Dec 16, 2024 · c16a7bf · c16a7bf
1 parent a04d22d
commit c16a7bf
Showing 1 changed file with 43 additions and 26 deletions.
diff --git a/apps/server/src/infra/sync/tsp/tsp-sync.strategy.ts b/apps/server/src/infra/sync/tsp/tsp-sync.strategy.ts
@@ -126,41 +126,58 @@ export class TspSyncStrategy extends SyncStrategy {
 	private async migrateTspTeachersBatch(system: System, oldToNewMappings: Map<string, string>): Promise<number> {
 		this.logger.info(new TspTeachersFetchedLoggable(oldToNewMappings.size));
 
-		const migrationIds = Array.from(oldToNewMappings.keys());
+		const oldIds = Array.from(oldToNewMappings.keys());
+		const batchSize = this.configService.get<number>('TSP_SYNC_MIGRATION_LIMIT', 100);
+
+		const batchCount = Math.ceil(oldIds.length / batchSize);
+		const batches: string[][] = [];
+		for (let i = 0; i < batchCount; i += 1) {
+			const start = i * batchSize;
+			const end = Math.min((i + 1) * batchSize, oldIds.length);
+			batches.push(oldIds.slice(start, end));
+		}
 
-		const users = await this.userService.findByTspUids(migrationIds);
-		this.logger.info(this.logForMsg(`Users fetched: ${users.length}`));
+		let total = 0;
+		for await (const oldIdsBatch of batches) {
+			this.logger.info(this.logForMsg('Start of new batch'));
 
-		const userIds = users.map((user) => user.id ?? '');
-		const accounts = await this.accountService.findMultipleByUserId(userIds);
-		this.logger.info(this.logForMsg(`Accounts loaded: ${accounts.length}`));
+			const users = await this.userService.findByTspUids(oldIdsBatch);
+			this.logger.info(this.logForMsg(`Users fetched: ${users.length}`));
 
-		const accountsForUserid = new Map<string, Account>();
-		accounts.forEach((account) => accountsForUserid.set(account.userId ?? '', account));
+			const userIds = users.map((user) => user.id ?? '');
+			const accounts = await this.accountService.findMultipleByUserId(userIds);
+			this.logger.info(this.logForMsg(`Accounts loaded: ${accounts.length}`));
 
-		users.forEach((user) => {
-			const newUid = oldToNewMappings.get(user.sourceOptions?.tspUid ?? '') ?? '';
-			const newEmailAndUsername = `${newUid}@schul-cloud.org`;
+			const accountsForUserid = new Map<string, Account>();
+			accounts.forEach((account) => accountsForUserid.set(account.userId ?? '', account));
 
-			user.email = newEmailAndUsername;
-			user.externalId = newUid;
-			user.previousExternalId = user.sourceOptions?.tspUid;
-			user.sourceOptions = new UserSourceOptions({ tspUid: newUid });
+			users.forEach((user) => {
+				const newUid = oldToNewMappings.get(user.sourceOptions?.tspUid ?? '') ?? '';
+				const newEmailAndUsername = `${newUid}@schul-cloud.org`;
 
-			const account = accountsForUserid.get(user.id ?? '');
-			if (account) {
-				account.username = newEmailAndUsername;
-				account.systemId = system.id;
-			}
-		});
+				user.email = newEmailAndUsername;
+				user.externalId = newUid;
+				user.previousExternalId = user.sourceOptions?.tspUid;
+				user.sourceOptions = new UserSourceOptions({ tspUid: newUid });
 
-		await this.userService.saveAll(users);
-		this.logger.info(this.logForMsg('Users saved'));
+				const account = accountsForUserid.get(user.id ?? '');
+				if (account) {
+					account.username = newEmailAndUsername;
+					account.systemId = system.id;
+				}
+			});
 
-		await this.accountService.saveAll(accounts);
-		this.logger.info(this.logForMsg('Accounts saved'));
+			await this.userService.saveAll(users);
+			this.logger.info(this.logForMsg('Users saved'));
+
+			await this.accountService.saveAll(accounts);
+			this.logger.info(this.logForMsg('Accounts saved'));
+
+			total += users.length;
+			this.logger.info(this.logForMsg(`Users so far ${total}`));
+		}
 
-		return users.length;
+		return total;
 	}
 
 	private async runMigration(system: System): Promise<void> {