Skip to content

Commit

Permalink
Bulk migration for students.
Browse files Browse the repository at this point in the history
  • Loading branch information
@mkreuzkam-cap
mkreuzkam-cap committed Dec 13, 2024
1 parent 18217e6 commit 643c843
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 158 deletions.
48 changes: 2 additions & 46 deletions apps/server/src/infra/sync/tsp/tsp-sync.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,13 @@ import { FederalStateService, SchoolYearService } from '@modules/legacy-school';
import { School, SchoolService } from '@modules/school';
import { System, SystemService, SystemType } from '@modules/system';
import { Injectable } from '@nestjs/common';
import { UserDO } from '@shared/domain/domainobject';
import { UserSourceOptions } from '@shared/domain/domainobject/user-source-options.do';
import { SystemProvisioningStrategy } from '@shared/domain/interface/system-provisioning.strategy';
import { EntityId, SchoolFeature } from '@shared/domain/types';
import { Account, AccountService } from '@src/modules/account';
import { SchoolFeature } from '@shared/domain/types';
import { FederalStateNames } from '@src/modules/legacy-school/types';
import { FederalState, FileStorageType } from '@src/modules/school/domain';
import { SchoolFactory } from '@src/modules/school/domain/factory';
import { SchoolPermissions } from '@src/modules/school/domain/type';
import { FederalStateEntityMapper, SchoolYearEntityMapper } from '@src/modules/school/repo/mikro-orm/mapper';
import { UserService } from '@src/modules/user';
import { ObjectId } from 'bson';
import { TspSystemNotFoundLoggableException } from './loggable/tsp-system-not-found.loggable-exception';

Expand All @@ -24,9 +20,7 @@ export class TspSyncService {
private readonly systemService: SystemService,
private readonly schoolService: SchoolService,
private readonly federalStateService: FederalStateService,
private readonly schoolYearService: SchoolYearService,
private readonly userService: UserService,
private readonly accountService: AccountService
private readonly schoolYearService: SchoolYearService
) {}

public async findTspSystemOrFail(): Promise<System> {
Expand Down Expand Up @@ -114,42 +108,4 @@ export class TspSyncService {
this.federalState = FederalStateEntityMapper.mapToDo(federalStateEntity);
return this.federalState;
}

public async findUserByTspUid(tspUid: string): Promise<UserDO | null> {
const tspUser = await this.userService.findUsers({ tspUid });

if (tspUser.data.length === 0) {
return null;
}

return tspUser.data[0];
}

public async findAccountByExternalId(externalId: string, systemId: EntityId): Promise<Account | null> {
const user = await this.userService.findByExternalId(externalId, systemId);

if (!user || !user.id) {
return null;
}

const account = await this.accountService.findByUserId(user.id);

return account;
}

public updateUser(user: UserDO, email: string, externalId: string, previousExternalId: string): Promise<UserDO> {
user.email = email;
user.externalId = externalId;
user.previousExternalId = previousExternalId;
user.sourceOptions = new UserSourceOptions({ tspUid: user.externalId });

return this.userService.save(user);
}

public updateAccount(account: Account, username: string, systemId: string): Promise<Account> {
account.username = username;
account.systemId = systemId;

return this.accountService.save(account);
}
}
131 changes: 19 additions & 112 deletions apps/server/src/infra/sync/tsp/tsp-sync.strategy.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import { School } from '@modules/school';
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { NotFoundLoggableException } from '@shared/common/loggable-exception';
import { UserDO } from '@shared/domain/domainobject';
import { UserSourceOptions } from '@shared/domain/domainobject/user-source-options.do';
import { Loggable, Logger } from '@src/core/logger';
import { Account, AccountService } from '@src/modules/account';
Expand All @@ -15,12 +13,9 @@ import { TspDataFetchedLoggable } from './loggable/tsp-data-fetched.loggable';
import { TspSchoolsFetchedLoggable } from './loggable/tsp-schools-fetched.loggable';
import { TspSchoolsSyncedLoggable } from './loggable/tsp-schools-synced.loggable';
import { TspSchulnummerMissingLoggable } from './loggable/tsp-schulnummer-missing.loggable';
import { TspStudentsFetchedLoggable } from './loggable/tsp-students-fetched.loggable';
import { TspStudentsMigratedLoggable } from './loggable/tsp-students-migrated.loggable';
import { TspSyncedUsersLoggable } from './loggable/tsp-synced-users.loggable';
import { TspSyncingUsersLoggable } from './loggable/tsp-syncing-users.loggable';
import { TspTeachersFetchedLoggable } from './loggable/tsp-teachers-fetched.loggable';
import { TspTeachersMigratedLoggable } from './loggable/tsp-teachers-migrated.loggable';
import { TspUsersMigratedLoggable } from './loggable/tsp-users-migrated.loggable';
import { TspFetchService } from './tsp-fetch.service';
import { TspLegacyMigrationService } from './tsp-legacy-migration.service';
Expand Down Expand Up @@ -80,10 +75,7 @@ export class TspSyncStrategy extends SyncStrategy {
const schools = await this.tspSyncService.findSchoolsForSystem(system);

if (this.migrationEnabled) {
const teacherMigrationResult = await this.migrateTspTeachersBatch(system);
const studentMigrationResult = await this.migrateTspStudents(system);
const totalMigrations = teacherMigrationResult.total + studentMigrationResult.total;
this.logger.info(new TspUsersMigratedLoggable(totalMigrations));
await this.runMigration(system);
}

await this.syncData(system, schools);
Expand Down Expand Up @@ -150,20 +142,10 @@ export class TspSyncStrategy extends SyncStrategy {
this.logger.info(new TspSyncedUsersLoggable(results.length));
}

private async migrateTspTeachersBatch(system: System): Promise<{ total: number }> {
this.logger.info(this.logForMsg('NEW NEW NEW!!!'));
const tspTeacherIds = await this.tspFetchService.fetchTspTeacherMigrations(system);
private async migrateTspTeachersBatch(system: System, oldToNewMappings: Map<string, string>): Promise<number> {
this.logger.info(new TspTeachersFetchedLoggable(oldToNewMappings.size));

const oldForNewIds = new Map<string, string>();
tspTeacherIds.forEach(({ lehrerUidAlt, lehrerUidNeu }) => {
if (lehrerUidAlt && lehrerUidNeu) {
oldForNewIds.set(lehrerUidAlt, lehrerUidNeu);
}
});

this.logger.info(new TspTeachersFetchedLoggable(tspTeacherIds.length));

const users = await this.userService.findByTspUids(tspTeacherIds.map(({ lehrerUidAlt }) => lehrerUidAlt ?? ''));
const users = await this.userService.findByTspUids(Array.from(oldToNewMappings.keys()));
this.logger.info(this.logForMsg(`Users fetched: ${users.length}`));

const userIds = users.map((user) => user.id ?? '');
Expand All @@ -172,10 +154,9 @@ export class TspSyncStrategy extends SyncStrategy {

const accountsForUserid = new Map<string, Account>();
accounts.forEach((account) => accountsForUserid.set(account.userId ?? '', account));
this.logger.info(this.logForMsg(`Accounts mapped: ${accountsForUserid.size}`));

users.forEach((user) => {
const newUid = oldForNewIds.get(user.sourceOptions?.tspUid ?? '') ?? '';
const newUid = oldToNewMappings.get(user.sourceOptions?.tspUid ?? '') ?? '';
const newEmailAndUsername = `${newUid}@schul-cloud.org`;

user.email = newEmailAndUsername;
Expand All @@ -189,108 +170,34 @@ export class TspSyncStrategy extends SyncStrategy {
account.systemId = system.id;
}
});
this.logger.info(this.logForMsg('Updated users & accounts'));

await this.userService.saveAll(users);
this.logger.info(this.logForMsg('Users saved'));

await this.accountService.saveAll(accounts);
this.logger.info(this.logForMsg('Accounts saved'));

return { total: users.length };
return users.length;
}

private async migrateTspTeachers(system: System): Promise<{ total: number }> {
const tspTeacherIds = await this.tspFetchService.fetchTspTeacherMigrations(system);
this.logger.info(new TspTeachersFetchedLoggable(tspTeacherIds.length));

const batches = Math.ceil(tspTeacherIds.length / this.migrationLimit);

let total = 0;
for await (const batch of Array.from(Array(batches).keys())) {
const currentBatch = tspTeacherIds.slice(batch * this.migrationLimit, (batch + 1) * this.migrationLimit);
const teacherMigrationPromises = currentBatch.map(async ({ lehrerUidAlt, lehrerUidNeu }) => {
if (lehrerUidAlt && lehrerUidNeu) {
await this.migrateTspUser(lehrerUidAlt, lehrerUidNeu, system.id);
return true;
}
return false;
});
const migratedTspTeachers = await Promise.allSettled(teacherMigrationPromises);
const batchSuccess = migratedTspTeachers.filter(
(result) => result.status === 'fulfilled' && result.value === true
).length;
const batchFulfilled = migratedTspTeachers.filter((result) => result.status === 'fulfilled').length;
const batchTotal = migratedTspTeachers.length;
const batchSize = currentBatch.length;

total += batchSuccess;

const msg = `Batch ${batch} of ${batches} done: This batch: Successful:${batchSuccess}, Fulfilled: ${batchFulfilled}, BatchTotal: ${batchTotal}, BatchSize: ${batchSize} , Total: ${total}`;
this.logger.info(this.logForMsg(msg));
}

this.logger.info(new TspTeachersMigratedLoggable(total));

return { total };
}

private async migrateTspStudents(system: System): Promise<{ total: number }> {
const tspStudentIds = await this.tspFetchService.fetchTspStudentMigrations(system);
this.logger.info(new TspStudentsFetchedLoggable(tspStudentIds.length));

const studentMigrationPromises = tspStudentIds.map(async ({ schuelerUidAlt, schuelerUidNeu }) => {
if (schuelerUidAlt && schuelerUidNeu) {
await this.migrateTspUser(schuelerUidAlt, schuelerUidNeu, system.id);
return true;
private async runMigration(system: System): Promise<void> {
const oldToNewMappings = new Map<string, string>();
const teacherMigrations = await this.tspFetchService.fetchTspTeacherMigrations(system);
teacherMigrations.forEach(({ lehrerUidAlt, lehrerUidNeu }) => {
if (lehrerUidAlt && lehrerUidNeu) {
oldToNewMappings.set(lehrerUidAlt, lehrerUidNeu);
}
return false;
});

const migratedStudents = await Promise.allSettled(studentMigrationPromises);

const total = migratedStudents.filter((result) => result.status === 'fulfilled' && result.value === true).length;
this.logger.info(new TspStudentsMigratedLoggable(total));

return { total };
}

private async migrateTspUser(
oldUid: string,
newUid: string,
systemId: string
): Promise<{ updatedUser: UserDO; updatedAccount: Account } | null> {
try {
const newEmailAndUsername = `${newUid}@schul-cloud.org`;
const user = await this.tspSyncService.findUserByTspUid(oldUid);

// console.log(`User found: ${user?.id ?? 'No Id'}`);
if (!user) {
throw new NotFoundLoggableException(UserDO.name, { oldUid });
}

const newEmail = newEmailAndUsername;
const updatedUser = await this.tspSyncService.updateUser(user, newEmail, newUid, oldUid);
// console.log(`User updated: ${user?.id ?? 'No Id'}`);

const account = await this.accountService.findByUserId(updatedUser.id ?? '');

// console.log(`Account found: ${account?.id ?? 'No Id'}`);

if (!account) {
throw new NotFoundLoggableException(Account.name, { oldUid });
const studentsMigrations = await this.tspFetchService.fetchTspStudentMigrations(system);
studentsMigrations.forEach(({ schuelerUidAlt, schuelerUidNeu }) => {
if (schuelerUidAlt && schuelerUidNeu) {
oldToNewMappings.set(schuelerUidAlt, schuelerUidNeu);
}
});

const newUsername = newEmailAndUsername;
const updatedAccount = await this.tspSyncService.updateAccount(account, newUsername, systemId);

// console.log(`Account updated: ${account?.id ?? 'No Id'}`);

return { updatedUser, updatedAccount };
} catch (e) {
console.log(`An error occurred: ${JSON.stringify(e)}`);
}
return null;
const migrationResult = await this.migrateTspTeachersBatch(system, oldToNewMappings);
this.logger.info(new TspUsersMigratedLoggable(migrationResult));
}

private logForMsg(msg: string): Loggable {
Expand Down

0 comments on commit 643c843

Please sign in to comment.