From 81ce327079567b1f4932fb8227fe65e122e2c68e Mon Sep 17 00:00:00 2001 From: Andrew Horth Date: Tue, 10 Dec 2024 15:38:26 +0000 Subject: [PATCH] Sync induction changes in real time from dqt (#1744) --- .../Commands.SyncPerson.cs | 2 + .../DataStore/Postgres/Models/Person.cs | 2 +- .../Services/TrsDataSync/TrsDataSyncHelper.cs | 97 +++++++-------- .../TrsDataSync/TrsDataSyncService.cs | 1 + .../TrsDataSyncServiceTests.Induction.cs | 115 ++++++++++++++++++ .../TrsDataSync/TrsDataSyncServiceTests.cs | 7 +- 6 files changed, 167 insertions(+), 57 deletions(-) create mode 100644 TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceTests.Induction.cs diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.SyncPerson.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.SyncPerson.cs index 6c9060729..3130583a2 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.SyncPerson.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Cli/Commands.SyncPerson.cs @@ -52,6 +52,8 @@ public static Command CreateSyncPersonCommand(IConfiguration configuration) } await syncHelper.SyncPersonAsync(contact, ignoreInvalid: false, dryRun: false); + + await syncHelper.SyncInductionsAsync([contact], ignoreInvalid: false, createMigratedEvent: false, dryRun: false); //return 0; }, connectionStringOption, diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Models/Person.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Models/Person.cs index bfabf04a8..4baa4eb7d 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Models/Person.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/DataStore/Postgres/Models/Person.cs @@ -17,7 +17,7 @@ public class Person public required DateOnly? DateOfBirth { get; set; } // A few DQT records in prod have a null DOB public string? EmailAddress { get; set; } public string? NationalInsuranceNumber { get; set; } - public InductionStatus InductionStatus { get; private set; } + public InductionStatus InductionStatus { get; set; } public InductionExemptionReasons InductionExemptionReasons { get; private set; } public DateOnly? InductionStartDate { get; private set; } public DateOnly? InductionCompletedDate { get; private set; } diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs index 7a5f5acf5..a4d266b57 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs @@ -319,7 +319,7 @@ public async Task SyncInductionsAsync( bool ignoreInvalid, bool createMigratedEvent, bool dryRun, - CancellationToken cancellationToken) + CancellationToken cancellationToken = default) { var inductionAttributeNames = new[] { @@ -328,6 +328,8 @@ public async Task SyncInductionsAsync( dfeta_induction.Fields.dfeta_InductionExemptionReason, dfeta_induction.Fields.dfeta_StartDate, dfeta_induction.Fields.dfeta_InductionStatus, + dfeta_induction.Fields.CreatedOn, + dfeta_induction.Fields.CreatedBy, dfeta_induction.Fields.ModifiedOn }; @@ -774,7 +776,7 @@ private async Task> GetAuditRec IEnumerable ids, CancellationToken cancellationToken) { - return (await Task.WhenAll(ids + return (IsFakeXrm ? new Dictionary() : (await Task.WhenAll(ids .Chunk(MaxAuditRequestsPerBatch) .Select(async chunk => { @@ -788,6 +790,7 @@ private async Task> GetAuditRec } }; + // The following is not supported by FakeXrmEasy hence the check above to allow more test coverage request.Requests.AddRange(chunk.Select(e => new RetrieveRecordChangeHistoryRequest() { Target = e.ToEntityReference(entityLogicalName) })); ExecuteMultipleResponse response; @@ -829,7 +832,7 @@ private async Task> GetAuditRec (r, e) => (Id: e, ((RetrieveRecordChangeHistoryResponse)r.Response).AuditDetailCollection)); }))) .SelectMany(b => b) - .ToDictionary(t => t.Id, t => t.AuditDetailCollection); + .ToDictionary(t => t.Id, t => t.AuditDetailCollection)); } private async Task GetEntitiesAsync( @@ -876,7 +879,8 @@ private static ModelTypeSyncInfo GetModelTypeSyncInfoForPerson() "dqt_modified_on", "dqt_first_name", "dqt_middle_name", - "dqt_last_name" + "dqt_last_name", + "induction_status", }; var columnsToUpdate = columnNames.Except(new[] { "person_id", "dqt_contact_id" }).ToArray(); @@ -905,6 +909,7 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on Contact.Fields.ContactId, Contact.Fields.StateCode, Contact.Fields.CreatedOn, + Contact.Fields.CreatedBy, Contact.Fields.ModifiedOn, Contact.Fields.dfeta_TRN, Contact.Fields.FirstName, @@ -915,7 +920,8 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on Contact.Fields.dfeta_StatedLastName, Contact.Fields.BirthDate, Contact.Fields.dfeta_NINumber, - Contact.Fields.EMailAddress1 + Contact.Fields.EMailAddress1, + Contact.Fields.dfeta_InductionStatus }; Action writeRecord = (writer, person) => @@ -937,6 +943,7 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on writer.WriteValueOrNull(person.DqtFirstName, NpgsqlDbType.Varchar); writer.WriteValueOrNull(person.DqtMiddleName, NpgsqlDbType.Varchar); writer.WriteValueOrNull(person.DqtLastName, NpgsqlDbType.Varchar); + writer.WriteValueOrNull((int?)person.InductionStatus, NpgsqlDbType.Integer); }; return new ModelTypeSyncInfo() @@ -1171,38 +1178,11 @@ private static List MapPersons(IEnumerable contacts) => contact DqtModifiedOn = c.ModifiedOn!.Value, DqtFirstName = c.FirstName ?? string.Empty, DqtMiddleName = c.MiddleName ?? string.Empty, - DqtLastName = c.LastName ?? string.Empty + DqtLastName = c.LastName ?? string.Empty, + InductionStatus = c.dfeta_InductionStatus.ToInductionStatus() }) .ToList(); - private static List MapInductions(IReadOnlyCollection contacts, IEnumerable inductions, bool ignoreInvalid) - { - var inductionLookup = inductions - .GroupBy(i => i.dfeta_PersonId.Id) - .ToDictionary(g => g.Key, g => g.ToArray()); - - return contacts - .Select(contact => - { - dfeta_induction? induction = null; - if (inductionLookup.TryGetValue(contact.ContactId!.Value, out var personInductions)) - { - // We shouldn't have multiple induction records for the same person in prod at all but we might in other environments - // so we'll just take the most recently modified one. - induction = personInductions.OrderByDescending(i => i.ModifiedOn).First(); - if (personInductions.Length > 1 && !ignoreInvalid) - { - throw new InvalidOperationException($"Contact '{contact.ContactId!.Value}' has multiple induction records."); - } - } - - return MapInductionInfoFromDqtInduction(induction, contact, ignoreInvalid); - }) - .Where(i => i is not null) - .Cast() - .ToList(); - } - private (List Inductions, List Events) MapInductionsAndAudits( IReadOnlyCollection contacts, IEnumerable inductionEntities, @@ -1247,39 +1227,46 @@ private static List MapInductions(IReadOnlyCollection co dfeta_induction.Fields.dfeta_InductionStatus, dfeta_induction.Fields.ModifiedOn }; - var inductionAudits = auditDetails[induction!.Id].AuditDetails; - var inductionVersions = GetEntityVersions(induction, inductionAudits, inductionAttributeNames); - events.Add(inductionAudits.Any(a => a.AuditRecord.ToEntity().Action == Audit_Action.Create) ? - MapCreatedEvent(inductionVersions.First()) : - MapImportedEvent(inductionVersions.First())); - - foreach (var (thisVersion, previousVersion) in inductionVersions.Skip(1).Zip(inductionVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion))) + if (auditDetails.TryGetValue(induction!.Id, out var inductionAudits)) { - var mappedEvent = MapUpdatedEvent(thisVersion, previousVersion); + var inductionAuditDetails = inductionAudits.AuditDetails; + var inductionVersions = GetEntityVersions(induction, inductionAuditDetails, inductionAttributeNames); - if (mappedEvent is not null) + events.Add(inductionAuditDetails.Any(a => a.AuditRecord.ToEntity().Action == Audit_Action.Create) ? + MapCreatedEvent(inductionVersions.First()) : + MapImportedEvent(inductionVersions.First())); + + foreach (var (thisVersion, previousVersion) in inductionVersions.Skip(1).Zip(inductionVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion))) { - events.Add(mappedEvent); + var mappedEvent = MapUpdatedEvent(thisVersion, previousVersion); + + if (mappedEvent is not null) + { + events.Add(mappedEvent); + } } - } - if (createMigratedEvent) - { - events.Add(MapMigratedEvent(inductionVersions.Last(), mapped)); + if (createMigratedEvent) + { + events.Add(MapMigratedEvent(inductionVersions.Last(), mapped)); + } } } - var contactAudits = auditDetails[contact.ContactId!.Value].AuditDetails; - var contactVersions = GetEntityVersions(contact, contactAudits, GetModelTypeSyncInfo(ModelTypes.Person).AttributeNames); - - foreach (var (thisVersion, previousVersion) in contactVersions.Skip(1).Zip(contactVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion))) + if (auditDetails.TryGetValue(contact.ContactId!.Value, out var contactAudits)) { - var mappedEvent = MapContactInductionStatusChangedEvent(thisVersion, previousVersion); + var contactAuditDetails = contactAudits.AuditDetails; + var contactVersions = GetEntityVersions(contact, contactAuditDetails, GetModelTypeSyncInfo(ModelTypes.Person).AttributeNames); - if (mappedEvent is not null) + foreach (var (thisVersion, previousVersion) in contactVersions.Skip(1).Zip(contactVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion))) { - events.Add(mappedEvent); + var mappedEvent = MapContactInductionStatusChangedEvent(thisVersion, previousVersion); + + if (mappedEvent is not null) + { + events.Add(mappedEvent); + } } } diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncService.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncService.cs index b2df8cac3..45b6bb596 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncService.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncService.cs @@ -61,6 +61,7 @@ internal async Task ProcessChangesAsync(CancellationToken cancellationToken) var modelTypesToSync = optionsAccessor.Value.ModelTypes; // Order is important here; the dependees should come before dependents + await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Induction); await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Person); await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Event); await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Alert); diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceTests.Induction.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceTests.Induction.cs new file mode 100644 index 000000000..ecfc4bc2d --- /dev/null +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceTests.Induction.cs @@ -0,0 +1,115 @@ +using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Messages; +using TeachingRecordSystem.Core.Dqt; +using TeachingRecordSystem.Core.Dqt.Models; +using TeachingRecordSystem.Core.Services.TrsDataSync; + +namespace TeachingRecordSystem.Core.Tests.Services.TrsDataSync; + +public partial class TrsDataSyncServiceTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Induction_NewRecord_WritesUpdatedPersonRecordToDatabase(bool personAlreadySynced) + { + // Arrange + var createPersonResult = await TestData.CreatePersonAsync(p => p.WithSyncOverride(personAlreadySynced)); + var contactId = createPersonResult.ContactId; + + var inductionStartDate = Clock.Today.AddYears(-1); + var inductionEndDate = Clock.Today.AddDays(-10); + var induction = new dfeta_induction() + { + Id = Guid.NewGuid(), + dfeta_PersonId = new EntityReference(Contact.EntityLogicalName, contactId), + dfeta_InductionStatus = dfeta_InductionStatus.Pass, + dfeta_StartDate = inductionStartDate.ToDateTimeWithDqtBstFix(isLocalTime: true), + dfeta_CompletionDate = inductionEndDate.ToDateTimeWithDqtBstFix(isLocalTime: true), + CreatedOn = Clock.UtcNow, + ModifiedOn = Clock.UtcNow + }; + + // Keep the contact induction status in sync with dfeta_induction otherwise the sync will fail + await TestData.OrganizationService.ExecuteAsync(new UpdateRequest() + { + Target = new Contact() + { + Id = contactId, + dfeta_InductionStatus = dfeta_InductionStatus.Pass + } + }); + + var newItem = new NewOrUpdatedItem(ChangeType.NewOrUpdated, induction); + + // Act + await fixture.PublishChangedItemAndConsumeAsync(TrsDataSyncHelper.ModelTypes.Induction, newItem); + + // Assert + await fixture.DbFixture.WithDbContextAsync(async dbContext => + { + var person = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == contactId); + Assert.NotNull(person); + Assert.Equal(InductionStatus.Passed, person.InductionStatus); + Assert.Equal(inductionStartDate, person.InductionStartDate); + Assert.Equal(inductionEndDate, person.InductionCompletedDate); + }); + } + + [Fact] + public async Task Induction_UpdatedRecord_WritesUpdatedPersonRecordToDatabase() + { + // Arrange + var originalInductionStatus = dfeta_InductionStatus.InProgress; + var originalInductionStartDate = Clock.Today.AddYears(-1); + var originalInductionEndDate = Clock.Today.AddDays(-10); + + var createPersonResult = await TestData.CreatePersonAsync( + p => p.WithSyncOverride(true) + .WithDqtInduction(originalInductionStatus, null, originalInductionStartDate, null)); + var contactId = createPersonResult.ContactId; + var existingInduction = createPersonResult.DqtInductions.Single(); + + var updatedInductionStatus = dfeta_InductionStatus.Pass; + var updatedInductionStartDate = Clock.Today.AddYears(-2); + var updatedInductionEndDate = Clock.Today.AddDays(-20); + var createdOn = Clock.UtcNow; + var modifiedOn = Clock.Advance(); + var updatedInduction = new dfeta_induction() + { + Id = existingInduction.InductionId, + dfeta_PersonId = new EntityReference(Contact.EntityLogicalName, contactId), + dfeta_InductionStatus = updatedInductionStatus, + dfeta_StartDate = updatedInductionStartDate.ToDateTimeWithDqtBstFix(isLocalTime: true), + dfeta_CompletionDate = updatedInductionEndDate.ToDateTimeWithDqtBstFix(isLocalTime: true), + CreatedOn = Clock.UtcNow, + ModifiedOn = modifiedOn + }; + + // Keep the contact induction status in sync with dfeta_induction otherwise the sync will fail + await TestData.OrganizationService.ExecuteAsync(new UpdateRequest() + { + Target = new Contact() + { + Id = contactId, + dfeta_InductionStatus = dfeta_InductionStatus.Pass + } + }); + + var updatedItem = new NewOrUpdatedItem(ChangeType.NewOrUpdated, updatedInduction); + + // Act + await fixture.PublishChangedItemAndConsumeAsync(TrsDataSyncHelper.ModelTypes.Induction, updatedItem); + + // Assert + await fixture.DbFixture.WithDbContextAsync(async dbContext => + { + var person = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == contactId); + Assert.NotNull(person); + Assert.Equal(InductionStatus.Passed, person.InductionStatus); + Assert.Equal(updatedInductionStartDate, person.InductionStartDate); + Assert.Equal(updatedInductionEndDate, person.InductionCompletedDate); + Assert.Equal(Clock.UtcNow, person.DqtInductionModifiedOn); + }); + } +} diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceTests.cs index 7a0252337..c2e1ffa2c 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceTests.cs @@ -1,9 +1,14 @@ + namespace TeachingRecordSystem.Core.Tests.Services.TrsDataSync; [Collection(nameof(TrsDataSyncTestCollection))] -public partial class TrsDataSyncServiceTests(TrsDataSyncServiceFixture fixture) : IClassFixture +public partial class TrsDataSyncServiceTests(TrsDataSyncServiceFixture fixture) : IClassFixture, IAsyncLifetime { private TestableClock Clock => fixture.Clock; private TestData TestData => fixture.TestData; + + Task IAsyncLifetime.DisposeAsync() => Task.CompletedTask; + + Task IAsyncLifetime.InitializeAsync() => fixture.DbFixture.DbHelper.ClearDataAsync(); }