Skip to content

Commit

Permalink
Sync induction changes in real time from dqt (#1744)
Browse files Browse the repository at this point in the history
  • Loading branch information
hortha authored Dec 10, 2024
1 parent 735ece1 commit 81ce327
Show file tree
Hide file tree
Showing 6 changed files with 167 additions and 57 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ public static Command CreateSyncPersonCommand(IConfiguration configuration)
}

await syncHelper.SyncPersonAsync(contact, ignoreInvalid: false, dryRun: false);

await syncHelper.SyncInductionsAsync([contact], ignoreInvalid: false, createMigratedEvent: false, dryRun: false);
//return 0;
},
connectionStringOption,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public class Person
public required DateOnly? DateOfBirth { get; set; } // A few DQT records in prod have a null DOB
public string? EmailAddress { get; set; }
public string? NationalInsuranceNumber { get; set; }
public InductionStatus InductionStatus { get; private set; }
public InductionStatus InductionStatus { get; set; }
public InductionExemptionReasons InductionExemptionReasons { get; private set; }
public DateOnly? InductionStartDate { get; private set; }
public DateOnly? InductionCompletedDate { get; private set; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ public async Task<int> SyncInductionsAsync(
bool ignoreInvalid,
bool createMigratedEvent,
bool dryRun,
CancellationToken cancellationToken)
CancellationToken cancellationToken = default)
{
var inductionAttributeNames = new[]
{
Expand All @@ -328,6 +328,8 @@ public async Task<int> SyncInductionsAsync(
dfeta_induction.Fields.dfeta_InductionExemptionReason,
dfeta_induction.Fields.dfeta_StartDate,
dfeta_induction.Fields.dfeta_InductionStatus,
dfeta_induction.Fields.CreatedOn,
dfeta_induction.Fields.CreatedBy,
dfeta_induction.Fields.ModifiedOn
};

Expand Down Expand Up @@ -774,7 +776,7 @@ private async Task<IReadOnlyDictionary<Guid, AuditDetailCollection>> GetAuditRec
IEnumerable<Guid> ids,
CancellationToken cancellationToken)
{
return (await Task.WhenAll(ids
return (IsFakeXrm ? new Dictionary<Guid, AuditDetailCollection>() : (await Task.WhenAll(ids
.Chunk(MaxAuditRequestsPerBatch)
.Select(async chunk =>
{
Expand All @@ -788,6 +790,7 @@ private async Task<IReadOnlyDictionary<Guid, AuditDetailCollection>> GetAuditRec
}
};

// The following is not supported by FakeXrmEasy hence the check above to allow more test coverage
request.Requests.AddRange(chunk.Select(e => new RetrieveRecordChangeHistoryRequest() { Target = e.ToEntityReference(entityLogicalName) }));

ExecuteMultipleResponse response;
Expand Down Expand Up @@ -829,7 +832,7 @@ private async Task<IReadOnlyDictionary<Guid, AuditDetailCollection>> GetAuditRec
(r, e) => (Id: e, ((RetrieveRecordChangeHistoryResponse)r.Response).AuditDetailCollection));
})))
.SelectMany(b => b)
.ToDictionary(t => t.Id, t => t.AuditDetailCollection);
.ToDictionary(t => t.Id, t => t.AuditDetailCollection));
}

private async Task<TEntity[]> GetEntitiesAsync<TEntity>(
Expand Down Expand Up @@ -876,7 +879,8 @@ private static ModelTypeSyncInfo GetModelTypeSyncInfoForPerson()
"dqt_modified_on",
"dqt_first_name",
"dqt_middle_name",
"dqt_last_name"
"dqt_last_name",
"induction_status",
};

var columnsToUpdate = columnNames.Except(new[] { "person_id", "dqt_contact_id" }).ToArray();
Expand Down Expand Up @@ -905,6 +909,7 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on
Contact.Fields.ContactId,
Contact.Fields.StateCode,
Contact.Fields.CreatedOn,
Contact.Fields.CreatedBy,
Contact.Fields.ModifiedOn,
Contact.Fields.dfeta_TRN,
Contact.Fields.FirstName,
Expand All @@ -915,7 +920,8 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on
Contact.Fields.dfeta_StatedLastName,
Contact.Fields.BirthDate,
Contact.Fields.dfeta_NINumber,
Contact.Fields.EMailAddress1
Contact.Fields.EMailAddress1,
Contact.Fields.dfeta_InductionStatus
};

Action<NpgsqlBinaryImporter, Person> writeRecord = (writer, person) =>
Expand All @@ -937,6 +943,7 @@ WHERE t.dqt_modified_on < EXCLUDED.dqt_modified_on
writer.WriteValueOrNull(person.DqtFirstName, NpgsqlDbType.Varchar);
writer.WriteValueOrNull(person.DqtMiddleName, NpgsqlDbType.Varchar);
writer.WriteValueOrNull(person.DqtLastName, NpgsqlDbType.Varchar);
writer.WriteValueOrNull((int?)person.InductionStatus, NpgsqlDbType.Integer);
};

return new ModelTypeSyncInfo<Person>()
Expand Down Expand Up @@ -1171,38 +1178,11 @@ private static List<Person> MapPersons(IEnumerable<Contact> contacts) => contact
DqtModifiedOn = c.ModifiedOn!.Value,
DqtFirstName = c.FirstName ?? string.Empty,
DqtMiddleName = c.MiddleName ?? string.Empty,
DqtLastName = c.LastName ?? string.Empty
DqtLastName = c.LastName ?? string.Empty,
InductionStatus = c.dfeta_InductionStatus.ToInductionStatus()
})
.ToList();

private static List<InductionInfo> MapInductions(IReadOnlyCollection<Contact> contacts, IEnumerable<dfeta_induction> inductions, bool ignoreInvalid)
{
var inductionLookup = inductions
.GroupBy(i => i.dfeta_PersonId.Id)
.ToDictionary(g => g.Key, g => g.ToArray());

return contacts
.Select(contact =>
{
dfeta_induction? induction = null;
if (inductionLookup.TryGetValue(contact.ContactId!.Value, out var personInductions))
{
// We shouldn't have multiple induction records for the same person in prod at all but we might in other environments
// so we'll just take the most recently modified one.
induction = personInductions.OrderByDescending(i => i.ModifiedOn).First();
if (personInductions.Length > 1 && !ignoreInvalid)
{
throw new InvalidOperationException($"Contact '{contact.ContactId!.Value}' has multiple induction records.");
}
}

return MapInductionInfoFromDqtInduction(induction, contact, ignoreInvalid);
})
.Where(i => i is not null)
.Cast<InductionInfo>()
.ToList();
}

private (List<InductionInfo> Inductions, List<EventBase> Events) MapInductionsAndAudits(
IReadOnlyCollection<Contact> contacts,
IEnumerable<dfeta_induction> inductionEntities,
Expand Down Expand Up @@ -1247,39 +1227,46 @@ private static List<InductionInfo> MapInductions(IReadOnlyCollection<Contact> co
dfeta_induction.Fields.dfeta_InductionStatus,
dfeta_induction.Fields.ModifiedOn
};
var inductionAudits = auditDetails[induction!.Id].AuditDetails;
var inductionVersions = GetEntityVersions(induction, inductionAudits, inductionAttributeNames);

events.Add(inductionAudits.Any(a => a.AuditRecord.ToEntity<Audit>().Action == Audit_Action.Create) ?
MapCreatedEvent(inductionVersions.First()) :
MapImportedEvent(inductionVersions.First()));

foreach (var (thisVersion, previousVersion) in inductionVersions.Skip(1).Zip(inductionVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion)))
if (auditDetails.TryGetValue(induction!.Id, out var inductionAudits))
{
var mappedEvent = MapUpdatedEvent(thisVersion, previousVersion);
var inductionAuditDetails = inductionAudits.AuditDetails;
var inductionVersions = GetEntityVersions(induction, inductionAuditDetails, inductionAttributeNames);

if (mappedEvent is not null)
events.Add(inductionAuditDetails.Any(a => a.AuditRecord.ToEntity<Audit>().Action == Audit_Action.Create) ?
MapCreatedEvent(inductionVersions.First()) :
MapImportedEvent(inductionVersions.First()));

foreach (var (thisVersion, previousVersion) in inductionVersions.Skip(1).Zip(inductionVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion)))
{
events.Add(mappedEvent);
var mappedEvent = MapUpdatedEvent(thisVersion, previousVersion);

if (mappedEvent is not null)
{
events.Add(mappedEvent);
}
}
}

if (createMigratedEvent)
{
events.Add(MapMigratedEvent(inductionVersions.Last(), mapped));
if (createMigratedEvent)
{
events.Add(MapMigratedEvent(inductionVersions.Last(), mapped));
}
}
}

var contactAudits = auditDetails[contact.ContactId!.Value].AuditDetails;
var contactVersions = GetEntityVersions(contact, contactAudits, GetModelTypeSyncInfo(ModelTypes.Person).AttributeNames);

foreach (var (thisVersion, previousVersion) in contactVersions.Skip(1).Zip(contactVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion)))
if (auditDetails.TryGetValue(contact.ContactId!.Value, out var contactAudits))
{
var mappedEvent = MapContactInductionStatusChangedEvent(thisVersion, previousVersion);
var contactAuditDetails = contactAudits.AuditDetails;
var contactVersions = GetEntityVersions(contact, contactAuditDetails, GetModelTypeSyncInfo(ModelTypes.Person).AttributeNames);

if (mappedEvent is not null)
foreach (var (thisVersion, previousVersion) in contactVersions.Skip(1).Zip(contactVersions, (thisVersion, previousVersion) => (thisVersion, previousVersion)))
{
events.Add(mappedEvent);
var mappedEvent = MapContactInductionStatusChangedEvent(thisVersion, previousVersion);

if (mappedEvent is not null)
{
events.Add(mappedEvent);
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ internal async Task ProcessChangesAsync(CancellationToken cancellationToken)
var modelTypesToSync = optionsAccessor.Value.ModelTypes;

// Order is important here; the dependees should come before dependents
await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Induction);
await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Person);
await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Event);
await SyncIfEnabledAsync(TrsDataSyncHelper.ModelTypes.Alert);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
using Microsoft.Xrm.Sdk;
using Microsoft.Xrm.Sdk.Messages;
using TeachingRecordSystem.Core.Dqt;
using TeachingRecordSystem.Core.Dqt.Models;
using TeachingRecordSystem.Core.Services.TrsDataSync;

namespace TeachingRecordSystem.Core.Tests.Services.TrsDataSync;

public partial class TrsDataSyncServiceTests
{
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Induction_NewRecord_WritesUpdatedPersonRecordToDatabase(bool personAlreadySynced)
{
// Arrange
var createPersonResult = await TestData.CreatePersonAsync(p => p.WithSyncOverride(personAlreadySynced));
var contactId = createPersonResult.ContactId;

var inductionStartDate = Clock.Today.AddYears(-1);
var inductionEndDate = Clock.Today.AddDays(-10);
var induction = new dfeta_induction()
{
Id = Guid.NewGuid(),
dfeta_PersonId = new EntityReference(Contact.EntityLogicalName, contactId),
dfeta_InductionStatus = dfeta_InductionStatus.Pass,
dfeta_StartDate = inductionStartDate.ToDateTimeWithDqtBstFix(isLocalTime: true),
dfeta_CompletionDate = inductionEndDate.ToDateTimeWithDqtBstFix(isLocalTime: true),
CreatedOn = Clock.UtcNow,
ModifiedOn = Clock.UtcNow
};

// Keep the contact induction status in sync with dfeta_induction otherwise the sync will fail
await TestData.OrganizationService.ExecuteAsync(new UpdateRequest()
{
Target = new Contact()
{
Id = contactId,
dfeta_InductionStatus = dfeta_InductionStatus.Pass
}
});

var newItem = new NewOrUpdatedItem(ChangeType.NewOrUpdated, induction);

// Act
await fixture.PublishChangedItemAndConsumeAsync(TrsDataSyncHelper.ModelTypes.Induction, newItem);

// Assert
await fixture.DbFixture.WithDbContextAsync(async dbContext =>
{
var person = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == contactId);
Assert.NotNull(person);
Assert.Equal(InductionStatus.Passed, person.InductionStatus);
Assert.Equal(inductionStartDate, person.InductionStartDate);
Assert.Equal(inductionEndDate, person.InductionCompletedDate);
});
}

[Fact]
public async Task Induction_UpdatedRecord_WritesUpdatedPersonRecordToDatabase()
{
// Arrange
var originalInductionStatus = dfeta_InductionStatus.InProgress;
var originalInductionStartDate = Clock.Today.AddYears(-1);
var originalInductionEndDate = Clock.Today.AddDays(-10);

var createPersonResult = await TestData.CreatePersonAsync(
p => p.WithSyncOverride(true)
.WithDqtInduction(originalInductionStatus, null, originalInductionStartDate, null));
var contactId = createPersonResult.ContactId;
var existingInduction = createPersonResult.DqtInductions.Single();

var updatedInductionStatus = dfeta_InductionStatus.Pass;
var updatedInductionStartDate = Clock.Today.AddYears(-2);
var updatedInductionEndDate = Clock.Today.AddDays(-20);
var createdOn = Clock.UtcNow;
var modifiedOn = Clock.Advance();
var updatedInduction = new dfeta_induction()
{
Id = existingInduction.InductionId,
dfeta_PersonId = new EntityReference(Contact.EntityLogicalName, contactId),
dfeta_InductionStatus = updatedInductionStatus,
dfeta_StartDate = updatedInductionStartDate.ToDateTimeWithDqtBstFix(isLocalTime: true),
dfeta_CompletionDate = updatedInductionEndDate.ToDateTimeWithDqtBstFix(isLocalTime: true),
CreatedOn = Clock.UtcNow,
ModifiedOn = modifiedOn
};

// Keep the contact induction status in sync with dfeta_induction otherwise the sync will fail
await TestData.OrganizationService.ExecuteAsync(new UpdateRequest()
{
Target = new Contact()
{
Id = contactId,
dfeta_InductionStatus = dfeta_InductionStatus.Pass
}
});

var updatedItem = new NewOrUpdatedItem(ChangeType.NewOrUpdated, updatedInduction);

// Act
await fixture.PublishChangedItemAndConsumeAsync(TrsDataSyncHelper.ModelTypes.Induction, updatedItem);

// Assert
await fixture.DbFixture.WithDbContextAsync(async dbContext =>
{
var person = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == contactId);
Assert.NotNull(person);
Assert.Equal(InductionStatus.Passed, person.InductionStatus);
Assert.Equal(updatedInductionStartDate, person.InductionStartDate);
Assert.Equal(updatedInductionEndDate, person.InductionCompletedDate);
Assert.Equal(Clock.UtcNow, person.DqtInductionModifiedOn);
});
}
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@

namespace TeachingRecordSystem.Core.Tests.Services.TrsDataSync;

[Collection(nameof(TrsDataSyncTestCollection))]
public partial class TrsDataSyncServiceTests(TrsDataSyncServiceFixture fixture) : IClassFixture<TrsDataSyncServiceFixture>
public partial class TrsDataSyncServiceTests(TrsDataSyncServiceFixture fixture) : IClassFixture<TrsDataSyncServiceFixture>, IAsyncLifetime
{
private TestableClock Clock => fixture.Clock;

private TestData TestData => fixture.TestData;

Task IAsyncLifetime.DisposeAsync() => Task.CompletedTask;

Task IAsyncLifetime.InitializeAsync() => fixture.DbFixture.DbHelper.ClearDataAsync();
}

0 comments on commit 81ce327

Please sign in to comment.