diff --git a/TeachingRecordSystem/Dockerfile b/TeachingRecordSystem/Dockerfile index 198bf0c36..34849082f 100644 --- a/TeachingRecordSystem/Dockerfile +++ b/TeachingRecordSystem/Dockerfile @@ -34,4 +34,14 @@ RUN apk --no-cache add msttcorefonts-installer fontconfig && \ update-ms-fonts && \ fc-cache -f +# Install SQL Server tools needed to be able to query the reporting DB to help debugging +RUN apk add curl + +RUN curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/msodbcsql18_18.0.1.1-1_amd64.apk && \ + curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/mssql-tools18_18.0.1.1-1_amd64.apk + +RUN apk add --allow-untrusted msodbcsql18_18.0.1.1-1_amd64.apk && \ + apk add --allow-untrusted mssql-tools18_18.0.1.1-1_amd64.apk && \ + rm -f msodbcsql18_18.0.1.1-1_amd64.apk mssql-tools18_18.0.1.1-1_amd64.apk + ENV PATH="${PATH}:/Apps/TrsCli" diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllInductionsFromCrmJob.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllInductionsFromCrmJob.cs index 47edfbcde..f322802b9 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllInductionsFromCrmJob.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllInductionsFromCrmJob.cs @@ -1,5 +1,6 @@ using System.ServiceModel; using Hangfire; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Query; @@ -14,15 +15,18 @@ public class SyncAllInductionsFromCrmJob private readonly ICrmServiceClientProvider _crmServiceClientProvider; private readonly TrsDataSyncHelper _trsDataSyncHelper; private readonly IOptions _syncOptionsAccessor; + private readonly ILogger _logger; public SyncAllInductionsFromCrmJob( ICrmServiceClientProvider crmServiceClientProvider, TrsDataSyncHelper trsDataSyncHelper, - IOptions syncOptionsAccessor) + IOptions syncOptionsAccessor, + ILoggerFactory loggerFactory) { _crmServiceClientProvider = crmServiceClientProvider; _trsDataSyncHelper = trsDataSyncHelper; _syncOptionsAccessor = syncOptionsAccessor; + _logger = loggerFactory.CreateLogger(); } public async Task ExecuteAsync(bool createMigratedEvent, bool dryRun, CancellationToken cancellationToken) @@ -57,6 +61,7 @@ public async Task ExecuteAsync(bool createMigratedEvent, bool dryRun, Cancellati } catch (FaultException fex) when (fex.IsCrmRateLimitException(out var retryAfter)) { + _logger.LogWarning("Hit CRM service limits; error code: {ErrorCode}", fex.Detail.ErrorCode); await Task.Delay(retryAfter, cancellationToken); continue; } diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs index a4d266b57..8e9681c63 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs @@ -4,6 +4,7 @@ using System.ServiceModel; using Microsoft.Crm.Sdk.Messages; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Messages; @@ -16,11 +17,7 @@ namespace TeachingRecordSystem.Core.Services.TrsDataSync; -public class TrsDataSyncHelper( - NpgsqlDataSource trsDbDataSource, - [FromKeyedServices(TrsDataSyncService.CrmClientName)] IOrganizationServiceAsync2 organizationService, - ReferenceDataCache referenceDataCache, - IClock clock) +public class TrsDataSyncHelper { private delegate Task SyncEntitiesHandler(IReadOnlyCollection entities, bool ignoreInvalid, bool dryRun, CancellationToken cancellationToken); @@ -37,10 +34,41 @@ public class TrsDataSyncHelper( }; private readonly ISubject _syncedEntitiesSubject = new Subject(); + private readonly NpgsqlDataSource _trsDbDataSource; + private readonly IOrganizationServiceAsync2 _organizationService; + private readonly ReferenceDataCache _referenceDataCache; + private readonly IClock _clock; + private readonly ILogger _logger; + private bool? isFakeXrm; + + public TrsDataSyncHelper( + NpgsqlDataSource trsDbDataSource, + [FromKeyedServices(TrsDataSyncService.CrmClientName)] IOrganizationServiceAsync2 organizationService, + ReferenceDataCache referenceDataCache, + IClock clock, + ILoggerFactory loggerFactory) + { + _trsDbDataSource = trsDbDataSource; + _organizationService = organizationService; + _referenceDataCache = referenceDataCache; + _clock = clock; + _logger = loggerFactory.CreateLogger(); + } public IObservable GetSyncedEntitiesObservable() => _syncedEntitiesSubject; - private bool IsFakeXrm { get; } = organizationService.GetType().FullName == "Castle.Proxies.ObjectProxy_2"; + private bool IsFakeXrm + { + get + { + if (isFakeXrm is null) + { + isFakeXrm = _organizationService.GetType().FullName == "Castle.Proxies.ObjectProxy_2"; + } + + return isFakeXrm!.Value; + } + } public static (string EntityLogicalName, string[] AttributeNames) GetEntityInfoForModelType(string modelType) { @@ -142,7 +170,7 @@ public async Task DeleteRecordsAsync(string modelType, IReadOnlyCollection throw new NotSupportedException($"Cannot delete a {modelType}."); } - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); using (var cmd = connection.CreateCommand()) { @@ -161,7 +189,7 @@ public async Task DeleteRecordsAsync(string modelType, IReadOnlyCollection return null; } - await using var connection = await trsDbDataSource.OpenConnectionAsync(); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(); using (var cmd = connection.CreateCommand()) { @@ -221,7 +249,7 @@ public async Task> SyncPersonsAsync(IReadOnlyCollectio var modelTypeSyncInfo = GetModelTypeSyncInfo(ModelTypes.Person); - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); using var txn = await connection.BeginTransactionAsync(cancellationToken); using (var createTempTableCommand = connection.CreateCommand()) @@ -247,7 +275,7 @@ public async Task> SyncPersonsAsync(IReadOnlyCollectio using (var mergeCommand = connection.CreateCommand()) { mergeCommand.CommandText = modelTypeSyncInfo.UpsertStatement; - mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, clock.UtcNow)); + mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, _clock.UtcNow)); mergeCommand.Transaction = txn; await mergeCommand.ExecuteNonQueryAsync(); } @@ -330,7 +358,8 @@ public async Task SyncInductionsAsync( dfeta_induction.Fields.dfeta_InductionStatus, dfeta_induction.Fields.CreatedOn, dfeta_induction.Fields.CreatedBy, - dfeta_induction.Fields.ModifiedOn + dfeta_induction.Fields.ModifiedOn, + dfeta_induction.Fields.StateCode }; var inductions = await GetEntitiesAsync( @@ -384,7 +413,7 @@ private async Task SyncInductionsAsync( { var modelTypeSyncInfo = GetModelTypeSyncInfo(ModelTypes.Induction); - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); var toSync = inductions.ToList(); @@ -415,7 +444,7 @@ private async Task SyncInductionsAsync( using (var mergeCommand = connection.CreateCommand()) { mergeCommand.CommandText = modelTypeSyncInfo.UpsertStatement; - mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, clock.UtcNow)); + mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, _clock.UtcNow)); mergeCommand.Transaction = txn; using var reader = await mergeCommand.ExecuteReaderAsync(); while (await reader.ReadAsync(cancellationToken)) @@ -457,7 +486,7 @@ private async Task SyncInductionsAsync( .Where(e => e is IEventWithPersonId && !unsyncedContactIds.Any(c => c == ((IEventWithPersonId)e).PersonId)) .ToArray(); - await txn.SaveEventsAsync(eventsForSyncedContacts, "events_import", clock, cancellationToken); + await txn.SaveEventsAsync(eventsForSyncedContacts, "events_import", _clock, cancellationToken); if (!dryRun) { @@ -528,7 +557,7 @@ private async Task SyncAlertsAsync( var modelTypeSyncInfo = GetModelTypeSyncInfo(ModelTypes.Alert); - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); var toSync = alerts.ToList(); @@ -559,12 +588,12 @@ private async Task SyncAlertsAsync( using (var mergeCommand = connection.CreateCommand()) { mergeCommand.CommandText = modelTypeSyncInfo.UpsertStatement; - mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, clock.UtcNow)); + mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, _clock.UtcNow)); mergeCommand.Transaction = txn; await mergeCommand.ExecuteNonQueryAsync(); } - await txn.SaveEventsAsync(events, "events_import", clock, cancellationToken); + await txn.SaveEventsAsync(events, "events_import", _clock, cancellationToken); if (!dryRun) { @@ -620,10 +649,10 @@ public async Task SyncEventsAsync(IReadOnlyCollection event var mapped = events.Select(e => EventInfo.Deserialize(e.dfeta_Payload).Event).ToArray(); - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); using var txn = await connection.BeginTransactionAsync(cancellationToken); - await txn.SaveEventsAsync(mapped, tempTableSuffix: "events_import", clock, cancellationToken); + await txn.SaveEventsAsync(mapped, tempTableSuffix: "events_import", _clock, cancellationToken); if (!dryRun) { @@ -644,6 +673,7 @@ private EntityVersionInfo[] GetEntityVersions(TEntity latest, .OfType() .Select(a => (AuditDetail: a, AuditRecord: a.AuditRecord.ToEntity())) .OrderBy(a => a.AuditRecord.CreatedOn) + .ThenBy(a => a.AuditRecord.Action == Audit_Action.Create ? 0 : 1) .ToArray(); if (ordered.Length == 0) @@ -798,10 +828,11 @@ private async Task> GetAuditRec { try { - response = (ExecuteMultipleResponse)await organizationService.ExecuteAsync(request, cancellationToken); + response = (ExecuteMultipleResponse)await _organizationService.ExecuteAsync(request, cancellationToken); } catch (FaultException fex) when (fex.IsCrmRateLimitException(out var retryAfter)) { + _logger.LogWarning("Hit CRM service limits; Fault exception"); await Task.Delay(retryAfter, cancellationToken); continue; } @@ -812,11 +843,13 @@ private async Task> GetAuditRec if (firstFault.IsCrmRateLimitFault(out var retryAfter)) { + _logger.LogWarning("Hit CRM service limits; CRM rate limit fault"); await Task.Delay(retryAfter, cancellationToken); continue; } else if (firstFault.Message.Contains("The HTTP status code of the response was not expected (429)")) { + _logger.LogWarning("Hit CRM service limits; 429 too many requests"); await Task.Delay(TimeSpan.FromMinutes(2), cancellationToken); continue; } @@ -852,7 +885,25 @@ private async Task GetEntitiesAsync( query.Criteria.AddCondition("statecode", ConditionOperator.Equal, 0); } - var response = await organizationService.RetrieveMultipleAsync(query, cancellationToken); + EntityCollection response; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + response = await _organizationService.RetrieveMultipleAsync(query, cancellationToken); + } + catch (FaultException fex) when (fex.IsCrmRateLimitException(out var retryAfter)) + { + _logger.LogWarning("Hit CRM service limits; error code: {ErrorCode}", fex.Detail.ErrorCode); + await Task.Delay(retryAfter, cancellationToken); + continue; + } + + break; + } + return response.Entities.Select(e => e.ToEntity()).ToArray(); } @@ -1012,7 +1063,21 @@ RETURNING t.person_id { Contact.PrimaryIdAttribute, Contact.Fields.dfeta_InductionStatus, - Contact.Fields.dfeta_qtlsdate + Contact.Fields.dfeta_qtlsdate, + Contact.Fields.CreatedOn, + Contact.Fields.CreatedBy, + Contact.Fields.StateCode, + Contact.Fields.ModifiedOn, + Contact.Fields.dfeta_TRN, + Contact.Fields.FirstName, + Contact.Fields.MiddleName, + Contact.Fields.LastName, + Contact.Fields.dfeta_StatedFirstName, + Contact.Fields.dfeta_StatedMiddleName, + Contact.Fields.dfeta_StatedLastName, + Contact.Fields.BirthDate, + Contact.Fields.dfeta_NINumber, + Contact.Fields.EMailAddress1, }; Action writeRecord = (writer, induction) => @@ -1225,7 +1290,8 @@ private static List MapPersons(IEnumerable contacts) => contact dfeta_induction.Fields.dfeta_InductionExemptionReason, dfeta_induction.Fields.dfeta_StartDate, dfeta_induction.Fields.dfeta_InductionStatus, - dfeta_induction.Fields.ModifiedOn + dfeta_induction.Fields.ModifiedOn, + dfeta_induction.Fields.StateCode }; if (auditDetails.TryGetValue(induction!.Id, out var inductionAudits)) @@ -1404,7 +1470,7 @@ EventBase MapMigratedEvent(EntityVersionInfo snapshot, Inductio { EventId = Guid.NewGuid(), Key = $"{snapshot.Entity.Id}-Migrated", - CreatedUtc = clock.UtcNow, + CreatedUtc = _clock.UtcNow, RaisedBy = EventModels.RaisedByUserInfo.FromUserId(Core.DataStore.Postgres.Models.SystemUser.SystemUserId), PersonId = snapshot.Entity.dfeta_PersonId.Id, InductionStartDate = mappedInduction.InductionStartDate, @@ -1434,8 +1500,8 @@ EventModels.DqtInduction GetEventDqtInduction(dfeta_induction induction) bool ignoreInvalid, bool createMigratedEvent) { - var sanctionCodes = await referenceDataCache.GetSanctionCodesAsync(activeOnly: false); - var alertTypes = await referenceDataCache.GetAlertTypesAsync(); + var sanctionCodes = await _referenceDataCache.GetSanctionCodesAsync(activeOnly: false); + var alertTypes = await _referenceDataCache.GetAlertTypesAsync(); var alerts = new List(); var events = new List(); @@ -1469,7 +1535,7 @@ EventModels.DqtInduction GetEventDqtInduction(dfeta_induction induction) // If the record is deactivated then it's migrated as deleted if (s.StateCode == dfeta_sanctionState.Inactive) { - mapped.DeletedOn = clock.UtcNow; + mapped.DeletedOn = _clock.UtcNow; } else if (createMigratedEvent) { @@ -1586,7 +1652,7 @@ EventBase MapMigratedEvent(EntityVersionInfo snapshot) { EventId = Guid.NewGuid(), Key = $"{snapshot.Entity.Id}-Migrated", - CreatedUtc = clock.UtcNow, + CreatedUtc = _clock.UtcNow, RaisedBy = EventModels.RaisedByUserInfo.FromUserId(Core.DataStore.Postgres.Models.SystemUser.SystemUserId), PersonId = snapshot.Entity.dfeta_PersonId.Id, Alert = GetEventAlert(snapshot.Entity, applyMigrationMappings: true), diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Worker/appsettings.Production.json b/TeachingRecordSystem/src/TeachingRecordSystem.Worker/appsettings.Production.json index bc027606a..081aaef29 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Worker/appsettings.Production.json +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Worker/appsettings.Production.json @@ -3,7 +3,9 @@ "MinimumLevel": { "Default": "Error", "Override": { - "TeachingRecordSystem.Worker": "Warning" + "TeachingRecordSystem.Worker": "Warning", + "TeachingRecordSystem.Core.Services.TrsDataSync": "Warning", + "TeachingRecordSystem.Core.Jobs": "Warning" } } }, diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs index 8285cc908..59cedd5fc 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.Services.TrnGenerationApi; using TeachingRecordSystem.TestCommon; @@ -18,10 +19,11 @@ public sealed class CrmClientFixture : IDisposable private readonly CancellationTokenSource _completedCts; private readonly EnvironmentLockManager _lockManager; private readonly IMemoryCache _memoryCache; + private readonly ILoggerFactory _loggerFactory; private readonly ITrnGenerationApiClient _trnGenerationApiClient; private readonly ReferenceDataCache _referenceDataCache; - public CrmClientFixture(ServiceClient serviceClient, DbFixture dbFixture, IConfiguration configuration, IMemoryCache memoryCache) + public CrmClientFixture(ServiceClient serviceClient, DbFixture dbFixture, IConfiguration configuration, IMemoryCache memoryCache, ILoggerFactory loggerFactory) { Clock = new Clock(); Configuration = configuration; @@ -31,6 +33,7 @@ public CrmClientFixture(ServiceClient serviceClient, DbFixture dbFixture, IConfi _lockManager = new EnvironmentLockManager(Configuration); _lockManager.AcquireLock(_completedCts.Token); _memoryCache = memoryCache; + _loggerFactory = loggerFactory; _trnGenerationApiClient = GetTrnGenerationApiClient(); _referenceDataCache = new ReferenceDataCache( new CrmQueryDispatcher(CreateQueryServiceProvider(_baseServiceClient, referenceDataCache: null), serviceClientName: null), @@ -65,7 +68,7 @@ public TestDataScope CreateTestDataScope(bool withSync = false) _referenceDataCache, Clock, () => _trnGenerationApiClient.GenerateTrnAsync(), - withSync ? TestDataSyncConfiguration.Sync(new(DbFixture.GetDataSource(), orgService, _referenceDataCache, Clock)) : TestDataSyncConfiguration.NoSync()), + withSync ? TestDataSyncConfiguration.Sync(new(DbFixture.GetDataSource(), orgService, _referenceDataCache, Clock, _loggerFactory)) : TestDataSyncConfiguration.NoSync()), _memoryCache, onAsyncDispose); } diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/ApiSchema/EventMapperTestBase.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/ApiSchema/EventMapperTestBase.cs index 94dff2794..20645935d 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/ApiSchema/EventMapperTestBase.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/ApiSchema/EventMapperTestBase.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.Dqt; using TeachingRecordSystem.Core.Services.TrsDataSync; @@ -30,7 +31,8 @@ public EventMapperFixture( IOrganizationServiceAsync2 organizationService, ICrmQueryDispatcher crmQueryDispatcher, FakeTrnGenerator trnGenerator, - IServiceProvider serviceProvider) + IServiceProvider serviceProvider, + ILoggerFactory loggerFactory) { Clock = new TestableClock(); DbFixture = dbFixture; @@ -40,7 +42,8 @@ public EventMapperFixture( dbFixture.GetDataSource(), organizationService, ReferenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncAllInductionsFromCrmJobTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncAllInductionsFromCrmJobTests.cs new file mode 100644 index 000000000..f42e48113 --- /dev/null +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncAllInductionsFromCrmJobTests.cs @@ -0,0 +1,60 @@ +using Microsoft.Extensions.Options; +using TeachingRecordSystem.Core.Dqt.Models; +using TeachingRecordSystem.Core.Jobs; +using TeachingRecordSystem.Core.Services.TrsDataSync; +using TeachingRecordSystem.Core.Tests.Services.TrsDataSync; + +namespace TeachingRecordSystem.Core.Tests.Jobs; + +[CollectionDefinition(nameof(TrsDataSyncTestCollection), DisableParallelization = true)] +public class SyncAllInductionsFromCrmJobTests : SyncFromCrmJobTestBase, IAsyncLifetime +{ + public SyncAllInductionsFromCrmJobTests(SyncFromCrmJobFixture jobFixture) : base(jobFixture) + { + } + + [Fact(Skip = "Causes deadlock on CI for some reason")] + public async Task SyncInductionsAsync_WithExistingDqtInduction_UpdatesPersonRecord() + { + // Arrange + var inductionStatus = dfeta_InductionStatus.Pass; + var inductionStartDate = Clock.Today.AddYears(-1); + var inductionCompletedDate = Clock.Today.AddDays(-5); + var options = Options.Create(new TrsDataSyncServiceOptions() + { + CrmConnectionString = "dummy", + ModelTypes = [TrsDataSyncHelper.ModelTypes.Person], + PollIntervalSeconds = 60, + IgnoreInvalidData = false, + RunService = false + }); + + var person = await TestData.CreatePersonAsync( + p => p.WithTrn() + .WithQts() + .WithDqtInduction(inductionStatus, null, inductionStartDate, inductionCompletedDate) + .WithSyncOverride(false)); + + // Act + var job = new SyncAllInductionsFromCrmJob( + CrmServiceClientProvider, + Helper, + options, + LoggerFactory); + + await job.ExecuteAsync(createMigratedEvent: false, dryRun: false, CancellationToken.None); + + // Assert + await DbFixture.WithDbContextAsync(async dbContext => + { + var updatedPerson = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == person.ContactId); + Assert.Equal(inductionStatus.ToInductionStatus(), updatedPerson!.InductionStatus); + Assert.Equal(inductionStartDate, updatedPerson.InductionStartDate); + Assert.Equal(inductionCompletedDate, updatedPerson.InductionCompletedDate); + }); + } + + Task IAsyncLifetime.DisposeAsync() => Task.CompletedTask; + + Task IAsyncLifetime.InitializeAsync() => JobFixture.DbFixture.DbHelper.ClearDataAsync(); +} diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobFixture.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobFixture.cs new file mode 100644 index 000000000..ffe611cf3 --- /dev/null +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobFixture.cs @@ -0,0 +1,70 @@ +using Microsoft.Extensions.Logging; +using Microsoft.PowerPlatform.Dataverse.Client; +using TeachingRecordSystem.Core.Dqt; +using TeachingRecordSystem.Core.Services.TrsDataSync; + +namespace TeachingRecordSystem.Core.Tests.Jobs; + +public class SyncFromCrmJobFixture : IAsyncLifetime +{ + public SyncFromCrmJobFixture( + DbFixture dbFixture, + IOrganizationServiceAsync2 organizationService, + ReferenceDataCache referenceDataCache, + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) + { + DbFixture = dbFixture; + LoggerFactory = loggerFactory; + Clock = new(); + LoggerFactory = loggerFactory; + + Helper = new TrsDataSyncHelper( + dbFixture.GetDataSource(), + organizationService, + referenceDataCache, + Clock, + loggerFactory); + + TestData = new TestData( + dbFixture.GetDbContextFactory(), + organizationService, + referenceDataCache, + Clock, + trnGenerator, + TestDataSyncConfiguration.Sync(Helper)); + + CrmServiceClientProvider = new TestCrmServiceClientProvider(organizationService); + } + + public TestableClock Clock { get; } + + public DbFixture DbFixture { get; } + + public ILoggerFactory LoggerFactory { get; } + + public TrsDataSyncHelper Helper { get; } + + public TestData TestData { get; } + + public ICrmServiceClientProvider CrmServiceClientProvider { get; } + + Task IAsyncLifetime.DisposeAsync() => Task.CompletedTask; + + Task IAsyncLifetime.InitializeAsync() => Task.CompletedTask; + + private class TestCrmServiceClientProvider : ICrmServiceClientProvider + { + private readonly IOrganizationServiceAsync2 _organizationService; + + public TestCrmServiceClientProvider(IOrganizationServiceAsync2 organizationService) + { + _organizationService = organizationService; + } + + public IOrganizationServiceAsync2 GetClient(string name) + { + return _organizationService; + } + } +} diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobTestBase.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobTestBase.cs new file mode 100644 index 000000000..32fbf4257 --- /dev/null +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobTestBase.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.Logging; +using TeachingRecordSystem.Core.Dqt; +using TeachingRecordSystem.Core.Services.TrsDataSync; + +namespace TeachingRecordSystem.Core.Tests.Jobs; + +public abstract class SyncFromCrmJobTestBase : IClassFixture +{ + public SyncFromCrmJobTestBase(SyncFromCrmJobFixture jobFixture) + { + JobFixture = jobFixture; + } + + public SyncFromCrmJobFixture JobFixture { get; } + + protected TestableClock Clock => JobFixture.Clock; + + protected ILoggerFactory LoggerFactory => JobFixture.LoggerFactory; + + protected DbFixture DbFixture => JobFixture.DbFixture; + + protected TrsDataSyncHelper Helper => JobFixture.Helper; + + protected TestData TestData => JobFixture.TestData; + + public ICrmServiceClientProvider CrmServiceClientProvider => JobFixture.CrmServiceClientProvider; +} diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/DqtOutbox/OutboxMessageHandlerTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/DqtOutbox/OutboxMessageHandlerTests.cs index c05544490..4b013d65f 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/DqtOutbox/OutboxMessageHandlerTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/DqtOutbox/OutboxMessageHandlerTests.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.DataStore.Postgres; using TeachingRecordSystem.Core.Dqt; @@ -87,7 +88,8 @@ public OutboxMessageHandlerFixture( IOrganizationServiceAsync2 organizationService, IDbContextFactory dbContextFactory, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { Clock = new TestableClock(); DbFixture = dbFixture; @@ -97,7 +99,8 @@ public OutboxMessageHandlerFixture( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Gias/EstablishmentRefresherTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Gias/EstablishmentRefresherTests.cs index df38f69fe..4744dd3a8 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Gias/EstablishmentRefresherTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Gias/EstablishmentRefresherTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.Services.Establishments.Gias; using TeachingRecordSystem.Core.Services.TrsDataSync; @@ -11,7 +12,8 @@ public EstablishmentRefresherTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -20,7 +22,8 @@ public EstablishmentRefresherTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Tps/TpsEstablishmentRefresherTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Tps/TpsEstablishmentRefresherTests.cs index 810c3bdbe..2df7f2dbb 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Tps/TpsEstablishmentRefresherTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Tps/TpsEstablishmentRefresherTests.cs @@ -1,4 +1,5 @@ using System.Text; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.DataStore.Postgres.Models; using TeachingRecordSystem.Core.Services.Establishments.Tps; @@ -28,7 +29,8 @@ public TpsEstablishmentRefresherTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -37,7 +39,8 @@ public TpsEstablishmentRefresherTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/PersonMatching/PersonMatchingServiceTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/PersonMatching/PersonMatchingServiceTests.cs index 37cdb23f1..e1349f98a 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/PersonMatching/PersonMatchingServiceTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/PersonMatching/PersonMatchingServiceTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.DataStore.Postgres.Models; using TeachingRecordSystem.Core.Services.PersonMatching; @@ -12,7 +13,8 @@ public PersonMatchingServiceTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -21,7 +23,8 @@ public PersonMatchingServiceTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.Induction.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.Induction.cs index 1a86e3842..d0aaf3d97 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.Induction.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.Induction.cs @@ -96,6 +96,32 @@ await DbFixture.WithDbContextAsync(async dbContext => }); } + [Fact] + public async Task SyncInductionsAsync_WithExistingDqtInduction_UpdatesPersonRecord() + { + // Arrange + var inductionStatus = dfeta_InductionStatus.InProgress; + var inductionStartDate = Clock.Today.AddYears(-1); + + var person = await TestData.CreatePersonAsync( + p => p.WithTrn() + .WithQts() + .WithDqtInduction(inductionStatus, null, inductionStartDate, null) + .WithSyncOverride(false)); + + // Act + await Helper.SyncInductionsAsync([person.Contact], ignoreInvalid: true, createMigratedEvent: false, dryRun: false, CancellationToken.None); + + // Assert + await DbFixture.WithDbContextAsync(async dbContext => + { + var updatedPerson = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == person.ContactId); + Assert.Equal(inductionStatus.ToInductionStatus(), updatedPerson!.InductionStatus); + Assert.Equal(inductionStartDate, updatedPerson.InductionStartDate); + + }); + } + [Fact] public async Task SyncInductionsAsync_WithQtlsButNotExemptAndIgnoreInvalidSetToFalse_ThrowsException() { @@ -691,7 +717,7 @@ private Task GetEventsForInduction(Guid inductionId) => FROM events as e WHERE (e.payload -> 'Induction' ->> 'InductionId')::uuid = {inductionId} OR (e.payload -> 'DqtInduction' ->> 'InductionId')::uuid = {inductionId} - ORDER BY e.created + ORDER BY e.created, (CASE WHEN e.event_name = 'InductionMigratedEvent' THEN 1 ELSE 0 END) """).ToArrayAsync(); return results.Select(r => EventBase.Deserialize(r.Payload, r.EventName)).ToArray(); diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.cs index 671a8fa3d..3fcca22a2 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.cs @@ -1,5 +1,6 @@ using FakeXrmEasy.Extensions; using Microsoft.Crm.Sdk.Messages; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using Microsoft.Xrm.Sdk; using TeachingRecordSystem.Core.Dqt.Models; @@ -14,7 +15,8 @@ public TrsDataSyncHelperTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -23,7 +25,8 @@ public TrsDataSyncHelperTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceFixture.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceFixture.cs index 4b1af1e4a..ea5c2b6ed 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceFixture.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceFixture.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.PowerPlatform.Dataverse.Client; @@ -12,7 +13,8 @@ public TrsDataSyncServiceFixture( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -21,7 +23,8 @@ public TrsDataSyncServiceFixture( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs index f88fef40c..3ecd91eb9 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.DataStore.Postgres.Models; using TeachingRecordSystem.Core.Services.TrsDataSync; @@ -12,7 +13,8 @@ public TpsCsvExtractProcessorTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -21,7 +23,8 @@ public TpsCsvExtractProcessorTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/WorkforceDataExporterTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/WorkforceDataExporterTests.cs index 428aba7ca..486f49b15 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/WorkforceDataExporterTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/WorkforceDataExporterTests.cs @@ -1,5 +1,6 @@ using Google.Apis.Upload; using Google.Cloud.Storage.V1; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.PowerPlatform.Dataverse.Client; using Parquet.Serialization; @@ -16,7 +17,8 @@ public WorkforceDataExporterTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -25,7 +27,8 @@ public WorkforceDataExporterTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(),