From 7f062cbad0325365456e6098e93af477b9356528 Mon Sep 17 00:00:00 2001 From: Andrew Horth Date: Mon, 16 Dec 2024 13:52:06 +0000 Subject: [PATCH] Added fixes while testing against CRM build environment data Removed temporary code to avoid running the sync job against all data in CRM build Skip test which is failing in CI for the moment Added logging to sync helper to highlight when we're getting CRM rate limiting Added more logging when rate limiting is hit --- TeachingRecordSystem/Dockerfile | 10 ++ .../Jobs/SyncAllInductionsFromCrmJob.cs | 7 +- .../Services/TrsDataSync/TrsDataSyncHelper.cs | 122 ++++++++++++++---- .../appsettings.Production.json | 4 +- .../CrmClientFixture.cs | 7 +- .../ApiSchema/EventMapperTestBase.cs | 7 +- .../Jobs/SyncAllInductionsFromCrmJobTests.cs | 60 +++++++++ .../Jobs/SyncFromCrmJobFixture.cs | 70 ++++++++++ .../Jobs/SyncFromCrmJobTestBase.cs | 27 ++++ .../DqtOutbox/OutboxMessageHandlerTests.cs | 7 +- .../Gias/EstablishmentRefresherTests.cs | 7 +- .../Tps/TpsEstablishmentRefresherTests.cs | 7 +- .../PersonMatchingServiceTests.cs | 7 +- .../TrsDataSyncHelperTests.Induction.cs | 28 +++- .../TrsDataSync/TrsDataSyncHelperTests.cs | 7 +- .../TrsDataSync/TrsDataSyncServiceFixture.cs | 7 +- .../TpsCsvExtractProcessorTests.cs | 7 +- .../WorkforceDataExporterTests.cs | 7 +- 18 files changed, 347 insertions(+), 51 deletions(-) create mode 100644 TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncAllInductionsFromCrmJobTests.cs create mode 100644 TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobFixture.cs create mode 100644 TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobTestBase.cs diff --git a/TeachingRecordSystem/Dockerfile b/TeachingRecordSystem/Dockerfile index 198bf0c36..34849082f 100644 --- a/TeachingRecordSystem/Dockerfile +++ b/TeachingRecordSystem/Dockerfile @@ -34,4 +34,14 @@ RUN apk --no-cache add msttcorefonts-installer fontconfig && \ update-ms-fonts && \ fc-cache -f +# Install SQL Server tools needed to be able to query the reporting DB to help debugging +RUN apk add curl + +RUN curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/msodbcsql18_18.0.1.1-1_amd64.apk && \ + curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/mssql-tools18_18.0.1.1-1_amd64.apk + +RUN apk add --allow-untrusted msodbcsql18_18.0.1.1-1_amd64.apk && \ + apk add --allow-untrusted mssql-tools18_18.0.1.1-1_amd64.apk && \ + rm -f msodbcsql18_18.0.1.1-1_amd64.apk mssql-tools18_18.0.1.1-1_amd64.apk + ENV PATH="${PATH}:/Apps/TrsCli" diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllInductionsFromCrmJob.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllInductionsFromCrmJob.cs index 47edfbcde..f322802b9 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllInductionsFromCrmJob.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllInductionsFromCrmJob.cs @@ -1,5 +1,6 @@ using System.ServiceModel; using Hangfire; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Query; @@ -14,15 +15,18 @@ public class SyncAllInductionsFromCrmJob private readonly ICrmServiceClientProvider _crmServiceClientProvider; private readonly TrsDataSyncHelper _trsDataSyncHelper; private readonly IOptions _syncOptionsAccessor; + private readonly ILogger _logger; public SyncAllInductionsFromCrmJob( ICrmServiceClientProvider crmServiceClientProvider, TrsDataSyncHelper trsDataSyncHelper, - IOptions syncOptionsAccessor) + IOptions syncOptionsAccessor, + ILoggerFactory loggerFactory) { _crmServiceClientProvider = crmServiceClientProvider; _trsDataSyncHelper = trsDataSyncHelper; _syncOptionsAccessor = syncOptionsAccessor; + _logger = loggerFactory.CreateLogger(); } public async Task ExecuteAsync(bool createMigratedEvent, bool dryRun, CancellationToken cancellationToken) @@ -57,6 +61,7 @@ public async Task ExecuteAsync(bool createMigratedEvent, bool dryRun, Cancellati } catch (FaultException fex) when (fex.IsCrmRateLimitException(out var retryAfter)) { + _logger.LogWarning("Hit CRM service limits; error code: {ErrorCode}", fex.Detail.ErrorCode); await Task.Delay(retryAfter, cancellationToken); continue; } diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs index a4d266b57..8e9681c63 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs @@ -4,6 +4,7 @@ using System.ServiceModel; using Microsoft.Crm.Sdk.Messages; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Messages; @@ -16,11 +17,7 @@ namespace TeachingRecordSystem.Core.Services.TrsDataSync; -public class TrsDataSyncHelper( - NpgsqlDataSource trsDbDataSource, - [FromKeyedServices(TrsDataSyncService.CrmClientName)] IOrganizationServiceAsync2 organizationService, - ReferenceDataCache referenceDataCache, - IClock clock) +public class TrsDataSyncHelper { private delegate Task SyncEntitiesHandler(IReadOnlyCollection entities, bool ignoreInvalid, bool dryRun, CancellationToken cancellationToken); @@ -37,10 +34,41 @@ public class TrsDataSyncHelper( }; private readonly ISubject _syncedEntitiesSubject = new Subject(); + private readonly NpgsqlDataSource _trsDbDataSource; + private readonly IOrganizationServiceAsync2 _organizationService; + private readonly ReferenceDataCache _referenceDataCache; + private readonly IClock _clock; + private readonly ILogger _logger; + private bool? isFakeXrm; + + public TrsDataSyncHelper( + NpgsqlDataSource trsDbDataSource, + [FromKeyedServices(TrsDataSyncService.CrmClientName)] IOrganizationServiceAsync2 organizationService, + ReferenceDataCache referenceDataCache, + IClock clock, + ILoggerFactory loggerFactory) + { + _trsDbDataSource = trsDbDataSource; + _organizationService = organizationService; + _referenceDataCache = referenceDataCache; + _clock = clock; + _logger = loggerFactory.CreateLogger(); + } public IObservable GetSyncedEntitiesObservable() => _syncedEntitiesSubject; - private bool IsFakeXrm { get; } = organizationService.GetType().FullName == "Castle.Proxies.ObjectProxy_2"; + private bool IsFakeXrm + { + get + { + if (isFakeXrm is null) + { + isFakeXrm = _organizationService.GetType().FullName == "Castle.Proxies.ObjectProxy_2"; + } + + return isFakeXrm!.Value; + } + } public static (string EntityLogicalName, string[] AttributeNames) GetEntityInfoForModelType(string modelType) { @@ -142,7 +170,7 @@ public async Task DeleteRecordsAsync(string modelType, IReadOnlyCollection throw new NotSupportedException($"Cannot delete a {modelType}."); } - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); using (var cmd = connection.CreateCommand()) { @@ -161,7 +189,7 @@ public async Task DeleteRecordsAsync(string modelType, IReadOnlyCollection return null; } - await using var connection = await trsDbDataSource.OpenConnectionAsync(); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(); using (var cmd = connection.CreateCommand()) { @@ -221,7 +249,7 @@ public async Task> SyncPersonsAsync(IReadOnlyCollectio var modelTypeSyncInfo = GetModelTypeSyncInfo(ModelTypes.Person); - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); using var txn = await connection.BeginTransactionAsync(cancellationToken); using (var createTempTableCommand = connection.CreateCommand()) @@ -247,7 +275,7 @@ public async Task> SyncPersonsAsync(IReadOnlyCollectio using (var mergeCommand = connection.CreateCommand()) { mergeCommand.CommandText = modelTypeSyncInfo.UpsertStatement; - mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, clock.UtcNow)); + mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, _clock.UtcNow)); mergeCommand.Transaction = txn; await mergeCommand.ExecuteNonQueryAsync(); } @@ -330,7 +358,8 @@ public async Task SyncInductionsAsync( dfeta_induction.Fields.dfeta_InductionStatus, dfeta_induction.Fields.CreatedOn, dfeta_induction.Fields.CreatedBy, - dfeta_induction.Fields.ModifiedOn + dfeta_induction.Fields.ModifiedOn, + dfeta_induction.Fields.StateCode }; var inductions = await GetEntitiesAsync( @@ -384,7 +413,7 @@ private async Task SyncInductionsAsync( { var modelTypeSyncInfo = GetModelTypeSyncInfo(ModelTypes.Induction); - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); var toSync = inductions.ToList(); @@ -415,7 +444,7 @@ private async Task SyncInductionsAsync( using (var mergeCommand = connection.CreateCommand()) { mergeCommand.CommandText = modelTypeSyncInfo.UpsertStatement; - mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, clock.UtcNow)); + mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, _clock.UtcNow)); mergeCommand.Transaction = txn; using var reader = await mergeCommand.ExecuteReaderAsync(); while (await reader.ReadAsync(cancellationToken)) @@ -457,7 +486,7 @@ private async Task SyncInductionsAsync( .Where(e => e is IEventWithPersonId && !unsyncedContactIds.Any(c => c == ((IEventWithPersonId)e).PersonId)) .ToArray(); - await txn.SaveEventsAsync(eventsForSyncedContacts, "events_import", clock, cancellationToken); + await txn.SaveEventsAsync(eventsForSyncedContacts, "events_import", _clock, cancellationToken); if (!dryRun) { @@ -528,7 +557,7 @@ private async Task SyncAlertsAsync( var modelTypeSyncInfo = GetModelTypeSyncInfo(ModelTypes.Alert); - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); var toSync = alerts.ToList(); @@ -559,12 +588,12 @@ private async Task SyncAlertsAsync( using (var mergeCommand = connection.CreateCommand()) { mergeCommand.CommandText = modelTypeSyncInfo.UpsertStatement; - mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, clock.UtcNow)); + mergeCommand.Parameters.Add(new NpgsqlParameter(NowParameterName, _clock.UtcNow)); mergeCommand.Transaction = txn; await mergeCommand.ExecuteNonQueryAsync(); } - await txn.SaveEventsAsync(events, "events_import", clock, cancellationToken); + await txn.SaveEventsAsync(events, "events_import", _clock, cancellationToken); if (!dryRun) { @@ -620,10 +649,10 @@ public async Task SyncEventsAsync(IReadOnlyCollection event var mapped = events.Select(e => EventInfo.Deserialize(e.dfeta_Payload).Event).ToArray(); - await using var connection = await trsDbDataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _trsDbDataSource.OpenConnectionAsync(cancellationToken); using var txn = await connection.BeginTransactionAsync(cancellationToken); - await txn.SaveEventsAsync(mapped, tempTableSuffix: "events_import", clock, cancellationToken); + await txn.SaveEventsAsync(mapped, tempTableSuffix: "events_import", _clock, cancellationToken); if (!dryRun) { @@ -644,6 +673,7 @@ private EntityVersionInfo[] GetEntityVersions(TEntity latest, .OfType() .Select(a => (AuditDetail: a, AuditRecord: a.AuditRecord.ToEntity())) .OrderBy(a => a.AuditRecord.CreatedOn) + .ThenBy(a => a.AuditRecord.Action == Audit_Action.Create ? 0 : 1) .ToArray(); if (ordered.Length == 0) @@ -798,10 +828,11 @@ private async Task> GetAuditRec { try { - response = (ExecuteMultipleResponse)await organizationService.ExecuteAsync(request, cancellationToken); + response = (ExecuteMultipleResponse)await _organizationService.ExecuteAsync(request, cancellationToken); } catch (FaultException fex) when (fex.IsCrmRateLimitException(out var retryAfter)) { + _logger.LogWarning("Hit CRM service limits; Fault exception"); await Task.Delay(retryAfter, cancellationToken); continue; } @@ -812,11 +843,13 @@ private async Task> GetAuditRec if (firstFault.IsCrmRateLimitFault(out var retryAfter)) { + _logger.LogWarning("Hit CRM service limits; CRM rate limit fault"); await Task.Delay(retryAfter, cancellationToken); continue; } else if (firstFault.Message.Contains("The HTTP status code of the response was not expected (429)")) { + _logger.LogWarning("Hit CRM service limits; 429 too many requests"); await Task.Delay(TimeSpan.FromMinutes(2), cancellationToken); continue; } @@ -852,7 +885,25 @@ private async Task GetEntitiesAsync( query.Criteria.AddCondition("statecode", ConditionOperator.Equal, 0); } - var response = await organizationService.RetrieveMultipleAsync(query, cancellationToken); + EntityCollection response; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + response = await _organizationService.RetrieveMultipleAsync(query, cancellationToken); + } + catch (FaultException fex) when (fex.IsCrmRateLimitException(out var retryAfter)) + { + _logger.LogWarning("Hit CRM service limits; error code: {ErrorCode}", fex.Detail.ErrorCode); + await Task.Delay(retryAfter, cancellationToken); + continue; + } + + break; + } + return response.Entities.Select(e => e.ToEntity()).ToArray(); } @@ -1012,7 +1063,21 @@ RETURNING t.person_id { Contact.PrimaryIdAttribute, Contact.Fields.dfeta_InductionStatus, - Contact.Fields.dfeta_qtlsdate + Contact.Fields.dfeta_qtlsdate, + Contact.Fields.CreatedOn, + Contact.Fields.CreatedBy, + Contact.Fields.StateCode, + Contact.Fields.ModifiedOn, + Contact.Fields.dfeta_TRN, + Contact.Fields.FirstName, + Contact.Fields.MiddleName, + Contact.Fields.LastName, + Contact.Fields.dfeta_StatedFirstName, + Contact.Fields.dfeta_StatedMiddleName, + Contact.Fields.dfeta_StatedLastName, + Contact.Fields.BirthDate, + Contact.Fields.dfeta_NINumber, + Contact.Fields.EMailAddress1, }; Action writeRecord = (writer, induction) => @@ -1225,7 +1290,8 @@ private static List MapPersons(IEnumerable contacts) => contact dfeta_induction.Fields.dfeta_InductionExemptionReason, dfeta_induction.Fields.dfeta_StartDate, dfeta_induction.Fields.dfeta_InductionStatus, - dfeta_induction.Fields.ModifiedOn + dfeta_induction.Fields.ModifiedOn, + dfeta_induction.Fields.StateCode }; if (auditDetails.TryGetValue(induction!.Id, out var inductionAudits)) @@ -1404,7 +1470,7 @@ EventBase MapMigratedEvent(EntityVersionInfo snapshot, Inductio { EventId = Guid.NewGuid(), Key = $"{snapshot.Entity.Id}-Migrated", - CreatedUtc = clock.UtcNow, + CreatedUtc = _clock.UtcNow, RaisedBy = EventModels.RaisedByUserInfo.FromUserId(Core.DataStore.Postgres.Models.SystemUser.SystemUserId), PersonId = snapshot.Entity.dfeta_PersonId.Id, InductionStartDate = mappedInduction.InductionStartDate, @@ -1434,8 +1500,8 @@ EventModels.DqtInduction GetEventDqtInduction(dfeta_induction induction) bool ignoreInvalid, bool createMigratedEvent) { - var sanctionCodes = await referenceDataCache.GetSanctionCodesAsync(activeOnly: false); - var alertTypes = await referenceDataCache.GetAlertTypesAsync(); + var sanctionCodes = await _referenceDataCache.GetSanctionCodesAsync(activeOnly: false); + var alertTypes = await _referenceDataCache.GetAlertTypesAsync(); var alerts = new List(); var events = new List(); @@ -1469,7 +1535,7 @@ EventModels.DqtInduction GetEventDqtInduction(dfeta_induction induction) // If the record is deactivated then it's migrated as deleted if (s.StateCode == dfeta_sanctionState.Inactive) { - mapped.DeletedOn = clock.UtcNow; + mapped.DeletedOn = _clock.UtcNow; } else if (createMigratedEvent) { @@ -1586,7 +1652,7 @@ EventBase MapMigratedEvent(EntityVersionInfo snapshot) { EventId = Guid.NewGuid(), Key = $"{snapshot.Entity.Id}-Migrated", - CreatedUtc = clock.UtcNow, + CreatedUtc = _clock.UtcNow, RaisedBy = EventModels.RaisedByUserInfo.FromUserId(Core.DataStore.Postgres.Models.SystemUser.SystemUserId), PersonId = snapshot.Entity.dfeta_PersonId.Id, Alert = GetEventAlert(snapshot.Entity, applyMigrationMappings: true), diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Worker/appsettings.Production.json b/TeachingRecordSystem/src/TeachingRecordSystem.Worker/appsettings.Production.json index bc027606a..081aaef29 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Worker/appsettings.Production.json +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Worker/appsettings.Production.json @@ -3,7 +3,9 @@ "MinimumLevel": { "Default": "Error", "Override": { - "TeachingRecordSystem.Worker": "Warning" + "TeachingRecordSystem.Worker": "Warning", + "TeachingRecordSystem.Core.Services.TrsDataSync": "Warning", + "TeachingRecordSystem.Core.Jobs": "Warning" } } }, diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs index 8285cc908..59cedd5fc 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Dqt.CrmIntegrationTests/CrmClientFixture.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.Services.TrnGenerationApi; using TeachingRecordSystem.TestCommon; @@ -18,10 +19,11 @@ public sealed class CrmClientFixture : IDisposable private readonly CancellationTokenSource _completedCts; private readonly EnvironmentLockManager _lockManager; private readonly IMemoryCache _memoryCache; + private readonly ILoggerFactory _loggerFactory; private readonly ITrnGenerationApiClient _trnGenerationApiClient; private readonly ReferenceDataCache _referenceDataCache; - public CrmClientFixture(ServiceClient serviceClient, DbFixture dbFixture, IConfiguration configuration, IMemoryCache memoryCache) + public CrmClientFixture(ServiceClient serviceClient, DbFixture dbFixture, IConfiguration configuration, IMemoryCache memoryCache, ILoggerFactory loggerFactory) { Clock = new Clock(); Configuration = configuration; @@ -31,6 +33,7 @@ public CrmClientFixture(ServiceClient serviceClient, DbFixture dbFixture, IConfi _lockManager = new EnvironmentLockManager(Configuration); _lockManager.AcquireLock(_completedCts.Token); _memoryCache = memoryCache; + _loggerFactory = loggerFactory; _trnGenerationApiClient = GetTrnGenerationApiClient(); _referenceDataCache = new ReferenceDataCache( new CrmQueryDispatcher(CreateQueryServiceProvider(_baseServiceClient, referenceDataCache: null), serviceClientName: null), @@ -65,7 +68,7 @@ public TestDataScope CreateTestDataScope(bool withSync = false) _referenceDataCache, Clock, () => _trnGenerationApiClient.GenerateTrnAsync(), - withSync ? TestDataSyncConfiguration.Sync(new(DbFixture.GetDataSource(), orgService, _referenceDataCache, Clock)) : TestDataSyncConfiguration.NoSync()), + withSync ? TestDataSyncConfiguration.Sync(new(DbFixture.GetDataSource(), orgService, _referenceDataCache, Clock, _loggerFactory)) : TestDataSyncConfiguration.NoSync()), _memoryCache, onAsyncDispose); } diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/ApiSchema/EventMapperTestBase.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/ApiSchema/EventMapperTestBase.cs index 94dff2794..20645935d 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/ApiSchema/EventMapperTestBase.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/ApiSchema/EventMapperTestBase.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.Dqt; using TeachingRecordSystem.Core.Services.TrsDataSync; @@ -30,7 +31,8 @@ public EventMapperFixture( IOrganizationServiceAsync2 organizationService, ICrmQueryDispatcher crmQueryDispatcher, FakeTrnGenerator trnGenerator, - IServiceProvider serviceProvider) + IServiceProvider serviceProvider, + ILoggerFactory loggerFactory) { Clock = new TestableClock(); DbFixture = dbFixture; @@ -40,7 +42,8 @@ public EventMapperFixture( dbFixture.GetDataSource(), organizationService, ReferenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncAllInductionsFromCrmJobTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncAllInductionsFromCrmJobTests.cs new file mode 100644 index 000000000..f42e48113 --- /dev/null +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncAllInductionsFromCrmJobTests.cs @@ -0,0 +1,60 @@ +using Microsoft.Extensions.Options; +using TeachingRecordSystem.Core.Dqt.Models; +using TeachingRecordSystem.Core.Jobs; +using TeachingRecordSystem.Core.Services.TrsDataSync; +using TeachingRecordSystem.Core.Tests.Services.TrsDataSync; + +namespace TeachingRecordSystem.Core.Tests.Jobs; + +[CollectionDefinition(nameof(TrsDataSyncTestCollection), DisableParallelization = true)] +public class SyncAllInductionsFromCrmJobTests : SyncFromCrmJobTestBase, IAsyncLifetime +{ + public SyncAllInductionsFromCrmJobTests(SyncFromCrmJobFixture jobFixture) : base(jobFixture) + { + } + + [Fact(Skip = "Causes deadlock on CI for some reason")] + public async Task SyncInductionsAsync_WithExistingDqtInduction_UpdatesPersonRecord() + { + // Arrange + var inductionStatus = dfeta_InductionStatus.Pass; + var inductionStartDate = Clock.Today.AddYears(-1); + var inductionCompletedDate = Clock.Today.AddDays(-5); + var options = Options.Create(new TrsDataSyncServiceOptions() + { + CrmConnectionString = "dummy", + ModelTypes = [TrsDataSyncHelper.ModelTypes.Person], + PollIntervalSeconds = 60, + IgnoreInvalidData = false, + RunService = false + }); + + var person = await TestData.CreatePersonAsync( + p => p.WithTrn() + .WithQts() + .WithDqtInduction(inductionStatus, null, inductionStartDate, inductionCompletedDate) + .WithSyncOverride(false)); + + // Act + var job = new SyncAllInductionsFromCrmJob( + CrmServiceClientProvider, + Helper, + options, + LoggerFactory); + + await job.ExecuteAsync(createMigratedEvent: false, dryRun: false, CancellationToken.None); + + // Assert + await DbFixture.WithDbContextAsync(async dbContext => + { + var updatedPerson = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == person.ContactId); + Assert.Equal(inductionStatus.ToInductionStatus(), updatedPerson!.InductionStatus); + Assert.Equal(inductionStartDate, updatedPerson.InductionStartDate); + Assert.Equal(inductionCompletedDate, updatedPerson.InductionCompletedDate); + }); + } + + Task IAsyncLifetime.DisposeAsync() => Task.CompletedTask; + + Task IAsyncLifetime.InitializeAsync() => JobFixture.DbFixture.DbHelper.ClearDataAsync(); +} diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobFixture.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobFixture.cs new file mode 100644 index 000000000..ffe611cf3 --- /dev/null +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobFixture.cs @@ -0,0 +1,70 @@ +using Microsoft.Extensions.Logging; +using Microsoft.PowerPlatform.Dataverse.Client; +using TeachingRecordSystem.Core.Dqt; +using TeachingRecordSystem.Core.Services.TrsDataSync; + +namespace TeachingRecordSystem.Core.Tests.Jobs; + +public class SyncFromCrmJobFixture : IAsyncLifetime +{ + public SyncFromCrmJobFixture( + DbFixture dbFixture, + IOrganizationServiceAsync2 organizationService, + ReferenceDataCache referenceDataCache, + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) + { + DbFixture = dbFixture; + LoggerFactory = loggerFactory; + Clock = new(); + LoggerFactory = loggerFactory; + + Helper = new TrsDataSyncHelper( + dbFixture.GetDataSource(), + organizationService, + referenceDataCache, + Clock, + loggerFactory); + + TestData = new TestData( + dbFixture.GetDbContextFactory(), + organizationService, + referenceDataCache, + Clock, + trnGenerator, + TestDataSyncConfiguration.Sync(Helper)); + + CrmServiceClientProvider = new TestCrmServiceClientProvider(organizationService); + } + + public TestableClock Clock { get; } + + public DbFixture DbFixture { get; } + + public ILoggerFactory LoggerFactory { get; } + + public TrsDataSyncHelper Helper { get; } + + public TestData TestData { get; } + + public ICrmServiceClientProvider CrmServiceClientProvider { get; } + + Task IAsyncLifetime.DisposeAsync() => Task.CompletedTask; + + Task IAsyncLifetime.InitializeAsync() => Task.CompletedTask; + + private class TestCrmServiceClientProvider : ICrmServiceClientProvider + { + private readonly IOrganizationServiceAsync2 _organizationService; + + public TestCrmServiceClientProvider(IOrganizationServiceAsync2 organizationService) + { + _organizationService = organizationService; + } + + public IOrganizationServiceAsync2 GetClient(string name) + { + return _organizationService; + } + } +} diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobTestBase.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobTestBase.cs new file mode 100644 index 000000000..32fbf4257 --- /dev/null +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Jobs/SyncFromCrmJobTestBase.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.Logging; +using TeachingRecordSystem.Core.Dqt; +using TeachingRecordSystem.Core.Services.TrsDataSync; + +namespace TeachingRecordSystem.Core.Tests.Jobs; + +public abstract class SyncFromCrmJobTestBase : IClassFixture +{ + public SyncFromCrmJobTestBase(SyncFromCrmJobFixture jobFixture) + { + JobFixture = jobFixture; + } + + public SyncFromCrmJobFixture JobFixture { get; } + + protected TestableClock Clock => JobFixture.Clock; + + protected ILoggerFactory LoggerFactory => JobFixture.LoggerFactory; + + protected DbFixture DbFixture => JobFixture.DbFixture; + + protected TrsDataSyncHelper Helper => JobFixture.Helper; + + protected TestData TestData => JobFixture.TestData; + + public ICrmServiceClientProvider CrmServiceClientProvider => JobFixture.CrmServiceClientProvider; +} diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/DqtOutbox/OutboxMessageHandlerTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/DqtOutbox/OutboxMessageHandlerTests.cs index c05544490..4b013d65f 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/DqtOutbox/OutboxMessageHandlerTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/DqtOutbox/OutboxMessageHandlerTests.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.DataStore.Postgres; using TeachingRecordSystem.Core.Dqt; @@ -87,7 +88,8 @@ public OutboxMessageHandlerFixture( IOrganizationServiceAsync2 organizationService, IDbContextFactory dbContextFactory, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { Clock = new TestableClock(); DbFixture = dbFixture; @@ -97,7 +99,8 @@ public OutboxMessageHandlerFixture( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Gias/EstablishmentRefresherTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Gias/EstablishmentRefresherTests.cs index df38f69fe..4744dd3a8 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Gias/EstablishmentRefresherTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Gias/EstablishmentRefresherTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.Services.Establishments.Gias; using TeachingRecordSystem.Core.Services.TrsDataSync; @@ -11,7 +12,8 @@ public EstablishmentRefresherTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -20,7 +22,8 @@ public EstablishmentRefresherTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Tps/TpsEstablishmentRefresherTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Tps/TpsEstablishmentRefresherTests.cs index 810c3bdbe..2df7f2dbb 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Tps/TpsEstablishmentRefresherTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/Establishments/Tps/TpsEstablishmentRefresherTests.cs @@ -1,4 +1,5 @@ using System.Text; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.DataStore.Postgres.Models; using TeachingRecordSystem.Core.Services.Establishments.Tps; @@ -28,7 +29,8 @@ public TpsEstablishmentRefresherTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -37,7 +39,8 @@ public TpsEstablishmentRefresherTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/PersonMatching/PersonMatchingServiceTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/PersonMatching/PersonMatchingServiceTests.cs index 37cdb23f1..e1349f98a 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/PersonMatching/PersonMatchingServiceTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/PersonMatching/PersonMatchingServiceTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.DataStore.Postgres.Models; using TeachingRecordSystem.Core.Services.PersonMatching; @@ -12,7 +13,8 @@ public PersonMatchingServiceTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -21,7 +23,8 @@ public PersonMatchingServiceTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.Induction.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.Induction.cs index 1a86e3842..d0aaf3d97 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.Induction.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.Induction.cs @@ -96,6 +96,32 @@ await DbFixture.WithDbContextAsync(async dbContext => }); } + [Fact] + public async Task SyncInductionsAsync_WithExistingDqtInduction_UpdatesPersonRecord() + { + // Arrange + var inductionStatus = dfeta_InductionStatus.InProgress; + var inductionStartDate = Clock.Today.AddYears(-1); + + var person = await TestData.CreatePersonAsync( + p => p.WithTrn() + .WithQts() + .WithDqtInduction(inductionStatus, null, inductionStartDate, null) + .WithSyncOverride(false)); + + // Act + await Helper.SyncInductionsAsync([person.Contact], ignoreInvalid: true, createMigratedEvent: false, dryRun: false, CancellationToken.None); + + // Assert + await DbFixture.WithDbContextAsync(async dbContext => + { + var updatedPerson = await dbContext.Persons.SingleOrDefaultAsync(p => p.DqtContactId == person.ContactId); + Assert.Equal(inductionStatus.ToInductionStatus(), updatedPerson!.InductionStatus); + Assert.Equal(inductionStartDate, updatedPerson.InductionStartDate); + + }); + } + [Fact] public async Task SyncInductionsAsync_WithQtlsButNotExemptAndIgnoreInvalidSetToFalse_ThrowsException() { @@ -691,7 +717,7 @@ private Task GetEventsForInduction(Guid inductionId) => FROM events as e WHERE (e.payload -> 'Induction' ->> 'InductionId')::uuid = {inductionId} OR (e.payload -> 'DqtInduction' ->> 'InductionId')::uuid = {inductionId} - ORDER BY e.created + ORDER BY e.created, (CASE WHEN e.event_name = 'InductionMigratedEvent' THEN 1 ELSE 0 END) """).ToArrayAsync(); return results.Select(r => EventBase.Deserialize(r.Payload, r.EventName)).ToArray(); diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.cs index 671a8fa3d..3fcca22a2 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncHelperTests.cs @@ -1,5 +1,6 @@ using FakeXrmEasy.Extensions; using Microsoft.Crm.Sdk.Messages; +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using Microsoft.Xrm.Sdk; using TeachingRecordSystem.Core.Dqt.Models; @@ -14,7 +15,8 @@ public TrsDataSyncHelperTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -23,7 +25,8 @@ public TrsDataSyncHelperTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceFixture.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceFixture.cs index 4b1af1e4a..ea5c2b6ed 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceFixture.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/TrsDataSync/TrsDataSyncServiceFixture.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.PowerPlatform.Dataverse.Client; @@ -12,7 +13,8 @@ public TrsDataSyncServiceFixture( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -21,7 +23,8 @@ public TrsDataSyncServiceFixture( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs index f88fef40c..3ecd91eb9 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/TpsCsvExtractProcessorTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.PowerPlatform.Dataverse.Client; using TeachingRecordSystem.Core.DataStore.Postgres.Models; using TeachingRecordSystem.Core.Services.TrsDataSync; @@ -12,7 +13,8 @@ public TpsCsvExtractProcessorTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -21,7 +23,8 @@ public TpsCsvExtractProcessorTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(), diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/WorkforceDataExporterTests.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/WorkforceDataExporterTests.cs index 428aba7ca..486f49b15 100644 --- a/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/WorkforceDataExporterTests.cs +++ b/TeachingRecordSystem/tests/TeachingRecordSystem.Core.Tests/Services/WorkforceData/WorkforceDataExporterTests.cs @@ -1,5 +1,6 @@ using Google.Apis.Upload; using Google.Cloud.Storage.V1; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.PowerPlatform.Dataverse.Client; using Parquet.Serialization; @@ -16,7 +17,8 @@ public WorkforceDataExporterTests( DbFixture dbFixture, IOrganizationServiceAsync2 organizationService, ReferenceDataCache referenceDataCache, - FakeTrnGenerator trnGenerator) + FakeTrnGenerator trnGenerator, + ILoggerFactory loggerFactory) { DbFixture = dbFixture; Clock = new(); @@ -25,7 +27,8 @@ public WorkforceDataExporterTests( dbFixture.GetDataSource(), organizationService, referenceDataCache, - Clock); + Clock, + loggerFactory); TestData = new TestData( dbFixture.GetDbContextFactory(),