Skip entities that already we already have audit detail for

Also increases production worker memory to 8GB.
DFE-Digital · Dec 30, 2024 · 7e5ef3b · 7e5ef3b
1 parent 3500f3c
commit 7e5ef3b
Show file tree

Hide file tree

Showing 7 changed files with 60 additions and 12 deletions.
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtContactAuditsJob.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtContactAuditsJob.cs
@@ -1,6 +1,6 @@
 using System.ServiceModel;
-using Hangfire;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using Microsoft.PowerPlatform.Dataverse.Client;
 using Microsoft.Xrm.Sdk;
 using Microsoft.Xrm.Sdk.Query;
@@ -9,10 +9,10 @@
 
 namespace TeachingRecordSystem.Core.Jobs;
 
-[AutomaticRetry(Attempts = 0)]
 public class SyncAllDqtContactAuditsJob(
     [FromKeyedServices(TrsDataSyncService.CrmClientName)] IOrganizationServiceAsync2 organizationService,
-    TrsDataSyncHelper trsDataSyncHelper)
+    TrsDataSyncHelper trsDataSyncHelper,
+    ILogger<SyncAllDqtContactAuditsJob> logger)
 {
     public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
@@ -32,6 +32,8 @@ public async Task ExecuteAsync(CancellationToken cancellationToken)
             }
         };
 
+        var fetched = 0;
+
         while (true)
         {
             cancellationToken.ThrowIfCancellationRequested();
@@ -47,7 +49,18 @@ public async Task ExecuteAsync(CancellationToken cancellationToken)
                 continue;
             }
 
-            await trsDataSyncHelper.SyncAuditAsync(Contact.EntityLogicalName, result.Entities.Select(e => e.Id), cancellationToken);
+            fetched += result.Entities.Count;
+
+            await trsDataSyncHelper.SyncAuditAsync(
+                Contact.EntityLogicalName,
+                result.Entities.Select(e => e.Id),
+                skipIfExists: true,
+                cancellationToken);
+
+            if (fetched > 0 && fetched % 50000 == 0)
+            {
+                logger.LogWarning("Synced {Count} contact audit records.", fetched);
+            }
 
             if (result.MoreRecords)
             {

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtInductionAuditsJob.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Jobs/SyncAllDqtInductionAuditsJob.cs
@@ -1,5 +1,4 @@
 using System.ServiceModel;
-using Hangfire;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.PowerPlatform.Dataverse.Client;
 using Microsoft.Xrm.Sdk;
@@ -9,7 +8,6 @@
 
 namespace TeachingRecordSystem.Core.Jobs;
 
-[AutomaticRetry(Attempts = 0)]
 public class SyncAllDqtInductionAuditsJob(
     [FromKeyedServices(TrsDataSyncService.CrmClientName)] IOrganizationServiceAsync2 organizationService,
     TrsDataSyncHelper trsDataSyncHelper)
@@ -47,7 +45,11 @@ public async Task ExecuteAsync(CancellationToken cancellationToken)
                 continue;
             }
 
-            await trsDataSyncHelper.SyncAuditAsync(dfeta_induction.EntityLogicalName, result.Entities.Select(e => e.Id), cancellationToken);
+            await trsDataSyncHelper.SyncAuditAsync(
+                dfeta_induction.EntityLogicalName,
+                result.Entities.Select(e => e.Id),
+                skipIfExists: true,
+                cancellationToken);
 
             if (result.MoreRecords)
             {

diff --git a/...rdSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/BlobStorageAuditRepository.cs b/...rdSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/BlobStorageAuditRepository.cs
@@ -35,6 +35,13 @@ public class BlobStorageAuditRepository(BlobServiceClient blobServiceClient) : I
         }
     }
 
+    public async Task<bool> HaveAuditDetailAsync(string entityLogicalName, Guid id)
+    {
+        var containerClient = blobServiceClient.GetBlobContainerClient(ContainerName);
+        var blobClient = containerClient.GetBlobClient(GetBlobName(entityLogicalName, id));
+        return await blobClient.ExistsAsync();
+    }
+
     public async Task SetAuditDetailAsync(string entityLogicalName, Guid id, AuditDetailCollection auditDetailCollection)
     {
         var containerClient = blobServiceClient.GetBlobContainerClient(ContainerName);

diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/IAuditRepository.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/IAuditRepository.cs
@@ -5,5 +5,6 @@ namespace TeachingRecordSystem.Core.Services.TrsDataSync;
 public interface IAuditRepository
 {
     Task<AuditDetailCollection?> GetAuditDetailAsync(string entityLogicalName, Guid id);
+    Task<bool> HaveAuditDetailAsync(string entityLogicalName, Guid id);
     Task SetAuditDetailAsync(string entityLogicalName, Guid id, AuditDetailCollection auditDetailCollection);
 }
diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs
@@ -51,9 +51,31 @@ public static (string EntityLogicalName, string[] AttributeNames) GetEntityInfoF
         return (modelTypeSyncInfo.EntityLogicalName, modelTypeSyncInfo.AttributeNames);
     }
 
-    public async Task SyncAuditAsync(string entityLogicalName, IEnumerable<Guid> ids, CancellationToken cancellationToken = default)
+    public async Task SyncAuditAsync(
+        string entityLogicalName,
+        IEnumerable<Guid> ids,
+        bool skipIfExists,
+        CancellationToken cancellationToken = default)
     {
-        var audits = await GetAuditRecordsAsync(entityLogicalName, ids, cancellationToken);
+        var idsToSync = ids.ToList();
+
+        if (skipIfExists)
+        {
+            var existingAudits = await Task.WhenAll(
+                idsToSync.Select(async id => (Id: id, HaveAudit: await auditRepository.HaveAuditDetailAsync(entityLogicalName, id))));
+
+            foreach (var (id, _) in existingAudits.Where(t => t.HaveAudit))
+            {
+                idsToSync.Remove(id);
+            }
+        }
+
+        if (idsToSync.Count == 0)
+        {
+            return;
+        }
+
+        var audits = await GetAuditRecordsAsync(entityLogicalName, idsToSync, cancellationToken);
 
         foreach (var (id, audit) in audits)
         {
@@ -370,8 +392,8 @@ public async Task<int> SyncInductionsAsync(
     {
         if (syncAudit)
         {
-            await SyncAuditAsync(Contact.EntityLogicalName, contacts.Select(q => q.ContactId!.Value), cancellationToken);
-            await SyncAuditAsync(dfeta_induction.EntityLogicalName, entities.Select(q => q.Id), cancellationToken);
+            await SyncAuditAsync(Contact.EntityLogicalName, contacts.Select(q => q.ContactId!.Value), skipIfExists: false, cancellationToken);
+            await SyncAuditAsync(dfeta_induction.EntityLogicalName, entities.Select(q => q.Id), skipIfExists: false, cancellationToken);
         }
 
         var contactAuditDetails = await GetAuditRecordsFromAuditRepositoryAsync(Contact.EntityLogicalName, contacts.Select(q => q.ContactId!.Value), cancellationToken);

diff --git a/TeachingRecordSystem/tests/TeachingRecordSystem.TestCommon/TestableAuditRepository.cs b/TeachingRecordSystem/tests/TeachingRecordSystem.TestCommon/TestableAuditRepository.cs
@@ -11,6 +11,9 @@ public class TestableAuditRepository : IAuditRepository
     public Task<AuditDetailCollection?> GetAuditDetailAsync(string entityLogicalName, Guid id) =>
         Task.FromResult(_audits.TryGetValue((entityLogicalName, id), out var audit) ? audit : null);
 
+    public Task<bool> HaveAuditDetailAsync(string entityLogicalName, Guid id) =>
+        Task.FromResult(_audits.ContainsKey((entityLogicalName, id)));
+
     public Task SetAuditDetailAsync(string entityLogicalName, Guid id, AuditDetailCollection auditDetailCollection)
     {
         _audits[(entityLogicalName, id)] = auditDetailCollection;

diff --git a/terraform/aks/config/production.tfvars.json b/terraform/aks/config/production.tfvars.json
@@ -14,7 +14,7 @@
   "authz_replicas": 2,
   "ui_replicas": 2,
   "worker_replicas": 2,
-  "worker_max_memory": "4Gi",
+  "worker_max_memory": "8Gi",
   "postgres_flexible_server_sku": "GP_Standard_D2ds_v4",
   "postgres_azure_storage_mb": 65536,
   "postgres_enable_high_availability": true,