diff --git a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs index fe5ed2be9..935ff03be 100644 --- a/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs +++ b/TeachingRecordSystem/src/TeachingRecordSystem.Core/Services/TrsDataSync/TrsDataSyncHelper.cs @@ -822,7 +822,30 @@ private async Task> GetAuditRec IEnumerable ids, CancellationToken cancellationToken) { - return (IsFakeXrm ? new Dictionary() : (await Task.WhenAll(ids + if (IsFakeXrm) + { + return new Dictionary(); + } + + // Throttle the amount of concurrent requests + using var requestThrottle = new SemaphoreSlim(20, 20); + + // Keep track of the last seen 'retry-after' value + var retryDelayUpdateLock = new object(); + var retryDelay = Task.Delay(0, cancellationToken); + + void UpdateRetryDelay(TimeSpan ts) + { + Task oldDelay; + lock (retryDelayUpdateLock) + { + oldDelay = retryDelay; + retryDelay = Task.Delay(ts, cancellationToken); + } + oldDelay.Dispose(); + } + + var audits = (await Task.WhenAll(ids .Chunk(MaxAuditRequestsPerBatch) .Select(async chunk => { @@ -842,6 +865,8 @@ private async Task> GetAuditRec ExecuteMultipleResponse response; while (true) { + await retryDelay; + await requestThrottle.WaitAsync(cancellationToken); try { response = (ExecuteMultipleResponse)await organizationService.ExecuteAsync(request, cancellationToken); @@ -849,9 +874,13 @@ private async Task> GetAuditRec catch (FaultException fex) when (fex.IsCrmRateLimitException(out var retryAfter)) { logger.LogWarning("Hit CRM service limits getting {entityLogicalName} audit records; Fault exception. Retrying after {retryAfter} seconds.", entityLogicalName, retryAfter.TotalSeconds); - await Task.Delay(retryAfter, cancellationToken); + UpdateRetryDelay(retryAfter); continue; } + finally + { + requestThrottle.Release(); + } if (response.IsFaulted) { @@ -860,13 +889,13 @@ private async Task> GetAuditRec if (firstFault.IsCrmRateLimitFault(out var retryAfter)) { logger.LogWarning("Hit CRM service limits getting {entityLogicalName} audit records; CRM rate limit fault. Retrying after {retryAfter} seconds.", entityLogicalName, retryAfter.TotalSeconds); - await Task.Delay(retryAfter, cancellationToken); + UpdateRetryDelay(retryAfter); continue; } else if (firstFault.Message.Contains("The HTTP status code of the response was not expected (429)")) { logger.LogWarning("Hit CRM service limits getting {entityLogicalName} audit records; 429 too many requests", entityLogicalName); - await Task.Delay(TimeSpan.FromMinutes(2), cancellationToken); + UpdateRetryDelay(TimeSpan.FromMinutes(2)); continue; } @@ -881,7 +910,11 @@ private async Task> GetAuditRec (r, e) => (Id: e, ((RetrieveRecordChangeHistoryResponse)r.Response).AuditDetailCollection)); }))) .SelectMany(b => b) - .ToDictionary(t => t.Id, t => t.AuditDetailCollection)); + .ToDictionary(t => t.Id, t => t.AuditDetailCollection); + + retryDelay.Dispose(); + + return audits; } private async Task> GetAuditRecordsFromAuditRepositoryAsync(