Merge pull request #1081 from solidify/bugfix/defer-execution-items

Add option to defer ExecutionItems until later.
solidify · Dec 18, 2024 · 4bf2f4f · 4bf2f4f
2 parents 4785d32 + e09b284
commit 4bf2f4f
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 3 deletions.
diff --git a/src/WorkItemMigrator/Migration.Common.Log/AttachmentNotFoundException.cs b/src/WorkItemMigrator/Migration.Common.Log/AttachmentNotFoundException.cs
@@ -0,0 +1,21 @@
+using System;
+using System.Runtime.Serialization;
+
+namespace Migration.Common.Log
+{
+    [Serializable]
+    public class AttachmentNotFoundException : Exception
+    {
+        protected AttachmentNotFoundException(SerializationInfo serializationInfo, StreamingContext streamingContext) : base(serializationInfo, streamingContext)
+        {
+
+        }
+
+        public AttachmentNotFoundException(string reason)
+        {
+            Reason = reason;
+        }
+
+        public string Reason { get; private set; }
+    }
+}
diff --git a/src/WorkItemMigrator/WorkItemImport/Agent.cs b/src/WorkItemMigrator/WorkItemImport/Agent.cs
@@ -126,6 +126,10 @@ public bool ImportRevision(WiRevision rev, WorkItem wi, Settings settings)
                     {
                         _witClientUtils.CorrectDescription(wi, _context.GetItem(rev.ParentOriginId), rev, _context.Journal.IsAttachmentMigrated);
                     }
+                    catch (AttachmentNotFoundException)
+                    {
+                        throw;
+                    }
                     catch (Exception ex)
                     {
                         Logger.Log(ex, $"Failed to correct description for '{wi.Id}', rev '{rev}'.");
@@ -152,6 +156,10 @@ public bool ImportRevision(WiRevision rev, WorkItem wi, Settings settings)
                                     _context.Journal.IsAttachmentMigrated
                                 );
                             }
+                            catch (AttachmentNotFoundException)
+                            {
+                                throw;
+                            }
                             catch (Exception ex)
                             {
                                 Logger.Log(ex, $"Failed to correct description for '{wi.Id}', rev '{rev}'.");
@@ -190,6 +198,10 @@ public bool ImportRevision(WiRevision rev, WorkItem wi, Settings settings)
             {
                 throw;
             }
+            catch (AttachmentNotFoundException)
+            {
+                throw;
+            }
             catch (FileNotFoundException ex)
             {
                 Logger.Log(LogLevel.Error, ex.Message);

diff --git a/src/WorkItemMigrator/WorkItemImport/ExecutionPlan.cs b/src/WorkItemMigrator/WorkItemImport/ExecutionPlan.cs
@@ -12,6 +12,7 @@ public class ExecutionItem
             public int WiId { get; set; } = -1;
             public WiRevision Revision { get; set; }
             public string WiType { get; internal set; }
+            public bool isDeferred { get; set; }
 
             public override string ToString()
             {
@@ -48,5 +49,17 @@ public bool TryPop(out ExecutionItem nextItem)
             else
                 return false;
         }
+
+        public bool TryPeek(out ExecutionItem nextItem)
+        {
+            nextItem = null;
+            if (ReferenceQueue.Count > 0)
+            {
+                nextItem = TransformToExecutionItem(ReferenceQueue.Peek());
+                return true;
+            }
+            else
+                return false;
+        }
     }
 }
diff --git a/src/WorkItemMigrator/WorkItemImport/ImportCommandLine.cs b/src/WorkItemMigrator/WorkItemImport/ImportCommandLine.cs
@@ -16,6 +16,7 @@ public class ImportCommandLine
     {
         private CommandLineApplication commandLineApplication;
         private string[] args;
+        private List<ExecutionPlan.ExecutionItem> deferredExecutionItems = new List<ExecutionPlan.ExecutionItem>();
 
         public ImportCommandLine(params string[] args)
         {
@@ -112,10 +113,18 @@ private bool ExecuteMigration(CommandOption token, CommandOption url, CommandOpt
 
                 BeginSession(configFileName, config, forceFresh, agent, itemCount, revisionCount);
 
-                while (plan.TryPop(out ExecutionPlan.ExecutionItem executionItem))
+                while (plan.ReferenceQueue.Count > 0 || deferredExecutionItems.Count > 0)
                 {
+                    ExecutionPlan.ExecutionItem executionItem = null;
                     try
                     {
+                        executionItem = GetDeferredItemIfAvailable(plan, executionItem);
+
+                        if (executionItem == null)
+                        {
+                            plan.TryPop(out executionItem);
+                        }
+
                         if (!forceFresh && context.Journal.IsItemMigrated(executionItem.OriginId, executionItem.Revision.Index))
                         {
                             continue;
@@ -156,7 +165,14 @@ private bool ExecuteMigration(CommandOption token, CommandOption url, CommandOpt
                             continue;
                         }
 
-                        agent.ImportRevision(executionItem.Revision, wi, settings);
+                        try
+                        {
+                            agent.ImportRevision(executionItem.Revision, wi, settings);
+                        }
+                        catch (AttachmentNotFoundException)
+                        {
+                            importedItems = DeferItem(importedItems, executionItem);
+                        }
 
                         // Artifical wait (optional) to avoid throttling for ADO Services
                         if (config.SleepTimeBetweenRevisionImportMilliseconds > 0)
@@ -199,6 +215,44 @@ private bool ExecuteMigration(CommandOption token, CommandOption url, CommandOpt
             return succeeded;
         }
 
+        private int DeferItem(int importedItems, ExecutionPlan.ExecutionItem executionItem)
+        {
+            if (!executionItem.isDeferred)
+            {
+                executionItem.Revision.Time = executionItem.Revision.Time.AddMinutes(5);
+                executionItem.isDeferred = true;
+                deferredExecutionItems.Add(executionItem);
+                importedItems--;
+            }
+
+            return importedItems;
+        }
+
+        private ExecutionPlan.ExecutionItem GetDeferredItemIfAvailable(ExecutionPlan plan, ExecutionPlan.ExecutionItem executionItem)
+        {
+            foreach (var executionItemDeferred in deferredExecutionItems)
+            {
+                if (plan.TryPeek(out var nextItem))
+                {
+                    if (executionItemDeferred.Revision.Time < nextItem.Revision.Time)
+                    {
+                        executionItem = executionItemDeferred;
+                        deferredExecutionItems.Remove(executionItem);
+                        executionItem.Revision.Time = nextItem.Revision.Time.AddMilliseconds(-5);
+                        break;
+                    }
+                }
+                else
+                {
+                    executionItem = executionItemDeferred;
+                    deferredExecutionItems.Remove(executionItem);
+                    break;
+                }
+            }
+
+            return executionItem;
+        }
+
         private static void BeginSession(string configFile, ConfigJson config, bool force, Agent agent, int itemsCount, int revisionCount)
         {
             var toolVersion = VersionInfo.GetVersionInfo();

diff --git a/src/WorkItemMigrator/WorkItemImport/WitClient/WitClientUtils.cs b/src/WorkItemMigrator/WorkItemImport/WitClient/WitClientUtils.cs
@@ -771,7 +771,10 @@ private void CorrectImagePath(WorkItem wi, WiItem wiItem, WiRevision rev, ref st
                         isUpdated = true;
                     }
                     else
-                        Logger.Log(LogLevel.Warning, $"Attachment '{att}' referenced in text but is missing from work item {wiItem.OriginId}/{wi.Id}.");
+                    {
+                        Logger.Log(LogLevel.Warning, $"Attachment '{att}' referenced in text but is missing from work item {wiItem.OriginId}/{wi.Id}. This revision will be deferred until later.");
+                        throw new AttachmentNotFoundException("Attachment not found on work item");
+                    }
                 }
             }
             if (isUpdated)