Retries calls to retrieve-release-history (#392)

* Adds ability to retry with an exponential delay to the `CommandInvoker` class * Makes some improvements to the log output when a command fails. * Sets the max number of retries for `retrieve-release-history` to 3 Fixes #391.
corgibytes · Nov 2, 2022 · 6b8a7a1 · 6b8a7a1
2 parents 456b003 + a59f5b6
commit 6b8a7a1
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 13 deletions.
diff --git a/.codeclimate.yml b/.codeclimate.yml
@@ -7,7 +7,7 @@ exclude_patterns:
 checks:
   method-lines:
     config:
-      threshold: 60
+      threshold: 65
   argument-count:
     enabled: true
     config:

diff --git a/Corgibytes.Freshli.Cli.Test/Services/AgentReaderTest.cs b/Corgibytes.Freshli.Cli.Test/Services/AgentReaderTest.cs
@@ -61,7 +61,7 @@ public void RetrieveReleaseHistoryWritesToCache()
             $"{_gammaPackage.PackageUrl.Version}\t{_gammaPackage.ReleasedAt:yyyy'-'MM'-'dd'T'HH':'mm':'ssK}\n";
 
         _commandInvoker.Setup(mock => mock.Run(agentExecutable,
-            $"retrieve-release-history {_packageUrl.FormatWithoutVersion()}", ".")).Returns(commandResponse);
+            $"retrieve-release-history {_packageUrl.FormatWithoutVersion()}", ".", 3)).Returns(commandResponse);
 
         var initialCachedPackages = new List<CachedPackage>();
 

diff --git a/Corgibytes.Freshli.Cli/Functionality/CommandInvoker.cs b/Corgibytes.Freshli.Cli/Functionality/CommandInvoker.cs
@@ -3,6 +3,7 @@
 using System.Text;
 using CliWrap;
 using Microsoft.Extensions.Logging;
+using Polly;
 
 namespace Corgibytes.Freshli.Cli.Functionality;
 
@@ -12,7 +13,7 @@ public class CommandInvoker : ICommandInvoker
 
     public CommandInvoker(ILogger<CommandInvoker>? logger = null) => _logger = logger;
 
-    public string Run(string executable, string arguments, string workingDirectory)
+    public string Run(string executable, string arguments, string workingDirectory, int maxRetries = 0)
     {
         var stdOutBuffer = new StringBuilder();
         var stdErrBuffer = new StringBuilder();
@@ -34,21 +35,53 @@ public string Run(string executable, string arguments, string workingDirectory)
 
         try
         {
-            using var task = command.ExecuteAsync().Task;
-            task.Wait();
+            Policy
+                .Handle<Exception>()
+                .WaitAndRetry(
+                    maxRetries,
+                    // delay progression: 2s, 4s, 8s, 16s, 32s
+                    retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
+                    onRetry: (_, retryCount) =>
+                {
+                    _logger?.LogWarning(
+                        "Command failed. Retry {RetryCount} of {MaxRetries}. Command: {Executable}; " +
+                        "Args: {Arguments}; WorkingDir: {WorkingDir}",
+                        retryCount,
+                        maxRetries,
+                        executable,
+                        arguments,
+                        workingDirectory
+                    );
+                })
+                .Execute(() =>
+                {
+                    using var task = command.ExecuteAsync().Task;
+                    task.Wait();
+                });
         }
-        catch (AggregateException error)
+        catch (Exception error)
         {
-            _logger?.LogError("{Exception}", error.ToString());
-            foreach (var innerError in error.InnerExceptions)
+            _logger?.LogError(
+                "Command failed. No more retries. Command: {Executable}; Args: {Arguments}; WorkingDir: {WorkingDir}; " +
+                "Exception: {Exception}",
+                executable,
+                arguments,
+                workingDirectory,
+                error.ToString()
+            );
+
+            if (error is AggregateException aggregate)
             {
-                _logger?.LogError("{InnerException}", innerError.ToString());
+                foreach (var innerError in aggregate.InnerExceptions)
+                {
+                    _logger?.LogError("{InnerException}", innerError.ToString());
+                }
             }
 
             _logger?.LogError("{StdOutBuffer}", stdOutBuffer.ToString());
             _logger?.LogError("{StdErrBuffer}", stdErrBuffer.ToString());
 
-            throw new IOException(stdErrBuffer.ToString());
+            throw;
         }
 
         return stdOutBuffer.ToString();

diff --git a/Corgibytes.Freshli.Cli/Functionality/ICommandInvoker.cs b/Corgibytes.Freshli.Cli/Functionality/ICommandInvoker.cs
@@ -2,5 +2,5 @@ namespace Corgibytes.Freshli.Cli.Functionality;
 
 public interface ICommandInvoker
 {
-    string Run(string executable, string arguments, string workingDirectory);
+    string Run(string executable, string arguments, string workingDirectory, int maxRetries = 0);
 }
diff --git a/Corgibytes.Freshli.Cli/Services/AgentReader.cs b/Corgibytes.Freshli.Cli/Services/AgentReader.cs
@@ -38,9 +38,9 @@ public List<Package> RetrieveReleaseHistory(PackageURL packageUrl)
         try
         {
             packageUrlsWithDate = _commandInvoker.Run(AgentExecutablePath,
-                $"retrieve-release-history {packageUrl.FormatWithoutVersion()}", ".");
+                $"retrieve-release-history {packageUrl.FormatWithoutVersion()}", ".", 3);
         }
-        catch (IOException)
+        catch
         {
             return packages;
         }