Add Vidarr retry failed provision-out command

Allows the new `/api/retry-provision-out` endpoint to be accessed via action command.
oicr-gsi · Feb 26, 2024 · 0ffb2f1 · 0ffb2f1
1 parent 67816aa
commit 0ffb2f1
Show file tree

Hide file tree

Showing 9 changed files with 186 additions and 64 deletions.
diff --git a/changes/add_vidarr_retry_provision_out.md b/changes/add_vidarr_retry_provision_out.md
@@ -0,0 +1 @@
+* Add Vidarr _retry failed provision-out_ command
diff --git a/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/AvailableCommands.java b/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/AvailableCommands.java
@@ -0,0 +1,98 @@
+package ca.on.oicr.gsi.shesmu.vidarr;
+
+import ca.on.oicr.gsi.shesmu.plugin.FrontEndIcon;
+import ca.on.oicr.gsi.shesmu.plugin.action.ActionCommand;
+import ca.on.oicr.gsi.shesmu.plugin.action.ActionCommand.Preference;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+public enum AvailableCommands {
+  RESET_ONLY(false) {
+    @Override
+    public Stream<ActionCommand<?>> commands() {
+      return Stream.of(RESET);
+    }
+  },
+  CAN_RETRY(true) {
+    @Override
+    public Stream<ActionCommand<?>> commands() {
+      return Stream.of(DELETE, RETRY_PROVISION_OUT, RESET);
+    }
+  },
+  CAN_REATTEMPT(true) {
+    @Override
+    public Stream<ActionCommand<?>> commands() {
+      return Stream.of(DELETE, REATTEMPT, RESET);
+    }
+  };
+  static final ActionCommand<SubmitAction> DELETE =
+      new ActionCommand<>(
+          SubmitAction.class,
+          "VIDARR-DELETE",
+          FrontEndIcon.PLUG,
+          "Delete and Purge",
+          Preference.ALLOW_BULK,
+          Preference.PROMPT,
+          Preference.ANNOY_USER) {
+        @Override
+        protected Response execute(SubmitAction action, Optional<String> user) {
+          return action.owner.get().url().map(action.state::delete).orElse(false)
+              ? Response.PURGE
+              : Response.IGNORED;
+        }
+      };
+  static final ActionCommand<SubmitAction> REATTEMPT =
+      new ActionCommand<>(
+          SubmitAction.class,
+          "VIDARR-REATTEMPT",
+          FrontEndIcon.ARROW_REPEAT,
+          "Reattempt Failed Workflow",
+          Preference.PROMPT,
+          Preference.ALLOW_BULK) {
+        @Override
+        protected Response execute(SubmitAction action, Optional<String> user) {
+          final var result = action.state.reattempt();
+          result.ifPresent(s -> action.state = s);
+          return result.isPresent() ? Response.RESET : Response.IGNORED;
+        }
+      };
+  static final ActionCommand<SubmitAction> RESET =
+      new ActionCommand<>(
+          SubmitAction.class,
+          "VIDARR-RESET",
+          FrontEndIcon.PLUG,
+          "Search Vidarr Again",
+          Preference.ALLOW_BULK) {
+        @Override
+        protected Response execute(SubmitAction action, Optional<String> user) {
+          action.state = new RunStateAttemptSubmit();
+          return Response.RESET;
+        }
+      };
+  static final ActionCommand<SubmitAction> RETRY_PROVISION_OUT =
+      new ActionCommand<>(
+          SubmitAction.class,
+          "VIDARR-RETRY-PROVISION-OUT",
+          FrontEndIcon.ARROW_RIGHT_SQUARE_FILL,
+          "Retry Provision Out",
+          Preference.ALLOW_BULK,
+          Preference.PROMPT) {
+        @Override
+        protected Response execute(SubmitAction action, Optional<String> user) {
+          return action.owner.get().url().map(action.state::retry).orElse(false)
+              ? Response.ACCEPTED
+              : Response.IGNORED;
+        }
+      };
+  private final boolean canRetry;
+
+  AvailableCommands(boolean canRetry) {
+    this.canRetry = canRetry;
+  }
+
+  public boolean canRetry() {
+    return canRetry;
+  }
+
+  public abstract Stream<ActionCommand<?>> commands();
+}
diff --git a/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunState.java b/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunState.java
@@ -43,7 +43,7 @@ public RunState nextState() {
     }
   }
 
-  public abstract boolean canReattempt();
+  public abstract AvailableCommands commands();
 
   public abstract boolean delete(URI vidarrUrl);
 
@@ -59,6 +59,8 @@ public abstract PerformResult perform(
 
   public abstract Optional<RunState> reattempt();
 
+  public abstract boolean retry(URI vidarrUrl);
+
   public abstract long retryMinutes();
 
   public abstract Stream<String> tags();

diff --git a/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateAttemptSubmit.java b/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateAttemptSubmit.java
@@ -36,8 +36,8 @@ public RunStateAttemptSubmit(int attempt) {
   }
 
   @Override
-  public boolean canReattempt() {
-    return false;
+  public AvailableCommands commands() {
+    return AvailableCommands.RESET_ONLY;
   }
 
   @Override
@@ -121,6 +121,11 @@ public Optional<RunState> reattempt() {
     return Optional.empty();
   }
 
+  @Override
+  public boolean retry(URI vidarrUrl) {
+    return false;
+  }
+
   @Override
   public long retryMinutes() {
     return retryMinutes;

diff --git a/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateConflicted.java b/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateConflicted.java
@@ -26,8 +26,8 @@ public RunStateConflicted(List<String> ids) {
   }
 
   @Override
-  public boolean canReattempt() {
-    return false;
+  public AvailableCommands commands() {
+    return AvailableCommands.RESET_ONLY;
   }
 
   @Override
@@ -59,6 +59,11 @@ public Optional<RunState> reattempt() {
     return Optional.empty();
   }
 
+  @Override
+  public boolean retry(URI vidarrUrl) {
+    return false;
+  }
+
   @Override
   public long retryMinutes() {
     return 60;

diff --git a/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateDead.java b/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateDead.java
@@ -14,8 +14,8 @@
 final class RunStateDead extends RunState {
 
   @Override
-  public boolean canReattempt() {
-    return false;
+  public AvailableCommands commands() {
+    return AvailableCommands.RESET_ONLY;
   }
 
   @Override
@@ -43,6 +43,11 @@ public Optional<RunState> reattempt() {
     return Optional.empty();
   }
 
+  @Override
+  public boolean retry(URI vidarrUrl) {
+    return false;
+  }
+
   @Override
   public long retryMinutes() {
     return 86400;

diff --git a/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateMissing.java b/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateMissing.java
@@ -34,8 +34,8 @@ public RunStateMissing(String id, List<ExternalKey> keys) {
   }
 
   @Override
-  public boolean canReattempt() {
-    return false;
+  public AvailableCommands commands() {
+    return AvailableCommands.RESET_ONLY;
   }
 
   @Override
@@ -67,6 +67,11 @@ public Optional<RunState> reattempt() {
     return Optional.empty();
   }
 
+  @Override
+  public boolean retry(URI vidarrUrl) {
+    return false;
+  }
+
   @Override
   public long retryMinutes() {
     return 60;

diff --git a/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateMonitor.java b/plugin-vidarr/src/main/java/ca/on/oicr/gsi/shesmu/vidarr/RunStateMonitor.java
@@ -1,17 +1,22 @@
 package ca.on.oicr.gsi.shesmu.vidarr;
 
+import static ca.on.oicr.gsi.shesmu.vidarr.VidarrPlugin.MAPPER;
+
 import ca.on.oicr.gsi.shesmu.plugin.action.ActionState;
 import ca.on.oicr.gsi.vidarr.JsonBodyHandler;
+import ca.on.oicr.gsi.vidarr.api.RetryProvisionOutRequest;
 import ca.on.oicr.gsi.vidarr.api.SubmitWorkflowRequest;
 import ca.on.oicr.gsi.vidarr.api.WorkflowRunStatusResponse;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.io.IOException;
 import java.net.URI;
 import java.net.http.HttpRequest;
+import java.net.http.HttpRequest.BodyPublishers;
 import java.net.http.HttpResponse.BodyHandlers;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Stream;
@@ -56,7 +61,7 @@ public static PerformResult create(URI vidarrUrl, String id)
     final var response =
         VidarrPlugin.CLIENT.send(
             HttpRequest.newBuilder(workflowRunUrl).GET().build(),
-            new JsonBodyHandler<>(VidarrPlugin.MAPPER, WorkflowRunStatusResponse.class));
+            new JsonBodyHandler<>(MAPPER, WorkflowRunStatusResponse.class));
     if (response.statusCode() == 200) {
       final var result = response.body().get();
       final var status = actionStatusForWorkflowRun(result);
@@ -88,20 +93,35 @@ public RunStateMonitor(String workflowRunUrl, WorkflowRunStatusResponse status)
   }
 
   @Override
-  public boolean canReattempt() {
+  public AvailableCommands commands() {
+    // Failed during provision out may be retried
+    if (status.getEnginePhase() != null
+        && (status.getEnginePhase().equals("FAILED")
+            && status.getOperations() != null
+            && status.getOperations().stream()
+                .anyMatch(
+                    operation ->
+                        operation.getEnginePhase().equals("PROVISION_OUT")
+                            && operation.getStatus().equals("FAILED")))) {
+      return AvailableCommands.CAN_RETRY;
+    } else if (
     // Failed operations may reattempt
-    return status.getOperationStatus().equals("FAILED")
+    status.getOperationStatus().equals("FAILED")
         // as may engine failures
         || (status.getEnginePhase() != null
             && (status.getEnginePhase().equals("WAITING_FOR_RESOURCES")
                 || status.getEnginePhase().equals("FAILED")))
         // Succeeded actions may not but otherwise-empty engine status probably
-        || (status.getEnginePhase() == null && !status.getOperationStatus().equals("N/A"));
+        || (status.getEnginePhase() == null && !status.getOperationStatus().equals("N/A"))) {
+      return AvailableCommands.CAN_REATTEMPT;
+    } else {
+      return AvailableCommands.RESET_ONLY;
+    }
   }
 
   @Override
   public boolean delete(URI vidarrUrl) {
-    if (canReattempt()) {
+    if (commands().canRetry()) {
       try {
         final var response =
             VidarrPlugin.CLIENT.send(
@@ -119,6 +139,32 @@ public boolean delete(URI vidarrUrl) {
     }
   }
 
+  @Override
+  public boolean retry(URI vidarrUrl) {
+    if (commands().canRetry()) {
+      try {
+        final var request = new RetryProvisionOutRequest();
+        request.setWorkflowRunIds(List.of(status.getId()));
+        final var response =
+            VidarrPlugin.CLIENT.send(
+                HttpRequest.newBuilder(vidarrUrl.resolve("/api/retry-provision-out"))
+                    .POST(BodyPublishers.ofByteArray(MAPPER.writeValueAsBytes(request)))
+                    .build(),
+                new JsonBodyHandler<>(MAPPER, String[].class));
+        if (response.statusCode() != 200) {
+          return false;
+        }
+        final var ids = response.body().get();
+        return Arrays.asList(ids).contains(status.getId());
+      } catch (InterruptedException | IOException e) {
+        e.printStackTrace();
+        return false;
+      }
+    } else {
+      return false;
+    }
+  }
+
   @Override
   public Optional<Instant> externalTimestamp() {
     return Optional.of(status.getModified().toInstant());
@@ -137,7 +183,7 @@ public PerformResult perform(
 
   @Override
   public Optional<RunState> reattempt() {
-    return canReattempt()
+    return commands().canRetry()
         ? Optional.of(new RunStateAttemptSubmit(status.getAttempt() + 1))
         : Optional.empty();
   }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		* Add Vidarr _retry failed provision-out_ command