feat: add retries for hard failures

.github/README.md

@@ -118,14 +118,17 @@ module.exports = {
   // Additional headers for network requests, undefined by default
   networkHeaders: {
     "User-Agent": "Custom",
-    "x-ms-blob-type": "BlockBlob"
+    "x-ms-blob-type": "BlockBlob",
   },
   e2e: {
     batchSize: 3, // orchestration batch size for e2e tests (Currents only, read below)
   },
   component: {
     batchSize: 5, // orchestration batch size for component tests (Currents only, read below)
   },
+  retry: {
+    hardFailureMaxRetries: 2, // max retries for hard Cypress failures, a hard failure is when Cyrpess crashes and doesn't report back any results (see https://docs.cypress.io/guides/guides/module-api#Handling-errors)
+  },
 };
 ```
 
@@ -162,7 +165,7 @@ The configuration variables will resolve as follows:
 
 ## Batched Orchestration
 
-This package uses its own orchestration and reporting protocol that is independent of cypress native implementation. The new [orchestration protocol]([https://currents.dev/readme/integration-with-cypress/cypress-cloud#batched-orchestration](https://currents.dev/readme/integration-with-cypress/cypress-cloud/batched-orchestration)) uses cypress in "offline" mode and allows batching multiple spec files for better efficiency. You can adjust the batching configuration in `currents.config.js` and use different values for e2e and component tests.
+This package uses its own orchestration and reporting protocol that is independent of cypress native implementation. The new [orchestration protocol](<[https://currents.dev/readme/integration-with-cypress/cypress-cloud#batched-orchestration](https://currents.dev/readme/integration-with-cypress/cypress-cloud/batched-orchestration)>) uses cypress in "offline" mode and allows batching multiple spec files for better efficiency. You can adjust the batching configuration in `currents.config.js` and use different values for e2e and component tests.
 
 ## API
 

examples/webapp/currents.config.js

@@ -9,5 +9,8 @@ module.exports = {
     ? "Ij0RfK"
     : "Ij0RfK",
   // cloudServiceUrl: "http://localhost:1234",
-  userAgent: "custom",
+  // userAgent: "custom",
+  retry: {
+    hardFailureMaxRetries: 2,
+  },
 };

packages/cypress-cloud/lib/config/config.ts

@@ -14,13 +14,18 @@ export type E2EConfig = {
 export type ComponentConfig = {
   batchSize: number;
 };
+
+type RetryConfig = {
+  hardFailureMaxRetries: number;
+};
 export type CurrentsConfig = {
   projectId?: string;
   recordKey?: string;
   cloudServiceUrl: string;
   e2e: E2EConfig;
   component: ComponentConfig;
   networkHeaders?: Record<string, string>;
+  retry?: RetryConfig;
 };
 
 let _config: CurrentsConfig | null = null;

packages/cypress-cloud/lib/cypress/cypress.ts

@@ -6,7 +6,7 @@ import {
 } from "cypress-cloud/types";
 import Debug from "debug";
 import _ from "lodash";
-import { getCypressRunAPIParams } from "../config";
+import { getCurrentsConfig, getCypressRunAPIParams } from "../config";
 import { safe } from "../lang";
 import { warn } from "../log";
 import { getWSSPort } from "../ws";
@@ -53,8 +53,39 @@ export async function runSpecFile(
     },
     spec,
   };
+
   debug("running cypress with options %o", options);
-  const result = await cypress.run(options);
+  let result = await cypress.run(options);
+
+  let retries = 0;
+  const currentsConfig = await getCurrentsConfig();
+
+  while (
+    currentsConfig.retry &&
+    retries < (currentsConfig.retry.hardFailureMaxRetries ?? 0) &&
+    result.status === "failed"
+  ) {
+    warn("Cypress runner failed with message: %s", result.message);
+    warn(
+      "[retry %d/%d] Retrying the following spec files because of retry config: %s",
+      retries + 1,
+      currentsConfig.retry.hardFailureMaxRetries,
+      spec
+        .split(",")
+        .map((i) => `\n - ${i}`)
+        .join("")
+    );
+    result = await cypress.run(options);
+    retries++;
+  }
+
+  if (currentsConfig.retry && retries > 0) {
+    warn(
+      "Exhausted max retries: %d/%d",
+      retries,
+      currentsConfig.retry.hardFailureMaxRetries
+    );
+  }
 
   if (result.status === "failed") {
     warn('Cypress runner failed with message: "%s"', result.message);
@@ -66,6 +97,7 @@ export async function runSpecFile(
         .join("")
     );
   }
+
   debug("cypress run result %o", result);
   return result;
 }