Added logic to fail all steps involved in an upload in case the uploa…

…d fails
JupiterOne · Oct 9, 2023 · 753c85c · 753c85c
1 parent 4f9eaaf
commit 753c85c
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 6 deletions.
diff --git a/packages/integration-sdk-runtime/src/execution/dependencyGraph.ts b/packages/integration-sdk-runtime/src/execution/dependencyGraph.ts
@@ -453,6 +453,9 @@ export function executeStepDependencyGraph<
       if (createStepGraphObjectDataUploader) {
         uploader = createStepGraphObjectDataUploader(lastStepId);
       }
+      const stepsInvolvedInUpload = graphObjectStore.getStepsStored
+        ? graphObjectStore.getStepsStored()
+        : [];
       await graphObjectStore.flush(
         async (entities) =>
           entities.length
@@ -476,12 +479,13 @@ export function executeStepDependencyGraph<
           name: IntegrationErrorEventName.UnexpectedError,
           description: 'Upload to persister failed',
         });
-
-        executionContext.logger.stepFailure(
-          workingGraph.getNodeData(lastStepId),
-          err,
-        );
-        stepResultsMap[lastStepId] = StepResultStatus.FAILURE;
+        for (const stepId of stepsInvolvedInUpload) {
+          executionContext.logger.stepFailure(
+            workingGraph.getNodeData(stepId),
+            err,
+          );
+          stepResultsMap[stepId] = StepResultStatus.FAILURE;
+        }
       }
     }
 

diff --git a/...egration-sdk-runtime/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts b/...egration-sdk-runtime/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts
@@ -152,6 +152,9 @@ export class FileSystemGraphObjectStore implements GraphObjectStore {
         integrationStepsToGraphObjectIndexMetadataMap(params.integrationSteps);
     }
   }
+  getStepsStored() {
+    return this.localGraphObjectStore.getStepsStored();
+  }
 
   async addEntities(
     stepId: string,

diff --git a/packages/integration-sdk-runtime/src/storage/memory.ts b/packages/integration-sdk-runtime/src/storage/memory.ts
@@ -323,4 +323,22 @@ export class InMemoryGraphObjectStore implements GraphObjectStore {
       0,
     );
   }
+
+  getStepsStored(): string[] {
+    const stepIds: string[] = [];
+
+    for (const graphObjectData of this.relationshipKeyToRelationshipMap.values()) {
+      const { stepId } = graphObjectData;
+      if (!stepIds.includes(stepId)) {
+        stepIds.push(stepId);
+      }
+    }
+    for (const graphObjectData of this.entityKeyToEntityMap.values()) {
+      const { stepId } = graphObjectData;
+      if (!stepIds.includes(stepId)) {
+        stepIds.push(stepId);
+      }
+    }
+    return stepIds;
+  }
 }