Merge remote-tracking branch 'origin/master' into more-engine-code-sh…

…oveling

.github/workflows/ci-postgres-mysql.yml

@@ -42,7 +42,7 @@ jobs:
         run: pnpm build:backend
 
       - name: Cache build artifacts
-        uses: actions/cache/save@v4.0.0
+        uses: actions/cache/save@v4.2.0
         with:
           path: ./packages/**/dist
           key: ${{ github.sha }}:db-tests
@@ -73,7 +73,7 @@ jobs:
         uses: rharkor/[email protected]
 
       - name: Restore cached build artifacts
-        uses: actions/cache/restore@v4.0.0
+        uses: actions/cache/restore@v4.2.0
         with:
           path: ./packages/**/dist
           key: ${{ github.sha }}:db-tests
@@ -107,7 +107,7 @@ jobs:
         uses: rharkor/[email protected]
 
       - name: Restore cached build artifacts
-        uses: actions/cache/restore@v4.0.0
+        uses: actions/cache/restore@v4.2.0
         with:
           path: ./packages/**/dist
           key: ${{ github.sha }}:db-tests
@@ -149,7 +149,7 @@ jobs:
         uses: rharkor/[email protected]
 
       - name: Restore cached build artifacts
-        uses: actions/cache/restore@v4.0.0
+        uses: actions/cache/restore@v4.2.0
         with:
           path: ./packages/**/dist
           key: ${{ github.sha }}:db-tests

.github/workflows/e2e-reusable.yml

@@ -99,7 +99,7 @@ jobs:
         run: pnpm cypress:install
 
       - name: Cache build artifacts
-        uses: actions/cache/save@v4.0.0
+        uses: actions/cache/save@v4.2.0
         with:
           path: |
             /github/home/.cache
@@ -133,7 +133,7 @@ jobs:
       - uses: pnpm/[email protected]
 
       - name: Restore cached pnpm modules
-        uses: actions/cache/restore@v4.0.0
+        uses: actions/cache/restore@v4.2.0
         with:
           path: |
             /github/home/.cache

.github/workflows/release-publish.yml

@@ -43,7 +43,7 @@ jobs:
         run: pnpm build
 
       - name: Cache build artifacts
-        uses: actions/cache/save@v4.0.0
+        uses: actions/cache/save@v4.2.0
         with:
           path: ./packages/**/dist
           key: ${{ github.sha }}-release:build
@@ -144,7 +144,7 @@ jobs:
     steps:
       - uses: actions/[email protected]
       - name: Restore cached build artifacts
-        uses: actions/cache/restore@v4.0.0
+        uses: actions/cache/restore@v4.2.0
         with:
           path: ./packages/**/dist
           key: ${{ github.sha }}-release:build

.github/workflows/test-workflows.yml

@@ -41,7 +41,7 @@ jobs:
         run: pnpm build:backend
 
       - name: Cache build artifacts
-        uses: actions/cache/save@v4.0.0
+        uses: actions/cache/save@v4.2.0
         with:
           path: ./packages/**/dist
           key: ${{ github.sha }}:workflow-tests
@@ -69,7 +69,7 @@ jobs:
         uses: rharkor/[email protected]
 
       - name: Restore cached build artifacts
-        uses: actions/cache/restore@v4.0.0
+        uses: actions/cache/restore@v4.2.0
         with:
           path: ./packages/**/dist
           key: ${{ github.sha }}:workflow-tests

cypress/e2e/233-AI-switch-to-logs-on-error.cy.ts

@@ -1,4 +1,4 @@
-import type { ExecutionError } from 'n8n-workflow/src';
+import type { ExecutionError } from 'n8n-workflow';
 
 import {
 	closeManualChatModal,

packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/test/InformationExtraction.node.test.ts

@@ -1,7 +1,7 @@
 import type { BaseLanguageModel } from '@langchain/core/language_models/base';
 import { FakeLLM, FakeListChatModel } from '@langchain/core/utils/testing';
 import get from 'lodash/get';
-import type { IDataObject, IExecuteFunctions } from 'n8n-workflow/src';
+import type { IDataObject, IExecuteFunctions } from 'n8n-workflow';
 
 import { makeZodSchemaFromAttributes } from '../helpers';
 import { InformationExtractor } from '../InformationExtractor.node';

packages/cli/src/commands/base-command.ts

@@ -19,6 +19,7 @@ import * as CrashJournal from '@/crash-journal';
 import * as Db from '@/db';
 import { getDataDeduplicationService } from '@/deduplication';
 import { DeprecationService } from '@/deprecation/deprecation.service';
+import { TestRunnerService } from '@/evaluation.ee/test-runner/test-runner.service.ee';
 import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus';
 import { TelemetryEventRelay } from '@/events/relays/telemetry.event-relay';
 import { initExpressionEvaluator } from '@/expression-evaluator';
@@ -259,6 +260,10 @@ export abstract class BaseCommand extends Command {
 		Container.get(WorkflowHistoryManager).init();
 	}
 
+	async cleanupTestRunner() {
+		await Container.get(TestRunnerService).cleanupIncompleteRuns();
+	}
+
 	async finally(error: Error | undefined) {
 		if (inTest || this.id === 'start') return;
 		if (Db.connectionState.connected) {

packages/cli/src/commands/start.ts

@@ -222,6 +222,11 @@ export class Start extends BaseCommand {
 		this.initWorkflowHistory();
 		this.logger.debug('Workflow history init complete');
 
+		if (!isMultiMainEnabled) {
+			await this.cleanupTestRunner();
+			this.logger.debug('Test runner cleanup complete');
+		}
+
 		if (!this.globalConfig.endpoints.disableUi) {
 			await this.generateStaticAssets();
 		}

packages/cli/src/databases/entities/test-case-execution.ee.ts

@@ -1,4 +1,5 @@
 import { Column, Entity, ManyToOne, OneToOne } from '@n8n/typeorm';
+import type { IDataObject } from 'n8n-workflow';
 
 import {
 	datetimeColumnType,
@@ -7,9 +8,19 @@ import {
 } from '@/databases/entities/abstract-entity';
 import type { ExecutionEntity } from '@/databases/entities/execution-entity';
 import { TestRun } from '@/databases/entities/test-run.ee';
+import type { TestCaseExecutionErrorCode } from '@/evaluation.ee/test-runner/errors.ee';
 
 export type TestCaseRunMetrics = Record<string, number | boolean>;
 
+export type TestCaseExecutionStatus =
+	| 'new' // Test case execution was created and added to the test run, but has not been started yet
+	| 'running' // Workflow under test is running
+	| 'evaluation_running' // Evaluation workflow is running
+	| 'success' // Both workflows have completed successfully
+	| 'error' // An error occurred during the execution of workflow under test or evaluation workflow
+	| 'warning' // There were warnings during the execution of workflow under test or evaluation workflow. Used only to signal possible issues to user, not to indicate a failure.
+	| 'cancelled';
+
 /**
  * This entity represents the linking between the test runs and individual executions.
  * It stores status, links to past, new and evaluation executions, and metrics produced by individual evaluation wf executions
@@ -49,7 +60,7 @@ export class TestCaseExecution extends WithStringId {
 	evaluationExecutionId: string | null;
 
 	@Column()
-	status: 'new' | 'running' | 'evaluation_running' | 'success' | 'error' | 'cancelled';
+	status: TestCaseExecutionStatus;
 
 	@Column({ type: datetimeColumnType, nullable: true })
 	runAt: Date | null;
@@ -58,10 +69,10 @@ export class TestCaseExecution extends WithStringId {
 	completedAt: Date | null;
 
 	@Column('varchar', { nullable: true })
-	errorCode: string | null;
+	errorCode: TestCaseExecutionErrorCode | null;
 
 	@Column(jsonColumnType, { nullable: true })
-	errorDetails: Record<string, unknown>;
+	errorDetails: IDataObject | null;
 
 	@Column(jsonColumnType, { nullable: true })
 	metrics: TestCaseRunMetrics;

packages/cli/src/databases/entities/test-run.ee.ts

@@ -1,13 +1,17 @@
-import { Column, Entity, Index, ManyToOne, RelationId } from '@n8n/typeorm';
+import { Column, Entity, Index, ManyToOne, OneToMany, RelationId } from '@n8n/typeorm';
+import type { IDataObject } from 'n8n-workflow';
 
 import {
 	datetimeColumnType,
 	jsonColumnType,
 	WithTimestampsAndStringId,
 } from '@/databases/entities/abstract-entity';
+import type { TestCaseExecution } from '@/databases/entities/test-case-execution.ee';
 import { TestDefinition } from '@/databases/entities/test-definition.ee';
+import type { TestRunFinalResult } from '@/databases/repositories/test-run.repository.ee';
+import type { TestRunErrorCode } from '@/evaluation.ee/test-runner/errors.ee';
 
-type TestRunStatus = 'new' | 'running' | 'completed' | 'error' | 'cancelled';
+export type TestRunStatus = 'new' | 'running' | 'completed' | 'error' | 'cancelled';
 
 export type AggregatedTestRunMetrics = Record<string, number | boolean>;
 
@@ -54,4 +58,26 @@ export class TestRun extends WithTimestampsAndStringId {
 	 */
 	@Column('integer', { nullable: true })
 	failedCases: number;
+
+	/**
+	 * This will contain the error code if the test run failed.
+	 * This is used for test run level errors, not for individual test case errors.
+	 */
+	@Column('varchar', { nullable: true, length: 255 })
+	errorCode: TestRunErrorCode | null;
+
+	/**
+	 * Optional details about the error that happened during the test run
+	 */
+	@Column(jsonColumnType, { nullable: true })
+	errorDetails: IDataObject | null;
+
+	@OneToMany('TestCaseExecution', 'testRun')
+	testCaseExecutions: TestCaseExecution[];
+
+	/**
+	 * Calculated property to determine the final result of the test run
+	 * depending on the statuses of test case executions
+	 */
+	finalResult?: TestRunFinalResult | null;
 }

packages/cli/src/databases/migrations/common/1737715421462-AddErrorColumnsToTestRuns.ts

@@ -0,0 +1,24 @@
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+// We have to use raw query migration instead of schemaBuilder helpers,
+// because the typeorm schema builder implements addColumns by a table recreate for sqlite
+// which causes weird issues with the migration
+export class AddErrorColumnsToTestRuns1737715421462 implements ReversibleMigration {
+	async up({ escape, runQuery }: MigrationContext) {
+		const tableName = escape.tableName('test_run');
+		const errorCodeColumnName = escape.columnName('errorCode');
+		const errorDetailsColumnName = escape.columnName('errorDetails');
+
+		await runQuery(`ALTER TABLE ${tableName} ADD COLUMN ${errorCodeColumnName} VARCHAR(255);`);
+		await runQuery(`ALTER TABLE ${tableName} ADD COLUMN ${errorDetailsColumnName} TEXT;`);
+	}
+
+	async down({ escape, runQuery }: MigrationContext) {
+		const tableName = escape.tableName('test_run');
+		const errorCodeColumnName = escape.columnName('errorCode');
+		const errorDetailsColumnName = escape.columnName('errorDetails');
+
+		await runQuery(`ALTER TABLE ${tableName} DROP COLUMN ${errorCodeColumnName};`);
+		await runQuery(`ALTER TABLE ${tableName} DROP COLUMN ${errorDetailsColumnName};`);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -78,6 +78,7 @@ import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/173
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
 import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
+import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns';
 
 export const mysqlMigrations: Migration[] = [
 	InitialMigration1588157391238,
@@ -158,4 +159,5 @@ export const mysqlMigrations: Migration[] = [
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
 	CreateTestCaseExecutionTable1736947513045,
+	AddErrorColumnsToTestRuns1737715421462,
 ];

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -78,6 +78,7 @@ import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/173
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
 import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
+import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns';
 
 export const postgresMigrations: Migration[] = [
 	InitialMigration1587669153312,
@@ -158,4 +159,5 @@ export const postgresMigrations: Migration[] = [
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
 	CreateTestCaseExecutionTable1736947513045,
+	AddErrorColumnsToTestRuns1737715421462,
 ];

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -75,6 +75,7 @@ import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/173
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
 import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
+import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns';
 
 const sqliteMigrations: Migration[] = [
 	InitialMigration1588102412422,
@@ -152,6 +153,7 @@ const sqliteMigrations: Migration[] = [
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
 	CreateTestCaseExecutionTable1736947513045,
+	AddErrorColumnsToTestRuns1737715421462,
 ];
 
 export { sqliteMigrations };