Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix snapshot auto-generation bug #619

Merged
merged 1 commit into from
Jan 8, 2024
Merged

Fix snapshot auto-generation bug #619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion tests/parse/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ const mergeTestFailureMetadata = (original: TestResult, incoming: TestResult) =>
return;
}

// If the incoming test result is a skipped, use the current failure mode
if (incoming.testResultStatus === "skipped") {
return;
}

// If the incoming test result is a failure and not assertion_failure, **invalidate the failure mode
if (incomingSuspectedFailureMode !== "assertion_failure") {
original.testFailureMetadata.set(testFullName, incomingSuspectedFailureMode);
Expand Down Expand Up @@ -343,7 +348,7 @@ const writeTestResults = (testResults: TestResultSummary) => {
if (status !== "passed" && status !== "skipped") {
const shouldRerunTest = Array.from(testFailureMetadata.values()).every(
// If any non-assertion-type failures occur, we can't proactively generate snapshot.
(failureMode) => failureMode === "assertion_failure" || failureMode === "passed",
(failureMode) => failureMode === "assertion_failure" || failureMode === "skipped",
);
if (shouldRerunTest) {
rerunPaths.push(testFilePath);
Expand Down
2 changes: 2 additions & 0 deletions tests/reporter/reporter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ export default class TestReporter implements CustomReporter {
if (individualResult.status === "failed") {
individualResult.suspectedFailureMode =
suspectedFailureModeMap.get(individualResult.fullName) ?? "unknown";
} else if (individualResult.status === "pending") {
individualResult.suspectedFailureMode = "skipped";
} else {
individualResult.suspectedFailureMode = "passed";
}
Expand Down
5 changes: 3 additions & 2 deletions tests/types/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,10 @@ export interface LandingState {
* - unknown: unknown, usually a test timeout, discrepancy, or other setup error -> requires investigation
* - task_failure: a task failure occurred, whether during execution or linter install -> requires investigation
* - passed: only during post-processing, if at least some of the tests passed -> can still generate a snapshot
* - assertion_failure: the exepcted diagnostics vary -> we can usually generate a snapshot proactively
* - assertion_failure: the expected diagnostics vary -> we can usually generate a snapshot proactively
* - skipped: the test was skipped -> defer to other tests
*/
export type FailureMode = "unknown" | "passed" | "task_failure" | "assertion_failure";
export type FailureMode = "unknown" | "passed" | "task_failure" | "assertion_failure" | "skipped";

/**
* Which OS the test was run on. Must be kept in sync with the matrix in nightly.yaml.
Expand Down
Loading