fix: support multiple diffs with toMatchScreenshot

gemini-testing · Oct 18, 2023 · 0c898c9 · 0c898c9
1 parent 4c1fb93
commit 0c898c9
Show file tree

Hide file tree

Showing 2 changed files with 75 additions and 14 deletions.
diff --git a/lib/test-adapter/playwright.ts b/lib/test-adapter/playwright.ts
@@ -18,6 +18,8 @@ export type PlaywrightAttachment = PlaywrightTestResult['attachments'][number];
 
 export type PwtImageDiffError = TestError & {meta?: {type: string, snapshotName: string, diffClusters: CoordBounds[]}};
 
+export type PwtNoRefImageError = TestError & {meta?: {type: string, snapshotName: string}};
+
 export enum PwtTestStatus {
     PASSED = 'passed',
     FAILED = 'failed',
@@ -111,24 +113,28 @@ export class PlaywrightTestAdapter implements ReporterTestResult {
 
     get assertViewResults(): AssertViewResult[] {
         return Object.entries(this._attachmentsByState).map(([state, attachments]): AssertViewResult | null => {
-            const refImg = getImageData(attachments.find(a => a.name?.endsWith(ImageTitleEnding.Expected)));
-            const diffImg = getImageData(attachments.find(a => a.name?.endsWith(ImageTitleEnding.Diff)));
-            const currImg = getImageData(attachments.find(a => a.name?.endsWith(ImageTitleEnding.Actual)));
+            const expectedAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Expected));
+            const diffAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Diff));
+            const actualAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Actual));
+
+            const [refImg, diffImg, currImg] = [expectedAttachment, diffAttachment, actualAttachment].map(getImageData);
 
-            if (this.error?.name === ErrorName.IMAGE_DIFF && refImg && diffImg && currImg) {
+            const error = this.getSuitableImageError({state, expectedAttachment, diffAttachment, actualAttachment}) || this.error;
+
+            if (error?.name === ErrorName.IMAGE_DIFF && refImg && diffImg && currImg) {
                 return {
                     name: ErrorName.IMAGE_DIFF,
                     stateName: state,
                     refImg,
                     diffImg,
                     currImg,
-                    diffClusters: this.getDiffClusters(state)
+                    diffClusters: _.get(error, 'diffClusters', [])
                 };
-            } else if (this.error?.name === ErrorName.NO_REF_IMAGE && currImg) {
+            } else if (error?.name === ErrorName.NO_REF_IMAGE && currImg) {
                 return {
                     name: ErrorName.NO_REF_IMAGE,
-                    message: this.error.message,
-                    stack: this.error.stack,
+                    message: error.message,
+                    stack: error.stack,
                     stateName: state,
                     currImg
                 };
@@ -256,14 +262,33 @@ export class PlaywrightTestAdapter implements ReporterTestResult {
         return '';
     }
 
-    private getDiffClusters(state: string): CoordBounds[] {
+    private getSuitableImageError({state, expectedAttachment, diffAttachment, actualAttachment}: {
+        state: string;
+        expectedAttachment?: PlaywrightAttachment;
+        diffAttachment?: PlaywrightAttachment;
+        actualAttachment?: PlaywrightAttachment;
+    }): (TestError & {diffClusters?: CoordBounds[]}) | null {
         const snapshotName = state + '.png';
-        const errors = this._testResult.errors as PwtImageDiffError[];
-        const snapshotImageDiffError = errors.find(err => {
-            return err.meta?.type === 'ImageDiffError' && err.meta.snapshotName === snapshotName;
+
+        if (expectedAttachment && diffAttachment && actualAttachment) {
+            const errors = this._testResult.errors as PwtImageDiffError[];
+            const imageDiffError = errors.find(err => {
+                return err.meta?.type === ErrorName.IMAGE_DIFF && err.meta.snapshotName === snapshotName;
+            });
+
+            return {name: ErrorName.IMAGE_DIFF, message: '', diffClusters: imageDiffError?.meta?.diffClusters};
+        }
+
+        // only supports toMatchScreenshot
+        const errors = this._testResult.errors as PwtNoRefImageError[];
+        const noRefImageError = errors.find(err => {
+            return err.meta?.type === ErrorName.NO_REF_IMAGE && err.meta.snapshotName === snapshotName;
         });
 
-        return snapshotImageDiffError?.meta?.diffClusters || [];
+        return noRefImageError ? {
+            name: ErrorName.NO_REF_IMAGE,
+            message: stripAnsi(noRefImageError?.message)
+        } : null;
     }
 
     private get _attachmentsByState(): Record<string, PlaywrightAttachment[]> {

diff --git a/test/unit/lib/test-adapter/playwright.ts b/test/unit/lib/test-adapter/playwright.ts
@@ -2,7 +2,7 @@ import sinon from 'sinon';
 import _ from 'lodash';
 import proxyquire from 'proxyquire';
 import {TestCase, TestResult} from '@playwright/test/reporter';
-import {ImageTitleEnding, PlaywrightAttachment, PlaywrightTestAdapterOptions, PwtImageDiffError, PwtTestStatus} from 'lib/test-adapter/playwright';
+import {ImageTitleEnding, PlaywrightAttachment, PlaywrightTestAdapterOptions, PwtImageDiffError, PwtNoRefImageError, PwtTestStatus} from 'lib/test-adapter/playwright';
 import {ErrorName, ImageDiffError, NoRefImageError} from 'lib/errors';
 import {TestStatus} from 'lib/constants';
 
@@ -105,6 +105,42 @@ describe('PlaywrightTestAdapter', () => {
 
             assert.deepEqual(results[0].diffClusters, [{left: 0, top: 0, right: 1, bottom: 1}]);
         });
+
+        it('should detect multiple different errors from toMatchScreenshot', () => {
+            const testCaseStub = mkTestCase();
+            const testResultStub = mkTestResult({
+                attachments: [
+                    createAttachment('state1' + ImageTitleEnding.Expected),
+                    createAttachment('state1' + ImageTitleEnding.Diff),
+                    createAttachment('state1' + ImageTitleEnding.Actual),
+                    createAttachment('state2' + ImageTitleEnding.Actual)
+                ],
+                errors: [
+                    {
+                        message: 'Screenshot comparison failed',
+                        meta: {
+                            type: 'ImageDiffError',
+                            snapshotName: 'state1.png',
+                            diffClusters: [{left: 0, top: 0, right: 1, bottom: 1}]
+                        }
+                    } as PwtImageDiffError,
+                    {
+                        message: '',
+                        meta: {
+                            type: 'NoRefImageError',
+                            snapshotName: 'state2.png'
+                        }
+                    } as PwtNoRefImageError
+                ]
+            });
+
+            const adapter = new PlaywrightTestAdapter(testCaseStub, testResultStub, mkAdapterOptions());
+
+            const results = adapter.assertViewResults as ImageDiffError[];
+
+            assert.deepEqual(results[0].name, 'ImageDiffError');
+            assert.deepEqual(results[1].name, 'NoRefImageError');
+        });
     });
 
     describe('attempt', () => {