From a92c13f87903c5f3376abced4ea129d9dbb81930 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 17 Oct 2023 22:37:02 +0200 Subject: [PATCH 1/7] (nut-tree/plugin-ocr#25) Made the confidence value of a match request optional --- lib/match-request.class.ts | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/match-request.class.ts b/lib/match-request.class.ts index a4afcc1..03d91bf 100644 --- a/lib/match-request.class.ts +++ b/lib/match-request.class.ts @@ -15,25 +15,25 @@ export class MatchRequest { public constructor( public readonly haystack: Image, public readonly needle: NEEDLE_TYPE, - public readonly confidence: number, - public readonly providerData?: PROVIDER_DATA_TYPE + public readonly confidence: number | undefined, + public readonly providerData?: PROVIDER_DATA_TYPE, ) {} } export function isImageMatchRequest( - matchRequest: any + matchRequest: any, ): matchRequest is MatchRequest { return isImage(matchRequest.needle); } export function isTextMatchRequest( - matchRequest: any + matchRequest: any, ): matchRequest is MatchRequest { return isTextQuery(matchRequest.needle); } export function isColorMatchRequest( - matchRequest: any + matchRequest: any, ): matchRequest is MatchRequest { return isColorQuery(matchRequest.needle); } @@ -42,25 +42,25 @@ export function createMatchRequest( providerRegistry: ProviderRegistry, needle: PointResultFindInput, searchRegion: Region, - minMatch: number, + minMatch: number | undefined, screenImage: Image, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): MatchRequest; export function createMatchRequest( providerRegistry: ProviderRegistry, needle: RegionResultFindInput, searchRegion: Region, - minMatch: number, + minMatch: number | undefined, screenImage: Image, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): MatchRequest; export function createMatchRequest( providerRegistry: ProviderRegistry, needle: RegionResultFindInput | PointResultFindInput, searchRegion: Region, - minMatch: number, + minMatch: number | undefined, screenImage: Image, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): | MatchRequest | MatchRequest; @@ -68,9 +68,9 @@ export function createMatchRequest( providerRegistry: ProviderRegistry, needle: RegionResultFindInput | PointResultFindInput, searchRegion: Region, - minMatch: number, + minMatch: number | undefined, screenImage: Image, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): | MatchRequest | MatchRequest { @@ -80,27 +80,27 @@ export function createMatchRequest( .info( `Searching for image ${ needle.id - } in region ${searchRegion.toString()}. Required confidence: ${minMatch}` + } in region ${searchRegion.toString()}. Required confidence: ${minMatch}`, ); return new MatchRequest( screenImage, needle, minMatch, - params?.providerData + params?.providerData, ); } else if (isTextQuery(needle)) { providerRegistry.getLogProvider().info( `Searching for ${isLineQuery(needle) ? "line" : "word"} { ${isLineQuery(needle) ? needle.by.line : needle.by.word} - } in region ${searchRegion.toString()}. Required confidence: ${minMatch}` + } in region ${searchRegion.toString()}. Required confidence: ${minMatch}`, ); return new MatchRequest( screenImage, needle, minMatch, - params?.providerData + params?.providerData, ); } else if (isColorQuery(needle)) { const color = needle.by.color; @@ -109,7 +109,7 @@ export function createMatchRequest( .info( `Searching for color RGBA(${color.R},${color.G},${color.B},${ color.A - }) in region ${searchRegion.toString()}.` + }) in region ${searchRegion.toString()}.`, ); return new MatchRequest(screenImage, needle, 1, params?.providerData); From ce952193e47afd8dab820a4c55235feae052d03e Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 17 Oct 2023 22:38:42 +0200 Subject: [PATCH 2/7] (nut-tree/plugin-ocr#25) Stop setting a default confidence value to not override other confidence value (e.g. the OCR one) with the explicit value of the match request --- lib/screen.class.spec.ts | 154 +++++++++++++++++++-------------------- lib/screen.class.ts | 154 ++++++++++++++++++++------------------- 2 files changed, 155 insertions(+), 153 deletions(-) diff --git a/lib/screen.class.spec.ts b/lib/screen.class.spec.ts index a63ac1c..3405fa4 100644 --- a/lib/screen.class.spec.ts +++ b/lib/screen.class.spec.ts @@ -48,8 +48,8 @@ const providerRegistryMock = mockPartial({ 3, "needle_image", 4, - searchRegion.width * 4 - ) + searchRegion.width * 4, + ), ); }, screenSize(): Promise { @@ -110,7 +110,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const needlePromise = Promise.resolve(needle); @@ -118,7 +118,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -144,7 +144,7 @@ describe("Screen.", () => { providerRegistryMock.getWindowFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -173,7 +173,7 @@ describe("Screen.", () => { providerRegistryMock.getColorFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -211,7 +211,7 @@ describe("Screen.", () => { providerRegistryMock.getTextFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -220,7 +220,7 @@ describe("Screen.", () => { // THEN expect(findMatchMock).toHaveBeenCalledTimes(1); - } + }, ); }); @@ -233,7 +233,7 @@ describe("Screen.", () => { // THEN await expect(result).rejects.toThrowError( - /find requires an Image, a text query, a color query or a window query.*/ + /find requires an Image, a text query, a color query or a window query.*/, ); }); @@ -248,7 +248,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const needlePromise = Promise.resolve(needle); @@ -256,7 +256,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -268,7 +268,7 @@ describe("Screen.", () => { const matchRequest = new MatchRequest( expect.any(Image), needle, - SUT.config.confidence + undefined, ); expect(findMatchMock).toHaveBeenCalledWith(matchRequest); }); @@ -280,7 +280,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -293,7 +293,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); SUT.on(needle, testCallback); @@ -312,7 +312,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -326,7 +326,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); SUT.on(needle, testCallback); SUT.on(needle, secondCallback); @@ -350,7 +350,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -363,7 +363,7 @@ describe("Screen.", () => { // THEN await expect(resultRegion).rejects.toThrowError( - `Searching for ${id} failed. Reason: '${expectedReason}'` + `Searching for ${id} failed. Reason: '${expectedReason}'`, ); }); @@ -374,7 +374,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -387,7 +387,7 @@ describe("Screen.", () => { // THEN await expect(resultRegion).rejects.toThrowError( - `Searching for ${id} failed. Reason: '${rejectionReason}'` + `Searching for ${id} failed. Reason: '${rejectionReason}'`, ); }); @@ -400,7 +400,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -413,7 +413,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const parameters = new OptionalSearchParameters(undefined, minMatch); @@ -425,7 +425,7 @@ describe("Screen.", () => { const matchRequest = new MatchRequest( expect.any(Image), needle, - minMatch + minMatch, ); expect(findMatchMock).toHaveBeenCalledWith(matchRequest); }); @@ -439,7 +439,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -452,13 +452,13 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const parameters = new OptionalSearchParameters(customSearchRegion); const expectedMatchRequest = new MatchRequest( expect.any(Image), needle, - SUT.config.confidence + undefined, ); // WHEN @@ -477,7 +477,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -489,16 +489,16 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const parameters = new OptionalSearchParameters( customSearchRegion, - minMatch + minMatch, ); const expectedMatchRequest = new MatchRequest( expect.any(Image), needle, - minMatch + minMatch, ); // WHEN @@ -518,14 +518,14 @@ describe("Screen.", () => { limitedSearchRegion.left + resultRegion.left, limitedSearchRegion.top + resultRegion.top, resultRegion.width, - resultRegion.height + resultRegion.height, ); const findMatchMock = jest.fn(() => Promise.resolve(matchResult)); providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -535,7 +535,7 @@ describe("Screen.", () => { new Image(100, 100, Buffer.from([]), 3, "needle_image", 4, 100 * 4), { searchRegion: limitedSearchRegion, - } + }, ); // THEN @@ -577,7 +577,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatch: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -590,7 +590,7 @@ describe("Screen.", () => { // THEN await expect(findPromise).rejects.toThrowError( - `Searching for ${id} failed. Reason:` + `Searching for ${id} failed. Reason:`, ); }); }); @@ -606,7 +606,7 @@ describe("Screen.", () => { // THEN await expect(result).rejects.toThrowError( - /findAll requires an Image, a text query, a color query or a window query.*/ + /findAll requires an Image, a text query, a color query or a window query.*/, ); }); @@ -622,7 +622,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const needlePromise = Promise.resolve(needle); @@ -630,7 +630,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -656,7 +656,7 @@ describe("Screen.", () => { providerRegistryMock.getWindowFinder = jest.fn(() => mockPartial({ findMatches: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -685,7 +685,7 @@ describe("Screen.", () => { providerRegistryMock.getColorFinder = jest.fn(() => mockPartial({ findMatches: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -723,7 +723,7 @@ describe("Screen.", () => { providerRegistryMock.getTextFinder = jest.fn(() => mockPartial({ findMatches: findMatchMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -732,7 +732,7 @@ describe("Screen.", () => { // THEN expect(findMatchMock).toHaveBeenCalledTimes(1); - } + }, ); }); it("should call registered hook before resolve", async () => { @@ -743,7 +743,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchesMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -756,7 +756,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); SUT.on(needle, testCallback); @@ -776,7 +776,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchesMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -790,7 +790,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); SUT.on(needle, testCallback); SUT.on(needle, secondCallback); @@ -812,7 +812,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchesMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -825,11 +825,11 @@ describe("Screen.", () => { // THEN await expect(resultRegion).rejects.toThrowError( - `Searching for ${id} failed. Reason: '${rejectionReason}'` + `Searching for ${id} failed. Reason: '${rejectionReason}'`, ); }); - it("should override default confidence value with parameter.", async () => { + it("should set confidence value of match request with parameter.", async () => { // GIVEN const minMatch = 0.8; const matchResult = new MatchResult(minMatch, searchRegion); @@ -838,7 +838,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchesMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -851,7 +851,7 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const parameters = new OptionalSearchParameters(undefined, minMatch); @@ -863,7 +863,7 @@ describe("Screen.", () => { const matchRequest = new MatchRequest( expect.any(Image), needle, - minMatch + minMatch, ); expect(findMatchesMock).toHaveBeenCalledWith(matchRequest); }); @@ -877,7 +877,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchesMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -890,13 +890,13 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const parameters = new OptionalSearchParameters(customSearchRegion); const expectedMatchRequest = new MatchRequest( expect.any(Image), needle, - SUT.config.confidence + undefined, ); // WHEN @@ -915,7 +915,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchesMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -927,16 +927,16 @@ describe("Screen.", () => { 3, "needle_image", 4, - 100 * 4 + 100 * 4, ); const parameters = new OptionalSearchParameters( customSearchRegion, - minMatch + minMatch, ); const expectedMatchRequest = new MatchRequest( expect.any(Image), needle, - minMatch + minMatch, ); // WHEN @@ -956,14 +956,14 @@ describe("Screen.", () => { limitedSearchRegion.left + resultRegion.left, limitedSearchRegion.top + resultRegion.top, resultRegion.width, - resultRegion.height + resultRegion.height, ); const findMatchesMock = jest.fn(() => Promise.resolve([matchResult])); providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchesMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -973,7 +973,7 @@ describe("Screen.", () => { new Image(100, 100, Buffer.from([]), 3, "needle_image", 4, 100 * 4), { searchRegion: limitedSearchRegion, - } + }, ); // THEN @@ -1015,7 +1015,7 @@ describe("Screen.", () => { providerRegistryMock.getImageFinder = jest.fn(() => mockPartial({ findMatches: findMatchesMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -1028,7 +1028,7 @@ describe("Screen.", () => { // THEN await expect(findPromise).rejects.toThrowError( - `Searching for ${id} failed. Reason:` + `Searching for ${id} failed. Reason:`, ); }); }); @@ -1040,7 +1040,7 @@ describe("Screen.", () => { providerRegistryMock.getScreen = jest.fn(() => mockPartial({ highlightScreenRegion: highlightMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -1056,13 +1056,13 @@ describe("Screen.", () => { // GIVEN const highlightRegion = new Region(10, 20, 30, 40); const highlightRegionPromise = new Promise((res) => - res(highlightRegion) + res(highlightRegion), ); const highlightMock = jest.fn((value: any) => Promise.resolve(value)); providerRegistryMock.getScreen = jest.fn(() => mockPartial({ highlightScreenRegion: highlightMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -1085,19 +1085,19 @@ describe("Screen.", () => { 4, "test", 4, - 100 * 4 + 100 * 4, ); const grabScreenMock = jest.fn(() => Promise.resolve(screenshot)); const saveImageMock = jest.fn(); providerRegistryMock.getScreen = jest.fn(() => mockPartial({ grabScreen: grabScreenMock, - }) + }), ); providerRegistryMock.getImageWriter = jest.fn(() => mockPartial({ store: saveImageMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -1125,7 +1125,7 @@ describe("Screen.", () => { providerRegistryMock.getScreen = jest.fn(() => mockPartial({ grabScreen: grabScreenMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -1137,7 +1137,7 @@ describe("Screen.", () => { // THEN expect(result).rejects.toThrowError( - /^capture requires an Image, but received/ + /^capture requires an Image, but received/, ); }); }); @@ -1152,7 +1152,7 @@ describe("Screen.", () => { 4, "test", 4, - 100 * 4 + 100 * 4, ); const regionToCapture = mockPartial({ top: 42, @@ -1165,12 +1165,12 @@ describe("Screen.", () => { providerRegistryMock.getScreen = jest.fn(() => mockPartial({ grabScreenRegion: grabScreenMock, - }) + }), ); providerRegistryMock.getImageWriter = jest.fn(() => mockPartial({ store: saveImageMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -1204,7 +1204,7 @@ describe("Screen.", () => { providerRegistryMock.getScreen = jest.fn(() => mockPartial({ grabScreenRegion: grabScreenMock, - }) + }), ); providerRegistryMock.getLogProvider = () => new NoopLogProvider(); @@ -1216,7 +1216,7 @@ describe("Screen.", () => { // THEN expect(result).rejects.toThrowError( - /^captureRegion requires an Image, but received/ + /^captureRegion requires an Image, but received/, ); }); }); diff --git a/lib/screen.class.ts b/lib/screen.class.ts index de4c794..c46b604 100644 --- a/lib/screen.class.ts +++ b/lib/screen.class.ts @@ -26,7 +26,7 @@ import { Window } from "./window.class"; export type WindowCallback = (target: Window) => void | Promise; export type MatchResultCallback = ( - target: MatchResult + target: MatchResult, ) => void | Promise; export type FindHookCallback = | WindowCallback @@ -36,7 +36,7 @@ export type FindHookCallback = function validateSearchRegion( search: Region, screen: Region, - providerRegistry: ProviderRegistry + providerRegistry: ProviderRegistry, ) { providerRegistry .getLogProvider() @@ -63,7 +63,7 @@ function validateSearchRegion( } if (search.width < 2 || search.height < 2) { const e = new Error( - `Search region is not large enough. Must be at least two pixels in both width and height.` + `Search region is not large enough. Must be at least two pixels in both width and height.`, ); providerRegistry.getLogProvider().error(e, { region: search }); throw e; @@ -73,7 +73,7 @@ function validateSearchRegion( search.top + search.height > screen.height ) { const e = new Error( - `Search region extends beyond screen boundaries (${screen.width}x${screen.height})` + `Search region extends beyond screen boundaries (${screen.width}x${screen.height})`, ); providerRegistry.getLogProvider().error(e, { region: search, screen }); throw e; @@ -119,13 +119,13 @@ export type FindInput = export type FindResult = Region | Point | Window; function isRegionResultFindInput( - input: RegionResultFindInput | PointResultFindInput + input: RegionResultFindInput | PointResultFindInput, ): input is RegionResultFindInput { return isImage(input) || isTextQuery(input); } function isPointResultFindInput( - input: RegionResultFindInput | PointResultFindInput + input: RegionResultFindInput | PointResultFindInput, ): input is PointResultFindInput { return isColorQuery(input); } @@ -152,7 +152,7 @@ export class ScreenClass { private findHooks: Map = new Map< FindInput, FindHookCallback[] - >() + >(), ) {} /** @@ -182,23 +182,23 @@ export class ScreenClass { */ public async find( searchInput: RegionResultFindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async find( searchInput: PointResultFindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async find( searchInput: WindowResultFindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async find( searchInput: FindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async find( searchInput: FindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise { const needle = await searchInput; this.providerRegistry.getLogProvider().info(`Searching for ${needle}`); @@ -236,7 +236,7 @@ export class ScreenClass { searchRegion, minMatch, screenImage, - params + params, ); if (isRegionResultFindInput(needle)) { @@ -245,7 +245,7 @@ export class ScreenClass { matchRequest as MatchRequest< RegionResultFindInput, PROVIDER_DATA_TYPE - > + >, ); this.providerRegistry @@ -265,7 +265,7 @@ export class ScreenClass { searchRegion.left + matchResult.location.left, searchRegion.top + matchResult.location.top, matchResult.location.width, - matchResult.location.height + matchResult.location.height, ); this.providerRegistry @@ -286,7 +286,7 @@ export class ScreenClass { matchRequest as MatchRequest< PointResultFindInput, PROVIDER_DATA_TYPE - > + >, ); this.providerRegistry @@ -304,7 +304,7 @@ export class ScreenClass { const resultPoint = new Point( searchRegion.left + matchResult.location.x, - searchRegion.top + matchResult.location.y + searchRegion.top + matchResult.location.y, ); this.providerRegistry @@ -315,11 +315,11 @@ export class ScreenClass { } } throw new Error( - `Search input is not supported. Please use a valid search input type.` + `Search input is not supported. Please use a valid search input type.`, ); } catch (e) { const error = new Error( - `Searching for ${needle.id} failed. Reason: '${e}'` + `Searching for ${needle.id} failed. Reason: '${e}'`, ); this.providerRegistry.getLogProvider().error(error); throw error; @@ -333,19 +333,19 @@ export class ScreenClass { */ public async findAll( searchInput: RegionResultFindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async findAll( searchInput: PointResultFindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async findAll( searchInput: WindowResultFindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async findAll( searchInput: FindInput | Promise, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise { const needle = await searchInput; this.providerRegistry.getLogProvider().info(`Searching for ${needle}`); @@ -359,13 +359,13 @@ export class ScreenClass { .findMatches(needle); const windows = matches.map( (windowHandle: number) => - new Window(this.providerRegistry, windowHandle) + new Window(this.providerRegistry, windowHandle), ); const possibleHooks = this.getHooksForInput(needle) || []; this.providerRegistry .getLogProvider() .debug( - `${possibleHooks.length} hooks triggered for ${windows.length} matches` + `${possibleHooks.length} hooks triggered for ${windows.length} matches`, ); for (const hook of possibleHooks) { for (const wnd of windows) { @@ -385,7 +385,7 @@ export class ScreenClass { searchRegion, minMatch, screenImage, - params + params, ); validateSearchRegion(searchRegion, screenSize, this.providerRegistry); @@ -393,13 +393,13 @@ export class ScreenClass { const matchResults = await getMatchResults( this.providerRegistry, - matchRequest + matchRequest, ); const possibleHooks = this.getHooksForInput(needle) || []; this.providerRegistry .getLogProvider() .debug( - `${possibleHooks.length} hooks triggered for ${matchResults.length} matches` + `${possibleHooks.length} hooks triggered for ${matchResults.length} matches`, ); for (const hook of possibleHooks) { for (const matchResult of matchResults) { @@ -412,7 +412,7 @@ export class ScreenClass { searchRegion.left + matchResult.location.left, searchRegion.top + matchResult.location.top, matchResult.location.width, - matchResult.location.height + matchResult.location.height, ); this.providerRegistry .getLogProvider() @@ -445,7 +445,7 @@ export class ScreenClass { searchRegion, 0, screenImage, - params + params, ); validateSearchRegion(searchRegion, screenSize, this.providerRegistry); @@ -453,13 +453,13 @@ export class ScreenClass { const matchResults = await getMatchResults( this.providerRegistry, - matchRequest + matchRequest, ); const possibleHooks = this.getHooksForInput(needle) || []; this.providerRegistry .getLogProvider() .debug( - `${possibleHooks.length} hooks triggered for ${matchResults.length} matches` + `${possibleHooks.length} hooks triggered for ${matchResults.length} matches`, ); for (const hook of possibleHooks) { for (const matchResult of matchResults) { @@ -470,7 +470,7 @@ export class ScreenClass { return matchResults.map((matchResult) => { const resultPoint = new Point( searchRegion.left + matchResult.location.x, - searchRegion.top + matchResult.location.y + searchRegion.top + matchResult.location.y, ); this.providerRegistry .getLogProvider() @@ -479,11 +479,11 @@ export class ScreenClass { }); } throw new Error( - `Search input is not supported. Please use a valid search input type.` + `Search input is not supported. Please use a valid search input type.`, ); } catch (e) { const error = new Error( - `Searching for ${needle.id} failed. Reason: '${e}'` + `Searching for ${needle.id} failed. Reason: '${e}'`, ); this.providerRegistry.getLogProvider().error(error); throw error; @@ -495,14 +495,14 @@ export class ScreenClass { * @param regionToHighlight The {@link Region} to highlight */ public async highlight( - regionToHighlight: Region | Promise + regionToHighlight: Region | Promise, ): Promise { const highlightRegion = await regionToHighlight; if (!isRegion(highlightRegion)) { const e = Error( `highlight requires an Region, but received ${JSON.stringify( - highlightRegion - )}` + highlightRegion, + )}`, ); this.providerRegistry.getLogProvider().error(e); throw e; @@ -512,14 +512,14 @@ export class ScreenClass { .info( `Highlighting ${highlightRegion.toString()} for ${ this.config.highlightDurationMs / 1000 - } with ${this.config.highlightOpacity * 100}% opacity` + } with ${this.config.highlightOpacity * 100}% opacity`, ); await this.providerRegistry .getScreen() .highlightScreenRegion( highlightRegion, this.config.highlightDurationMs, - this.config.highlightOpacity + this.config.highlightOpacity, ); return highlightRegion; } @@ -535,25 +535,25 @@ export class ScreenClass { searchInput: RegionResultFindInput | Promise, timeoutMs?: number, updateInterval?: number, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async waitFor( searchInput: PointResultFindInput | Promise, timeoutMs?: number, updateInterval?: number, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async waitFor( searchInput: WindowResultFindInput | Promise, timeoutMs?: number, updateInterval?: number, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise; public async waitFor( searchInput: FindInput | Promise, timeoutMs?: number, updateInterval?: number, - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ): Promise { const needle = await searchInput; @@ -567,7 +567,7 @@ export class ScreenClass { .info( `Waiting for ${needle.id} to appear on screen. Timeout: ${ timeoutValue / 1000 - } seconds, interval: ${updateIntervalValue} ms` + } seconds, interval: ${updateIntervalValue} ms`, ); return timeout( updateIntervalValue, @@ -577,7 +577,7 @@ export class ScreenClass { }, { signal: params?.abort, - } + }, ); } @@ -589,11 +589,11 @@ export class ScreenClass { public on(searchInput: WindowResultFindInput, callback: WindowCallback): void; public on( searchInput: PointResultFindInput, - callback: MatchResultCallback + callback: MatchResultCallback, ): void; public on( searchInput: RegionResultFindInput, - callback: MatchResultCallback + callback: MatchResultCallback, ): void; public on(searchInput: FindInput, callback: FindHookCallback): void { this.validateSearchInput("on", searchInput); @@ -605,7 +605,7 @@ export class ScreenClass { .info( `Registered callback for image ${searchInput.id}. There are currently ${ existingHooks.length + 1 - } hooks registered` + } hooks registered`, ); } @@ -622,14 +622,14 @@ export class ScreenClass { fileFormat: FileType = FileType.PNG, filePath: string = cwd(), fileNamePrefix: string = "", - fileNamePostfix: string = "" + fileNamePostfix: string = "", ): Promise { const currentScreen = await this.providerRegistry.getScreen().grabScreen(); if (!isImage(currentScreen)) { const e = new Error( `capture requires an Image, but received ${JSON.stringify( - currentScreen - )}` + currentScreen, + )}`, ); this.providerRegistry.getLogProvider().error(e); throw e; @@ -637,7 +637,7 @@ export class ScreenClass { this.providerRegistry .getLogProvider() .info( - `Capturing whole screen (0, 0, ${currentScreen.width}, ${currentScreen.height})` + `Capturing whole screen (0, 0, ${currentScreen.width}, ${currentScreen.height})`, ); return this.saveImage( currentScreen, @@ -645,7 +645,7 @@ export class ScreenClass { fileFormat, filePath, fileNamePrefix, - fileNamePostfix + fileNamePostfix, ); } @@ -657,7 +657,7 @@ export class ScreenClass { this.providerRegistry .getLogProvider() .info( - `Grabbed whole screen (0, 0, ${currentScreen.width}, ${currentScreen.height})` + `Grabbed whole screen (0, 0, ${currentScreen.width}, ${currentScreen.height})`, ); return currentScreen; } @@ -677,14 +677,14 @@ export class ScreenClass { fileFormat: FileType = FileType.PNG, filePath: string = cwd(), fileNamePrefix: string = "", - fileNamePostfix: string = "" + fileNamePostfix: string = "", ): Promise { const targetRegion = await regionToCapture; if (!isRegion(targetRegion)) { const e = new Error( `captureRegion requires an Region, but received ${JSON.stringify( - targetRegion - )}` + targetRegion, + )}`, ); this.providerRegistry.getLogProvider().error(e); throw e; @@ -698,8 +698,8 @@ export class ScreenClass { if (!isImage(regionImage)) { const e = new Error( `captureRegion requires an Image, but received ${JSON.stringify( - regionImage - )}` + regionImage, + )}`, ); this.providerRegistry.getLogProvider().error(e); throw e; @@ -710,7 +710,7 @@ export class ScreenClass { fileFormat, filePath, fileNamePrefix, - fileNamePostfix + fileNamePostfix, ); } @@ -719,14 +719,14 @@ export class ScreenClass { * @param regionToGrab The screen region to grab */ public async grabRegion( - regionToGrab: Region | Promise + regionToGrab: Region | Promise, ): Promise { const targetRegion = await regionToGrab; if (!isRegion(targetRegion)) { const e = new Error( `grabRegion requires an Region, but received ${JSON.stringify( - targetRegion - )}` + targetRegion, + )}`, ); this.providerRegistry.getLogProvider().error(e); throw e; @@ -749,21 +749,23 @@ export class ScreenClass { const inputPoint = await point; if (!isPoint(inputPoint)) { const e = new Error( - `colorAt requires a Point, but received ${JSON.stringify(inputPoint)}` + `colorAt requires a Point, but received ${JSON.stringify(inputPoint)}`, ); this.providerRegistry.getLogProvider().error(e); throw e; } const scaledPoint = new Point( inputPoint.x * screenContent.pixelDensity.scaleX, - inputPoint.y * screenContent.pixelDensity.scaleY + inputPoint.y * screenContent.pixelDensity.scaleY, ); this.providerRegistry .getLogProvider() .debug( `Point ${inputPoint.toString()} has been scaled by (${ screenContent.pixelDensity.scaleX - }, ${screenContent.pixelDensity.scaleY}) into ${scaledPoint.toString()}` + }, ${ + screenContent.pixelDensity.scaleY + }) into ${scaledPoint.toString()}`, ); const color = await this.providerRegistry .getImageProcessor() @@ -780,7 +782,7 @@ export class ScreenClass { fileFormat: FileType, filePath: string, fileNamePrefix: string, - fileNamePostfix: string + fileNamePostfix: string, ) { const outputPath = generateOutputPath(fileName, { path: filePath, @@ -799,9 +801,9 @@ export class ScreenClass { } private async getFindParameters( - params?: OptionalSearchParameters + params?: OptionalSearchParameters, ) { - const minMatch = params?.confidence ?? this.config.confidence; + const minMatch = params?.confidence; const screenSize = await this.providerRegistry.getScreen().screenSize(); const searchRegion = (await params?.searchRegion) ?? screenSize; const screenImage = await this.providerRegistry @@ -826,13 +828,13 @@ export class ScreenClass { private getHooksForInput(input: WindowResultFindInput): WindowCallback[]; private getHooksForInput( - input: RegionResultFindInput + input: RegionResultFindInput, ): MatchResultCallback[]; private getHooksForInput( - input: PointResultFindInput + input: PointResultFindInput, ): MatchResultCallback[]; private getHooksForInput( - input: FindInput + input: FindInput, ): | MatchResultCallback[] | MatchResultCallback[] @@ -863,7 +865,7 @@ export class ScreenClass { | LineQuery | WindowResultFindInput | PointResultFindInput - | Promise + | Promise, ) { if ( !isImage(needle) && @@ -873,8 +875,8 @@ export class ScreenClass { ) { const e = Error( `${functionName} requires an Image, a text query, a color query or a window query, but received ${JSON.stringify( - needle - )}` + needle, + )}`, ); this.providerRegistry.getLogProvider().error(e, { needle }); throw e; From bc78ce1c30fc4a1075f946c555d6d854e8b649e8 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 17 Oct 2023 22:50:26 +0200 Subject: [PATCH 3/7] (nut-tree/plugin-ocr#25) Removed redefinition of sleep and used import --- e2e/window-test/test.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/e2e/window-test/test.js b/e2e/window-test/test.js index ed9a757..b435483 100644 --- a/e2e/window-test/test.js +++ b/e2e/window-test/test.js @@ -1,11 +1,7 @@ const { _electron: electron } = require("playwright"); -const { getActiveWindow, getWindows } = require("@nut-tree/nut-js"); +const { sleep, getActiveWindow, getWindows } = require("@nut-tree/nut-js"); const { POS_X, POS_Y, WIDTH, HEIGTH, TITLE } = require("./constants"); -const sleep = async (ms) => { - return new Promise((resolve) => setTimeout(resolve, ms)); -}; - let app; let page; let windowHandle; From 3c4a8aa5da5714796da78b7422c3ff5c0401529a Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 17 Oct 2023 22:51:03 +0200 Subject: [PATCH 4/7] (nut-tree/plugin-ocr#25) Updated workflow definitions --- .github/workflows/ci.yaml | 13 +++++++------ .github/workflows/snapshot_release.yaml | 15 ++++++++------- .github/workflows/tagged_release.yaml | 14 +++++++------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8d37135..89565b7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,6 +7,7 @@ on: # paths-ignore: # - '**/*.md' pull_request: + workflow_dispatch: jobs: sonar: @@ -14,11 +15,11 @@ jobs: if: "!contains(github.event.head_commit.message, 'skip ci')" steps: - name: Set up Git repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - name: Setup Docker run: | docker pull s1hofmann/nut-ci:latest @@ -57,13 +58,13 @@ jobs: strategy: matrix: os: [windows-latest, macos-latest] - node: [16] + node: [18] runs-on: ${{matrix.os}} steps: - name: Set up Git repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{matrix.node}} - name: Setup Docker diff --git a/.github/workflows/snapshot_release.yaml b/.github/workflows/snapshot_release.yaml index a8ccbd5..0ed0de1 100644 --- a/.github/workflows/snapshot_release.yaml +++ b/.github/workflows/snapshot_release.yaml @@ -8,19 +8,20 @@ on: repository_dispatch: types: - snapshot-release + workflow_dispatch: jobs: test: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - node: [16] + node: [18] runs-on: ${{matrix.os}} steps: - name: Set up Git repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{matrix.node}} - name: Setup Docker @@ -54,11 +55,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Git repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 registry-url: "https://registry.npmjs.org" - name: Install run: npm ci @@ -70,7 +71,7 @@ jobs: run: npm run publish-next env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: registry-url: "https://npm.pkg.github.com" - name: Publish snapshot release to GPR diff --git a/.github/workflows/tagged_release.yaml b/.github/workflows/tagged_release.yaml index df8126b..66cc374 100644 --- a/.github/workflows/tagged_release.yaml +++ b/.github/workflows/tagged_release.yaml @@ -9,13 +9,13 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - node: [16] + node: [18] runs-on: ${{matrix.os}} steps: - name: Set up Git repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{matrix.node}} - name: Setup Docker @@ -47,11 +47,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Git repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 registry-url: "https://registry.npmjs.org" - name: Install run: npm ci @@ -67,7 +67,7 @@ jobs: run: npm publish env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: registry-url: "https://npm.pkg.github.com" - name: Publish tagged release to GPR From f88f31b2fc6ba9153deff9dd9424f3de9bcd3b64 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 17 Oct 2023 23:00:30 +0200 Subject: [PATCH 5/7] (nut-tree/plugin-ocr#25) Run npx playwright install before running tests --- .github/workflows/ci.yaml | 4 +++- .github/workflows/snapshot_release.yaml | 4 +++- .github/workflows/tagged_release.yaml | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 89565b7..2433cd0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,7 +81,9 @@ jobs: - name: Generate coverage report uses: GabrielBB/xvfb-action@v1 with: - run: npm run coverage -- --coverageDirectory=coverage/unit + run: | + npx playwright install --with-deps + npm run coverage -- --coverageDirectory=coverage/unit - name: Run Docker E2E tests if: ${{matrix.os == 'ubuntu-latest'}} run: docker exec nut-ci bash -c "bash $PWD/.build/build.sh ${PWD} ${{matrix.node}}" diff --git a/.github/workflows/snapshot_release.yaml b/.github/workflows/snapshot_release.yaml index 0ed0de1..f2ecf1e 100644 --- a/.github/workflows/snapshot_release.yaml +++ b/.github/workflows/snapshot_release.yaml @@ -40,7 +40,9 @@ jobs: - name: Run tests uses: GabrielBB/xvfb-action@v1 with: - run: npm test + run: | + npx playwright install --with-deps + npm test - name: Run Docker E2E tests if: ${{matrix.os == 'ubuntu-latest'}} run: docker exec nut-ci bash -c "bash $PWD/.build/build.sh ${PWD} ${{matrix.node}}" diff --git a/.github/workflows/tagged_release.yaml b/.github/workflows/tagged_release.yaml index 66cc374..756ffd2 100644 --- a/.github/workflows/tagged_release.yaml +++ b/.github/workflows/tagged_release.yaml @@ -32,7 +32,9 @@ jobs: - name: Generate coverage report uses: GabrielBB/xvfb-action@v1 with: - run: npm test + run: | + npx playwright install --with-deps + npm test - name: Run Docker E2E tests if: ${{matrix.os == 'ubuntu-latest'}} run: docker exec nut-ci bash -c "bash $PWD/.build/build.sh ${PWD} ${{matrix.node}}" From 7f5047b841f2b30e10a591eba4ab1a8ab61cd937 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 17 Oct 2023 23:09:32 +0200 Subject: [PATCH 6/7] (nut-tree/plugin-ocr#25) Specify working directory where to load the Electron main from in window tests --- e2e/window-test/test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/window-test/test.js b/e2e/window-test/test.js index b435483..14857eb 100644 --- a/e2e/window-test/test.js +++ b/e2e/window-test/test.js @@ -9,7 +9,7 @@ let windowHandle; const APP_TIMEOUT = 10000; beforeEach(async () => { - app = await electron.launch({ args: ["main.js"] }); + app = await electron.launch({ args: ["main.js"], cwd: __dirname }); page = await app.firstWindow({ timeout: APP_TIMEOUT }); windowHandle = await app.browserWindow(page); await page.waitForLoadState("domcontentloaded"); From fd49680a072b14e79f2e04176ac6f346e4f0fffa Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Tue, 17 Oct 2023 23:18:51 +0200 Subject: [PATCH 7/7] (nut-tree/plugin-ocr#25) Increased Jest timeout for window tests to accommodate startup times, removed superfluous Docker stuf from CI runs on macOS and Windows, since tests would be running in Linux again which is just confusing in this place --- .github/workflows/ci.yaml | 8 -------- e2e/window-test/test.js | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2433cd0..fa4a9ae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,11 +67,6 @@ jobs: uses: actions/setup-node@v3 with: node-version: ${{matrix.node}} - - name: Setup Docker - if: ${{matrix.os == 'ubuntu-latest'}} - run: | - docker pull s1hofmann/nut-ci:latest - docker run -it -d --name nut-ci --shm-size 8gb --user $(id -u):$(id -g) -v ${PWD}:${PWD}:rw s1hofmann/nut-ci:latest bash - name: Install run: npm ci - name: Compile @@ -84,9 +79,6 @@ jobs: run: | npx playwright install --with-deps npm run coverage -- --coverageDirectory=coverage/unit - - name: Run Docker E2E tests - if: ${{matrix.os == 'ubuntu-latest'}} - run: docker exec nut-ci bash -c "bash $PWD/.build/build.sh ${PWD} ${{matrix.node}}" - name: Run Electron e2e test subpackage uses: GabrielBB/xvfb-action@v1 with: diff --git a/e2e/window-test/test.js b/e2e/window-test/test.js index 14857eb..1a7e16a 100644 --- a/e2e/window-test/test.js +++ b/e2e/window-test/test.js @@ -7,6 +7,7 @@ let page; let windowHandle; const APP_TIMEOUT = 10000; +jest.setTimeout(APP_TIMEOUT); beforeEach(async () => { app = await electron.launch({ args: ["main.js"], cwd: __dirname });