diff --git a/test/reporter-cucumber-msg/features/retry/common.steps.ts b/test/reporter-cucumber-msg/features/retry/common.steps.ts new file mode 100644 index 00000000..77f5b70b --- /dev/null +++ b/test/reporter-cucumber-msg/features/retry/common.steps.ts @@ -0,0 +1,38 @@ +import { Given, isPlaywrightRun } from '../fixtures'; + +Given('a step that always passes', function () { + // no-op +}); + +// keep this global to store counter between retries +let secondTimePass = 0; +Given('a step that passes the second time', function () { + if (isPlaywrightRun) { + if (this.testInfo.retry < 1) { + throw new Error('Exception in step'); + } + } else { + secondTimePass++; + if (secondTimePass < 2) { + throw new Error('Exception in step'); + } + } +}); + +let thirdTimePass = 0; +Given('a step that passes the third time', function () { + if (isPlaywrightRun) { + if (this.testInfo.retry < 2) { + throw new Error('Exception in step'); + } + } else { + thirdTimePass++; + if (thirdTimePass < 3) { + throw new Error('Exception in step'); + } + } +}); + +Given('a step that always fails', function () { + throw new Error('Exception in step'); +}); diff --git a/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts b/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts deleted file mode 100644 index 80f09969..00000000 --- a/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Given } from '@cucumber/cucumber'; - -Given('a step that always passes', function () { - // no-op -}); - -let secondTimePass = 0; -Given('a step that passes the second time', function () { - secondTimePass++; - if (secondTimePass < 2) { - throw new Error('Exception in step'); - } -}); - -let thirdTimePass = 0; -Given('a step that passes the third time', function () { - thirdTimePass++; - if (thirdTimePass < 3) { - throw new Error('Exception in step'); - } -}); - -Given('a step that always fails', function () { - throw new Error('Exception in step'); -}); diff --git a/test/reporter-cucumber-msg/features/retry/expected-reports/json-report.json b/test/reporter-cucumber-msg/features/retry/expected-reports/json-report.json index 48b7a6b7..a08684c4 100644 --- a/test/reporter-cucumber-msg/features/retry/expected-reports/json-report.json +++ b/test/reporter-cucumber-msg/features/retry/expected-reports/json-report.json @@ -15,11 +15,11 @@ "line": 9, "name": "a step that always passes", "match": { - "location": "features/retry/cucumber.steps.ts:3" + "location": "features/fixtures.ts:19" }, "result": { "status": "passed", - "duration": 469249 + "duration": 810999 } } ], @@ -39,11 +39,11 @@ "line": 12, "name": "a step that passes the second time", "match": { - "location": "features/retry/cucumber.steps.ts:8" + "location": "features/fixtures.ts:19" }, "result": { "status": "passed", - "duration": 26166 + "duration": 56125 } } ], @@ -63,11 +63,11 @@ "line": 15, "name": "a step that passes the third time", "match": { - "location": "features/retry/cucumber.steps.ts:16" + "location": "features/fixtures.ts:19" }, "result": { "status": "passed", - "duration": 32542 + "duration": 30124 } } ], @@ -87,12 +87,12 @@ "line": 18, "name": "a step that always fails", "match": { - "location": "features/retry/cucumber.steps.ts:23" + "location": "features/fixtures.ts:19" }, "result": { "status": "failed", - "duration": 33125, - "error_message": "Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:24:9)" + "duration": 175083, + "error_message": "Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:37:9)" } } ], diff --git a/test/reporter-cucumber-msg/features/retry/expected-reports/messages.ndjson b/test/reporter-cucumber-msg/features/retry/expected-reports/messages.ndjson index 7b90a059..54702788 100644 --- a/test/reporter-cucumber-msg/features/retry/expected-reports/messages.ndjson +++ b/test/reporter-cucumber-msg/features/retry/expected-reports/messages.ndjson @@ -1,53 +1,53 @@ {"meta":{"protocolVersion":"24.1.0","implementation":{"version":"11.1.0","name":"cucumber-js"},"cpu":{"name":"arm64"},"os":{"name":"darwin","version":"22.6.0"},"runtime":{"name":"node.js","version":"20.14.0"}}} {"source":{"data":"Feature: Retry\n Some Cucumber implementations support a Retry mechanism, where test cases that fail\n can be retried up to a limited number of attempts in the same test run.\n\n Non-passing statuses other than FAILED won't trigger a retry, as they are not\n going to pass however many times we attempt them.\n\n Scenario: Test cases that pass aren't retried\n Given a step that always passes\n\n Scenario: Test cases that fail are retried if within the --retry limit\n Given a step that passes the second time\n\n Scenario: Test cases that fail will continue to retry up to the --retry limit\n Given a step that passes the third time\n\n Scenario: Test cases won't retry after failing more than the --retry limit\n Given a step that always fails\n\n\n# Scenario: Test cases won't retry when the status is UNDEFINED\n# Given a non-existent step","uri":"features/retry/retry.feature","mediaType":"text/x.cucumber.gherkin+plain"}} -{"gherkinDocument":{"feature":{"tags":[],"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"Retry","description":" Some Cucumber implementations support a Retry mechanism, where test cases that fail\n can be retried up to a limited number of attempts in the same test run.\n\n Non-passing statuses other than FAILED won't trigger a retry, as they are not\n going to pass however many times we attempt them.","children":[{"scenario":{"id":"2727e4a5-ba73-4e66-956a-0c08d15005ac","tags":[],"location":{"line":8,"column":3},"keyword":"Scenario","name":"Test cases that pass aren't retried","description":"","steps":[{"id":"0fa18720-6372-4989-ab4d-3f659e129064","location":{"line":9,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step that always passes"}],"examples":[]}},{"scenario":{"id":"f559eaa0-5c95-4351-86be-9641c4779513","tags":[],"location":{"line":11,"column":3},"keyword":"Scenario","name":"Test cases that fail are retried if within the --retry limit","description":"","steps":[{"id":"9947b81a-a6b2-4fa6-aab3-58936208637f","location":{"line":12,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step that passes the second time"}],"examples":[]}},{"scenario":{"id":"cae80329-6e10-4275-a0c2-ef714106a52a","tags":[],"location":{"line":14,"column":3},"keyword":"Scenario","name":"Test cases that fail will continue to retry up to the --retry limit","description":"","steps":[{"id":"871c5cac-39ea-4ef1-a0ea-70343f345b3c","location":{"line":15,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step that passes the third time"}],"examples":[]}},{"scenario":{"id":"9cd1823f-3614-4124-96d6-1a18b73e1870","tags":[],"location":{"line":17,"column":3},"keyword":"Scenario","name":"Test cases won't retry after failing more than the --retry limit","description":"","steps":[{"id":"c41656a7-a69f-4fb1-8080-6836d386a8be","location":{"line":18,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step that always fails"}],"examples":[]}}]},"comments":[{"location":{"line":21,"column":1},"text":"# Scenario: Test cases won't retry when the status is UNDEFINED"},{"location":{"line":22,"column":1},"text":"# Given a non-existent step"}],"uri":"features/retry/retry.feature"}} -{"pickle":{"id":"836fb5ff-9e9f-492b-8a51-d755baf3a380","uri":"features/retry/retry.feature","astNodeIds":["2727e4a5-ba73-4e66-956a-0c08d15005ac"],"tags":[],"name":"Test cases that pass aren't retried","language":"en","steps":[{"id":"d9b57409-0029-402a-83e7-5455f481b8a8","text":"a step that always passes","type":"Context","astNodeIds":["0fa18720-6372-4989-ab4d-3f659e129064"]}]}} -{"pickle":{"id":"f7a858ca-0b6e-4354-96b9-fadc25c344d6","uri":"features/retry/retry.feature","astNodeIds":["f559eaa0-5c95-4351-86be-9641c4779513"],"tags":[],"name":"Test cases that fail are retried if within the --retry limit","language":"en","steps":[{"id":"cf61fdaa-524c-46fc-93ad-0f37afe1d833","text":"a step that passes the second time","type":"Context","astNodeIds":["9947b81a-a6b2-4fa6-aab3-58936208637f"]}]}} -{"pickle":{"id":"cbda704d-db09-495e-880f-fb818bdb3e69","uri":"features/retry/retry.feature","astNodeIds":["cae80329-6e10-4275-a0c2-ef714106a52a"],"tags":[],"name":"Test cases that fail will continue to retry up to the --retry limit","language":"en","steps":[{"id":"f4d91dda-9bd0-4120-a12b-671dc2792085","text":"a step that passes the third time","type":"Context","astNodeIds":["871c5cac-39ea-4ef1-a0ea-70343f345b3c"]}]}} -{"pickle":{"id":"299f3b17-686e-44c0-96a1-b2a7bbd860c5","uri":"features/retry/retry.feature","astNodeIds":["9cd1823f-3614-4124-96d6-1a18b73e1870"],"tags":[],"name":"Test cases won't retry after failing more than the --retry limit","language":"en","steps":[{"id":"45c81fdc-a1b3-4663-9026-11bdce90d482","text":"a step that always fails","type":"Context","astNodeIds":["c41656a7-a69f-4fb1-8080-6836d386a8be"]}]}} -{"stepDefinition":{"id":"e9a027bc-c64c-4b8d-a5a2-d406442b81fd","pattern":{"source":"a step that always passes","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"uri":"features/retry/cucumber.steps.ts","location":{"line":3}}}} -{"stepDefinition":{"id":"5f68be11-8373-4060-8cb2-da975deab507","pattern":{"source":"a step that passes the second time","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"uri":"features/retry/cucumber.steps.ts","location":{"line":8}}}} -{"stepDefinition":{"id":"66feb82a-7418-48b0-babe-6035a7b548cf","pattern":{"source":"a step that passes the third time","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"uri":"features/retry/cucumber.steps.ts","location":{"line":16}}}} -{"stepDefinition":{"id":"5c256c8e-bb01-4415-a60d-0b94e4a833c8","pattern":{"source":"a step that always fails","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"uri":"features/retry/cucumber.steps.ts","location":{"line":23}}}} -{"testRunStarted":{"timestamp":{"seconds":1733310637,"nanos":993000000}}} -{"testCase":{"pickleId":"836fb5ff-9e9f-492b-8a51-d755baf3a380","id":"39ec8823-964c-4d91-ae5b-413a34937d1f","testSteps":[{"id":"bbd5533a-80e7-4d92-abb7-847a7e332c40","pickleStepId":"d9b57409-0029-402a-83e7-5455f481b8a8","stepDefinitionIds":["e9a027bc-c64c-4b8d-a5a2-d406442b81fd"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} -{"testCase":{"pickleId":"f7a858ca-0b6e-4354-96b9-fadc25c344d6","id":"0234118d-8adb-4f0d-8c65-89efec9549a9","testSteps":[{"id":"44af7034-7b52-4627-bda9-58f78f4d0af4","pickleStepId":"cf61fdaa-524c-46fc-93ad-0f37afe1d833","stepDefinitionIds":["5f68be11-8373-4060-8cb2-da975deab507"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} -{"testCase":{"pickleId":"cbda704d-db09-495e-880f-fb818bdb3e69","id":"d3872f31-db7c-4bcc-9c1c-1f5e0365508f","testSteps":[{"id":"921b8c48-ee1d-469a-b2cb-892910eb98eb","pickleStepId":"f4d91dda-9bd0-4120-a12b-671dc2792085","stepDefinitionIds":["66feb82a-7418-48b0-babe-6035a7b548cf"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} -{"testCase":{"pickleId":"299f3b17-686e-44c0-96a1-b2a7bbd860c5","id":"17194778-9da7-4673-afe8-969cdc2f156a","testSteps":[{"id":"71f56509-ee87-4690-96fa-d4385efd2738","pickleStepId":"45c81fdc-a1b3-4663-9026-11bdce90d482","stepDefinitionIds":["5c256c8e-bb01-4415-a60d-0b94e4a833c8"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} -{"testCaseStarted":{"attempt":0,"testCaseId":"39ec8823-964c-4d91-ae5b-413a34937d1f","id":"453d0af9-04e7-4459-9c8d-8190126e9063","timestamp":{"seconds":1733310637,"nanos":997000000}}} -{"testStepStarted":{"testCaseStartedId":"453d0af9-04e7-4459-9c8d-8190126e9063","testStepId":"bbd5533a-80e7-4d92-abb7-847a7e332c40","timestamp":{"seconds":1733310637,"nanos":997000000}}} -{"testStepFinished":{"testCaseStartedId":"453d0af9-04e7-4459-9c8d-8190126e9063","testStepId":"bbd5533a-80e7-4d92-abb7-847a7e332c40","testStepResult":{"duration":{"seconds":0,"nanos":469249},"status":"PASSED"},"timestamp":{"seconds":1733310637,"nanos":998000000}}} -{"testCaseFinished":{"testCaseStartedId":"453d0af9-04e7-4459-9c8d-8190126e9063","timestamp":{"seconds":1733310637,"nanos":998000000},"willBeRetried":false}} -{"testCaseStarted":{"attempt":0,"testCaseId":"0234118d-8adb-4f0d-8c65-89efec9549a9","id":"412e539c-a830-4efd-8f7b-7a66aa86a12a","timestamp":{"seconds":1733310637,"nanos":998000000}}} -{"testStepStarted":{"testCaseStartedId":"412e539c-a830-4efd-8f7b-7a66aa86a12a","testStepId":"44af7034-7b52-4627-bda9-58f78f4d0af4","timestamp":{"seconds":1733310637,"nanos":998000000}}} -{"testStepFinished":{"testCaseStartedId":"412e539c-a830-4efd-8f7b-7a66aa86a12a","testStepId":"44af7034-7b52-4627-bda9-58f78f4d0af4","testStepResult":{"duration":{"seconds":0,"nanos":115209},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:11:11)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:11:11)"}},"timestamp":{"seconds":1733310637,"nanos":999000000}}} -{"testCaseFinished":{"testCaseStartedId":"412e539c-a830-4efd-8f7b-7a66aa86a12a","timestamp":{"seconds":1733310637,"nanos":999000000},"willBeRetried":true}} -{"testCaseStarted":{"attempt":1,"testCaseId":"0234118d-8adb-4f0d-8c65-89efec9549a9","id":"e96c9de5-16c7-45d5-a89f-087c72b03b25","timestamp":{"seconds":1733310637,"nanos":999000000}}} -{"testStepStarted":{"testCaseStartedId":"e96c9de5-16c7-45d5-a89f-087c72b03b25","testStepId":"44af7034-7b52-4627-bda9-58f78f4d0af4","timestamp":{"seconds":1733310637,"nanos":999000000}}} -{"testStepFinished":{"testCaseStartedId":"e96c9de5-16c7-45d5-a89f-087c72b03b25","testStepId":"44af7034-7b52-4627-bda9-58f78f4d0af4","testStepResult":{"duration":{"seconds":0,"nanos":26166},"status":"PASSED"},"timestamp":{"seconds":1733310637,"nanos":999000000}}} -{"testCaseFinished":{"testCaseStartedId":"e96c9de5-16c7-45d5-a89f-087c72b03b25","timestamp":{"seconds":1733310637,"nanos":999000000},"willBeRetried":false}} -{"testCaseStarted":{"attempt":0,"testCaseId":"d3872f31-db7c-4bcc-9c1c-1f5e0365508f","id":"3a60ec6d-c7d5-4efb-a7ea-d7e41766da71","timestamp":{"seconds":1733310637,"nanos":999000000}}} -{"testStepStarted":{"testCaseStartedId":"3a60ec6d-c7d5-4efb-a7ea-d7e41766da71","testStepId":"921b8c48-ee1d-469a-b2cb-892910eb98eb","timestamp":{"seconds":1733310637,"nanos":999000000}}} -{"testStepFinished":{"testCaseStartedId":"3a60ec6d-c7d5-4efb-a7ea-d7e41766da71","testStepId":"921b8c48-ee1d-469a-b2cb-892910eb98eb","testStepResult":{"duration":{"seconds":0,"nanos":38791},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:19:11)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:19:11)"}},"timestamp":{"seconds":1733310638,"nanos":0}}} -{"testCaseFinished":{"testCaseStartedId":"3a60ec6d-c7d5-4efb-a7ea-d7e41766da71","timestamp":{"seconds":1733310638,"nanos":0},"willBeRetried":true}} -{"testCaseStarted":{"attempt":1,"testCaseId":"d3872f31-db7c-4bcc-9c1c-1f5e0365508f","id":"642ff9e9-afc9-4c3d-9fcc-601484cdb3e7","timestamp":{"seconds":1733310638,"nanos":0}}} -{"testStepStarted":{"testCaseStartedId":"642ff9e9-afc9-4c3d-9fcc-601484cdb3e7","testStepId":"921b8c48-ee1d-469a-b2cb-892910eb98eb","timestamp":{"seconds":1733310638,"nanos":0}}} -{"testStepFinished":{"testCaseStartedId":"642ff9e9-afc9-4c3d-9fcc-601484cdb3e7","testStepId":"921b8c48-ee1d-469a-b2cb-892910eb98eb","testStepResult":{"duration":{"seconds":0,"nanos":46624},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:19:11)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:19:11)"}},"timestamp":{"seconds":1733310638,"nanos":0}}} -{"testCaseFinished":{"testCaseStartedId":"642ff9e9-afc9-4c3d-9fcc-601484cdb3e7","timestamp":{"seconds":1733310638,"nanos":0},"willBeRetried":true}} -{"testCaseStarted":{"attempt":2,"testCaseId":"d3872f31-db7c-4bcc-9c1c-1f5e0365508f","id":"6c993d4a-8ab8-42c9-8210-9b8fbc4ff85c","timestamp":{"seconds":1733310638,"nanos":0}}} -{"testStepStarted":{"testCaseStartedId":"6c993d4a-8ab8-42c9-8210-9b8fbc4ff85c","testStepId":"921b8c48-ee1d-469a-b2cb-892910eb98eb","timestamp":{"seconds":1733310638,"nanos":0}}} -{"testStepFinished":{"testCaseStartedId":"6c993d4a-8ab8-42c9-8210-9b8fbc4ff85c","testStepId":"921b8c48-ee1d-469a-b2cb-892910eb98eb","testStepResult":{"duration":{"seconds":0,"nanos":32542},"status":"PASSED"},"timestamp":{"seconds":1733310638,"nanos":0}}} -{"testCaseFinished":{"testCaseStartedId":"6c993d4a-8ab8-42c9-8210-9b8fbc4ff85c","timestamp":{"seconds":1733310638,"nanos":0},"willBeRetried":false}} -{"testCaseStarted":{"attempt":0,"testCaseId":"17194778-9da7-4673-afe8-969cdc2f156a","id":"f330f34c-7fef-45ba-8330-2eedc619fa05","timestamp":{"seconds":1733310638,"nanos":0}}} -{"testStepStarted":{"testCaseStartedId":"f330f34c-7fef-45ba-8330-2eedc619fa05","testStepId":"71f56509-ee87-4690-96fa-d4385efd2738","timestamp":{"seconds":1733310638,"nanos":0}}} -{"testStepFinished":{"testCaseStartedId":"f330f34c-7fef-45ba-8330-2eedc619fa05","testStepId":"71f56509-ee87-4690-96fa-d4385efd2738","testStepResult":{"duration":{"seconds":0,"nanos":103667},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:24:9)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:24:9)"}},"timestamp":{"seconds":1733310638,"nanos":0}}} -{"testCaseFinished":{"testCaseStartedId":"f330f34c-7fef-45ba-8330-2eedc619fa05","timestamp":{"seconds":1733310638,"nanos":0},"willBeRetried":true}} -{"testCaseStarted":{"attempt":1,"testCaseId":"17194778-9da7-4673-afe8-969cdc2f156a","id":"e9fad387-6f3d-4d1d-b4b2-8513ede3febf","timestamp":{"seconds":1733310638,"nanos":0}}} -{"testStepStarted":{"testCaseStartedId":"e9fad387-6f3d-4d1d-b4b2-8513ede3febf","testStepId":"71f56509-ee87-4690-96fa-d4385efd2738","timestamp":{"seconds":1733310638,"nanos":0}}} -{"testStepFinished":{"testCaseStartedId":"e9fad387-6f3d-4d1d-b4b2-8513ede3febf","testStepId":"71f56509-ee87-4690-96fa-d4385efd2738","testStepResult":{"duration":{"seconds":0,"nanos":28167},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:24:9)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:24:9)"}},"timestamp":{"seconds":1733310638,"nanos":1000000}}} -{"testCaseFinished":{"testCaseStartedId":"e9fad387-6f3d-4d1d-b4b2-8513ede3febf","timestamp":{"seconds":1733310638,"nanos":1000000},"willBeRetried":true}} -{"testCaseStarted":{"attempt":2,"testCaseId":"17194778-9da7-4673-afe8-969cdc2f156a","id":"78f069ed-7958-4eb4-a50d-19b131289549","timestamp":{"seconds":1733310638,"nanos":1000000}}} -{"testStepStarted":{"testCaseStartedId":"78f069ed-7958-4eb4-a50d-19b131289549","testStepId":"71f56509-ee87-4690-96fa-d4385efd2738","timestamp":{"seconds":1733310638,"nanos":1000000}}} -{"testStepFinished":{"testCaseStartedId":"78f069ed-7958-4eb4-a50d-19b131289549","testStepId":"71f56509-ee87-4690-96fa-d4385efd2738","testStepResult":{"duration":{"seconds":0,"nanos":33125},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:24:9)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/cucumber.steps.ts:24:9)"}},"timestamp":{"seconds":1733310638,"nanos":1000000}}} -{"testCaseFinished":{"testCaseStartedId":"78f069ed-7958-4eb4-a50d-19b131289549","timestamp":{"seconds":1733310638,"nanos":1000000},"willBeRetried":false}} -{"testRunFinished":{"timestamp":{"seconds":1733310638,"nanos":1000000},"success":false}} +{"gherkinDocument":{"feature":{"tags":[],"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"Retry","description":" Some Cucumber implementations support a Retry mechanism, where test cases that fail\n can be retried up to a limited number of attempts in the same test run.\n\n Non-passing statuses other than FAILED won't trigger a retry, as they are not\n going to pass however many times we attempt them.","children":[{"scenario":{"id":"eeac5b20-e796-4903-95a3-b5025565e273","tags":[],"location":{"line":8,"column":3},"keyword":"Scenario","name":"Test cases that pass aren't retried","description":"","steps":[{"id":"cd8479a2-8bdf-4e9b-ae77-cd66cfdfb9bd","location":{"line":9,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step that always passes"}],"examples":[]}},{"scenario":{"id":"16a52c5d-0599-40a4-9fb6-45f04778fd4d","tags":[],"location":{"line":11,"column":3},"keyword":"Scenario","name":"Test cases that fail are retried if within the --retry limit","description":"","steps":[{"id":"99511fb7-bb82-4a11-8255-24166420aa7d","location":{"line":12,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step that passes the second time"}],"examples":[]}},{"scenario":{"id":"547a8047-2759-4c72-92cf-a0386172c8cd","tags":[],"location":{"line":14,"column":3},"keyword":"Scenario","name":"Test cases that fail will continue to retry up to the --retry limit","description":"","steps":[{"id":"59ca6609-3cef-44f8-888e-be2a5c98c1b1","location":{"line":15,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step that passes the third time"}],"examples":[]}},{"scenario":{"id":"4a99cac3-7b3a-4834-9737-b74ef812ee79","tags":[],"location":{"line":17,"column":3},"keyword":"Scenario","name":"Test cases won't retry after failing more than the --retry limit","description":"","steps":[{"id":"b9152274-5917-4dd5-b7ba-caa8e4f988fd","location":{"line":18,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step that always fails"}],"examples":[]}}]},"comments":[{"location":{"line":21,"column":1},"text":"# Scenario: Test cases won't retry when the status is UNDEFINED"},{"location":{"line":22,"column":1},"text":"# Given a non-existent step"}],"uri":"features/retry/retry.feature"}} +{"pickle":{"id":"983bfe72-3c91-4bb1-8720-90e624370731","uri":"features/retry/retry.feature","astNodeIds":["eeac5b20-e796-4903-95a3-b5025565e273"],"tags":[],"name":"Test cases that pass aren't retried","language":"en","steps":[{"id":"f9920632-d559-4790-a7e0-5c34a66958c0","text":"a step that always passes","type":"Context","astNodeIds":["cd8479a2-8bdf-4e9b-ae77-cd66cfdfb9bd"]}]}} +{"pickle":{"id":"64cb84c1-dac9-411a-bb63-249959a869b4","uri":"features/retry/retry.feature","astNodeIds":["16a52c5d-0599-40a4-9fb6-45f04778fd4d"],"tags":[],"name":"Test cases that fail are retried if within the --retry limit","language":"en","steps":[{"id":"d94fb52a-b506-435b-b18f-8e02efe0a126","text":"a step that passes the second time","type":"Context","astNodeIds":["99511fb7-bb82-4a11-8255-24166420aa7d"]}]}} +{"pickle":{"id":"c23030bb-e08a-4c26-a40c-6d560d145904","uri":"features/retry/retry.feature","astNodeIds":["547a8047-2759-4c72-92cf-a0386172c8cd"],"tags":[],"name":"Test cases that fail will continue to retry up to the --retry limit","language":"en","steps":[{"id":"6f66fa0e-ecdc-4c37-b9e9-9320e1a89470","text":"a step that passes the third time","type":"Context","astNodeIds":["59ca6609-3cef-44f8-888e-be2a5c98c1b1"]}]}} +{"pickle":{"id":"0786d4e7-47d0-44eb-a3af-fac4f779a61e","uri":"features/retry/retry.feature","astNodeIds":["4a99cac3-7b3a-4834-9737-b74ef812ee79"],"tags":[],"name":"Test cases won't retry after failing more than the --retry limit","language":"en","steps":[{"id":"dddaee47-c099-48f6-ac4a-be2d268856c5","text":"a step that always fails","type":"Context","astNodeIds":["b9152274-5917-4dd5-b7ba-caa8e4f988fd"]}]}} +{"stepDefinition":{"id":"6d8270b5-5e87-45c2-acad-4e624ed8c1df","pattern":{"source":"a step that always passes","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"uri":"features/fixtures.ts","location":{"line":19}}}} +{"stepDefinition":{"id":"b627542a-f210-4993-a68b-0f230e4e8f60","pattern":{"source":"a step that passes the second time","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"uri":"features/fixtures.ts","location":{"line":19}}}} +{"stepDefinition":{"id":"85944345-3a25-4fe4-858c-0acca1b8024e","pattern":{"source":"a step that passes the third time","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"uri":"features/fixtures.ts","location":{"line":19}}}} +{"stepDefinition":{"id":"9cd25aba-de67-4741-9410-62f80cd99534","pattern":{"source":"a step that always fails","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"uri":"features/fixtures.ts","location":{"line":19}}}} +{"testRunStarted":{"timestamp":{"seconds":1733479324,"nanos":966000000}}} +{"testCase":{"pickleId":"983bfe72-3c91-4bb1-8720-90e624370731","id":"71101f1d-5f94-45e5-996b-d4763fb7ba73","testSteps":[{"id":"79345e19-0e6b-462e-a454-5e16c8c7aa81","pickleStepId":"f9920632-d559-4790-a7e0-5c34a66958c0","stepDefinitionIds":["6d8270b5-5e87-45c2-acad-4e624ed8c1df"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} +{"testCase":{"pickleId":"64cb84c1-dac9-411a-bb63-249959a869b4","id":"9657ef63-7f64-48fe-8a9b-27b3058a8c7a","testSteps":[{"id":"134b997f-b09a-429b-bdae-f75ba5440d99","pickleStepId":"d94fb52a-b506-435b-b18f-8e02efe0a126","stepDefinitionIds":["b627542a-f210-4993-a68b-0f230e4e8f60"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} +{"testCase":{"pickleId":"c23030bb-e08a-4c26-a40c-6d560d145904","id":"aca0b071-d818-412a-b290-2f9bd9ee26c4","testSteps":[{"id":"cc654de8-530d-4d4a-b60c-b14a6934a284","pickleStepId":"6f66fa0e-ecdc-4c37-b9e9-9320e1a89470","stepDefinitionIds":["85944345-3a25-4fe4-858c-0acca1b8024e"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} +{"testCase":{"pickleId":"0786d4e7-47d0-44eb-a3af-fac4f779a61e","id":"02c6b682-48fa-43eb-acce-44972d5e4774","testSteps":[{"id":"96ca0406-c6b6-4e8b-8d88-bfe69d7b9580","pickleStepId":"dddaee47-c099-48f6-ac4a-be2d268856c5","stepDefinitionIds":["9cd25aba-de67-4741-9410-62f80cd99534"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} +{"testCaseStarted":{"attempt":0,"testCaseId":"71101f1d-5f94-45e5-996b-d4763fb7ba73","id":"14d7351b-b6c4-4363-baee-bd2848965972","timestamp":{"seconds":1733479324,"nanos":971000000}}} +{"testStepStarted":{"testCaseStartedId":"14d7351b-b6c4-4363-baee-bd2848965972","testStepId":"79345e19-0e6b-462e-a454-5e16c8c7aa81","timestamp":{"seconds":1733479324,"nanos":971000000}}} +{"testStepFinished":{"testCaseStartedId":"14d7351b-b6c4-4363-baee-bd2848965972","testStepId":"79345e19-0e6b-462e-a454-5e16c8c7aa81","testStepResult":{"duration":{"seconds":0,"nanos":810999},"status":"PASSED"},"timestamp":{"seconds":1733479324,"nanos":973000000}}} +{"testCaseFinished":{"testCaseStartedId":"14d7351b-b6c4-4363-baee-bd2848965972","timestamp":{"seconds":1733479324,"nanos":973000000},"willBeRetried":false}} +{"testCaseStarted":{"attempt":0,"testCaseId":"9657ef63-7f64-48fe-8a9b-27b3058a8c7a","id":"64a0489b-eb9c-422a-8583-05b9b76c952a","timestamp":{"seconds":1733479324,"nanos":974000000}}} +{"testStepStarted":{"testCaseStartedId":"64a0489b-eb9c-422a-8583-05b9b76c952a","testStepId":"134b997f-b09a-429b-bdae-f75ba5440d99","timestamp":{"seconds":1733479324,"nanos":974000000}}} +{"testStepFinished":{"testCaseStartedId":"64a0489b-eb9c-422a-8583-05b9b76c952a","testStepId":"134b997f-b09a-429b-bdae-f75ba5440d99","testStepResult":{"duration":{"seconds":0,"nanos":152624},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:17:13)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:17:13)"}},"timestamp":{"seconds":1733479324,"nanos":977000000}}} +{"testCaseFinished":{"testCaseStartedId":"64a0489b-eb9c-422a-8583-05b9b76c952a","timestamp":{"seconds":1733479324,"nanos":977000000},"willBeRetried":true}} +{"testCaseStarted":{"attempt":1,"testCaseId":"9657ef63-7f64-48fe-8a9b-27b3058a8c7a","id":"289f53c6-f4e5-4368-acf7-e55ae56a7202","timestamp":{"seconds":1733479324,"nanos":977000000}}} +{"testStepStarted":{"testCaseStartedId":"289f53c6-f4e5-4368-acf7-e55ae56a7202","testStepId":"134b997f-b09a-429b-bdae-f75ba5440d99","timestamp":{"seconds":1733479324,"nanos":977000000}}} +{"testStepFinished":{"testCaseStartedId":"289f53c6-f4e5-4368-acf7-e55ae56a7202","testStepId":"134b997f-b09a-429b-bdae-f75ba5440d99","testStepResult":{"duration":{"seconds":0,"nanos":56125},"status":"PASSED"},"timestamp":{"seconds":1733479324,"nanos":977000000}}} +{"testCaseFinished":{"testCaseStartedId":"289f53c6-f4e5-4368-acf7-e55ae56a7202","timestamp":{"seconds":1733479324,"nanos":977000000},"willBeRetried":false}} +{"testCaseStarted":{"attempt":0,"testCaseId":"aca0b071-d818-412a-b290-2f9bd9ee26c4","id":"80e3637d-bfda-412a-b73a-a81c27163afd","timestamp":{"seconds":1733479324,"nanos":977000000}}} +{"testStepStarted":{"testCaseStartedId":"80e3637d-bfda-412a-b73a-a81c27163afd","testStepId":"cc654de8-530d-4d4a-b60c-b14a6934a284","timestamp":{"seconds":1733479324,"nanos":977000000}}} +{"testStepFinished":{"testCaseStartedId":"80e3637d-bfda-412a-b73a-a81c27163afd","testStepId":"cc654de8-530d-4d4a-b60c-b14a6934a284","testStepResult":{"duration":{"seconds":0,"nanos":59417},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:31:13)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:31:13)"}},"timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testCaseFinished":{"testCaseStartedId":"80e3637d-bfda-412a-b73a-a81c27163afd","timestamp":{"seconds":1733479324,"nanos":978000000},"willBeRetried":true}} +{"testCaseStarted":{"attempt":1,"testCaseId":"aca0b071-d818-412a-b290-2f9bd9ee26c4","id":"9da5767b-3688-4f42-b9b0-73928677898b","timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testStepStarted":{"testCaseStartedId":"9da5767b-3688-4f42-b9b0-73928677898b","testStepId":"cc654de8-530d-4d4a-b60c-b14a6934a284","timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testStepFinished":{"testCaseStartedId":"9da5767b-3688-4f42-b9b0-73928677898b","testStepId":"cc654de8-530d-4d4a-b60c-b14a6934a284","testStepResult":{"duration":{"seconds":0,"nanos":73167},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:31:13)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:31:13)"}},"timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testCaseFinished":{"testCaseStartedId":"9da5767b-3688-4f42-b9b0-73928677898b","timestamp":{"seconds":1733479324,"nanos":978000000},"willBeRetried":true}} +{"testCaseStarted":{"attempt":2,"testCaseId":"aca0b071-d818-412a-b290-2f9bd9ee26c4","id":"cf7e78fe-f703-4fa3-878e-760eb09dacbc","timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testStepStarted":{"testCaseStartedId":"cf7e78fe-f703-4fa3-878e-760eb09dacbc","testStepId":"cc654de8-530d-4d4a-b60c-b14a6934a284","timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testStepFinished":{"testCaseStartedId":"cf7e78fe-f703-4fa3-878e-760eb09dacbc","testStepId":"cc654de8-530d-4d4a-b60c-b14a6934a284","testStepResult":{"duration":{"seconds":0,"nanos":30124},"status":"PASSED"},"timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testCaseFinished":{"testCaseStartedId":"cf7e78fe-f703-4fa3-878e-760eb09dacbc","timestamp":{"seconds":1733479324,"nanos":978000000},"willBeRetried":false}} +{"testCaseStarted":{"attempt":0,"testCaseId":"02c6b682-48fa-43eb-acce-44972d5e4774","id":"05d407d0-c2a2-4552-993a-7abe13b605d4","timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testStepStarted":{"testCaseStartedId":"05d407d0-c2a2-4552-993a-7abe13b605d4","testStepId":"96ca0406-c6b6-4e8b-8d88-bfe69d7b9580","timestamp":{"seconds":1733479324,"nanos":978000000}}} +{"testStepFinished":{"testCaseStartedId":"05d407d0-c2a2-4552-993a-7abe13b605d4","testStepId":"96ca0406-c6b6-4e8b-8d88-bfe69d7b9580","testStepResult":{"duration":{"seconds":0,"nanos":316125},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:37:9)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:37:9)"}},"timestamp":{"seconds":1733479324,"nanos":979000000}}} +{"testCaseFinished":{"testCaseStartedId":"05d407d0-c2a2-4552-993a-7abe13b605d4","timestamp":{"seconds":1733479324,"nanos":979000000},"willBeRetried":true}} +{"testCaseStarted":{"attempt":1,"testCaseId":"02c6b682-48fa-43eb-acce-44972d5e4774","id":"de10a3ef-7854-4e94-b2b4-d369de455d8b","timestamp":{"seconds":1733479324,"nanos":979000000}}} +{"testStepStarted":{"testCaseStartedId":"de10a3ef-7854-4e94-b2b4-d369de455d8b","testStepId":"96ca0406-c6b6-4e8b-8d88-bfe69d7b9580","timestamp":{"seconds":1733479324,"nanos":979000000}}} +{"testStepFinished":{"testCaseStartedId":"de10a3ef-7854-4e94-b2b4-d369de455d8b","testStepId":"96ca0406-c6b6-4e8b-8d88-bfe69d7b9580","testStepResult":{"duration":{"seconds":0,"nanos":48457},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:37:9)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:37:9)"}},"timestamp":{"seconds":1733479324,"nanos":980000000}}} +{"testCaseFinished":{"testCaseStartedId":"de10a3ef-7854-4e94-b2b4-d369de455d8b","timestamp":{"seconds":1733479324,"nanos":980000000},"willBeRetried":true}} +{"testCaseStarted":{"attempt":2,"testCaseId":"02c6b682-48fa-43eb-acce-44972d5e4774","id":"0d2ee87a-2a9e-4e03-8498-4a76c0bf0306","timestamp":{"seconds":1733479324,"nanos":980000000}}} +{"testStepStarted":{"testCaseStartedId":"0d2ee87a-2a9e-4e03-8498-4a76c0bf0306","testStepId":"96ca0406-c6b6-4e8b-8d88-bfe69d7b9580","timestamp":{"seconds":1733479324,"nanos":980000000}}} +{"testStepFinished":{"testCaseStartedId":"0d2ee87a-2a9e-4e03-8498-4a76c0bf0306","testStepId":"96ca0406-c6b6-4e8b-8d88-bfe69d7b9580","testStepResult":{"duration":{"seconds":0,"nanos":175083},"status":"FAILED","message":"Error: Exception in step\n at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:37:9)","exception":{"type":"Error","message":"Exception in step","stackTrace":" at World. (/Users/vitalets/projects/playwright-bdd/test/reporter-cucumber-msg/features/retry/common.steps.ts:37:9)"}},"timestamp":{"seconds":1733479324,"nanos":980000000}}} +{"testCaseFinished":{"testCaseStartedId":"0d2ee87a-2a9e-4e03-8498-4a76c0bf0306","timestamp":{"seconds":1733479324,"nanos":980000000},"willBeRetried":false}} +{"testRunFinished":{"timestamp":{"seconds":1733479324,"nanos":980000000},"success":false}} diff --git a/test/reporter-cucumber-msg/features/retry/pw.steps.ts b/test/reporter-cucumber-msg/features/retry/pw.steps.ts deleted file mode 100644 index e7d2fb77..00000000 --- a/test/reporter-cucumber-msg/features/retry/pw.steps.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Given } from '../fixtures'; - -Given('a step that always passes', function () { - // no-op -}); - -Given('a step that passes the second time', function () { - if (this.testInfo.retry < 1) { - throw new Error('Exception in step'); - } -}); - -Given('a step that passes the third time', function () { - if (this.testInfo.retry < 2) { - throw new Error('Exception in step'); - } -}); - -Given('a step that always fails', function () { - throw new Error('Exception in step'); -});