automated test validation

e2e/cypress-12-demo/currents.config.js

@@ -1,6 +1,6 @@
 module.exports = {
   e2e: {
-    batchSize: 3, // how many specs to send in one batch
+    batchSize: 4, // how many specs to send in one batch
   },
   component: {
     batchSize: 5, // how many specs to send in one batch

e2e/cypress-12-demo/cypress/e2e/a.spec.js

@@ -0,0 +1,7 @@
+describe('Failing test with 2 attempts', () => {
+  it('should try 2 times', {
+    retries: 2,
+  }, () => {
+    cy.wrap(false).should('be.true');
+  });
+});

e2e/cypress-12-demo/cypress/e2e/b.spec.js

@@ -0,0 +1,21 @@
+let attempt = 0;
+describe('Failing test with 2 attempts, passed test and flaky test with 2 attempts', () => {
+  it('should try 2 times', {
+    retries: 2,
+  }, () => {
+    cy.wrap(false).should('be.true');
+  });
+
+  it('should assert that true is true', () => {
+    cy.wrap(true).should('be.true');
+  });
+
+  it('should fail on the first attempt and pass on the second', { retries: 2 }, () => {
+    if (attempt === 0) {
+      attempt++;
+      cy.wrap(false).should('be.true');  // This will fail on the first attempt
+    } else {
+      cy.wrap(true).should('be.true');   // This will pass on the second attempt
+    }
+  });
+});

e2e/cypress-12-demo/cypress/e2e/c.spec.js

@@ -0,0 +1,5 @@
+describe('Ignored test', () => {
+  it.skip('should be a skipped test', () => {
+    cy.wrap(false).should('be.true');
+  });
+});

e2e/cypress-12-demo/cypress/e2e/d.spec.js

@@ -0,0 +1,13 @@
+describe('Passed test with screenshot, passed test with no screenshots', () => {
+  it('should assert and take a screenshot', () => {
+    // Assert that true is true
+    cy.wrap(true).should('be.true');
+
+    // Take a screenshot
+    cy.screenshot('internal-assert-screenshot');
+  });
+
+  it('should assert that true is true', () => {
+    cy.wrap(true).should('be.true');
+  });
+});

e2e/cypress-12-demo/data-references/ccy-1.10-cypress-12/currents-api-output-reference.json

@@ -0,0 +1 @@
+{"status":"OK","data":{"runId":"77cec387a5c57c87","projectId":"2cI1I5","createdAt":"2023-09-08T13:42:31.499Z","tags":[],"cypressVersion":"12.17.4","cancellation":null,"timeout":{"isTimeout":false},"groups":[{"groupId":"run-api-smoke-2023-09-08T13:42:28.508Z","platform":{"osName":"darwin","osVersion":"22.5.0","browserName":"Electron","browserVersion":"106.0.5249.51"},"createdAt":"2023-09-08T13:42:31.499Z","instances":{"overall":4,"claimed":4,"complete":4,"passes":2,"failures":2},"tests":{"overall":7,"passes":4,"failures":2,"pending":1,"skipped":0,"retries":1,"flaky":1}}],"meta":{"ciBuildId":"run-api-smoke-2023-09-08T13:42:28.508Z","commit":{"branch":"fix/cypress-13-validation","remoteOrigin":null,"authorEmail":"[email protected]","authorName":"Miguel Langarano","message":"added viewer for results comparison\n","sha":"675f830a7b1fb6ff74de7830a312bc8fb417f8ce"},"platform":{"osName":"darwin","osVersion":"22.5.0","browserName":"Electron","browserVersion":"106.0.5249.51"}},"specs":[{"groupId":"run-api-smoke-2023-09-08T13:42:28.508Z","spec":"cypress/e2e/a.spec.js","instanceId":"qMAYSIcFMtXQ","claimedAt":"2023-09-08T13:42:31.887Z","completedAt":"2023-09-08T13:42:52.389Z","machineId":"yHaFqM5V6gKh","worker":null,"results":{"videoUrl":"https://fs.currents.dev/64af8eee5411be7416b22d5f/qMAYSIcFMtXQ_tJPh3Axtz77x.mp4?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=OfUb0HaYtfK9IrP3c2-UIHT7bmm~DCGqySgHcfQmZCYLbB~W~pI3gjtJ1NvDUA1SYzL3PrbPe6UVtmhdzd0ABgsIBTwzqUdzEuMOofUDNgI7BsPongzGm86VJ~Lg5aw9ZaV861cAbrn-FAV1x5EQvYOE5HTd1PYdHsp5CdR80FEyPUvJqbFRxqkhaVjFWHdSI5kjOUoXhB3dUBMZPQM2ps1b2zTGJ99g2OZ3orwgTJJwh1dFt57-nOnWBH1iceiqnw0qmf1i-dDtP0VYXqnfC8Z4FTocNwhmDi1fsWaYhXWGrurDSUrmzUv8kDv85y~qEGrIvyzdgvvro2M1eJgZhw__","stats":{"duration":13846,"endedAt":"2023-09-08T13:42:50.598Z","startedAt":"2023-09-08T13:42:36.752Z","failures":1,"passes":0,"pending":0,"skipped":0,"suites":1,"tests":1,"wallClockDuration":13846,"wallClockStartedAt":"2023-09-08T13:42:36.752Z","wallClockEndedAt":"2023-09-08T13:42:50.598Z"},"screenshots":[{"testAttemptIndex":0,"size":349423,"takenAt":"2023-09-08T13:42:41.446Z","dimensions":{"width":2560,"height":1440},"multipart":false,"specName":"a.spec.js","testFailure":true,"scaled":true,"blackout":[],"duration":304,"testId":"r0","name":"screenshot","screenshotId":"OIFkUXw8B2U8OqJAdZ0o1","screenshotURL":"https://fs.currents.dev/64af8eee5411be7416b22d5f/ZlKLfoSVgYCc.png?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=a1sVvwQiIT8GjfjP60bArZsnt1IebqEkbOqTKL-Xu1v2XL3dcFbJoOMORT1xqieGVa5Gs2qgWMJwxQ7yUg9UiEmIJ4bMHowVNm41DYEPH9zeM7ZxSBwonCixdMboQY2C2LSqjTjaJhogvFLPerAdlq7ZijB3RS9c8tEndwe5xW2DgpmRdCMhG~Ugu6jQjprvzCbCUQOL4czb3AHJv9o51rzandeUBUnIx7ExZ9H20JJUMvVQOoJ4IipfFFqpy5W~AHrhgZR8MCD1pfCIE7HbvB0btm6OXudUKdIxFebbj7aaZdoM6mXju3N~u-8rMzZLE6lJTggMui6zj-8B4gJEiQ__","height":0,"width":0},{"testAttemptIndex":1,"size":360089,"takenAt":"2023-09-08T13:42:45.908Z","dimensions":{"width":2560,"height":1440},"multipart":false,"specName":"a.spec.js","testFailure":true,"scaled":true,"blackout":[],"duration":214,"testId":"r0","name":"screenshot","screenshotId":"5BJFDWKXTwbubxcj1owJc","screenshotURL":"https://fs.currents.dev/64af8eee5411be7416b22d5f/7DEScNt5RYar.png?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=jWIwCe42mSjR~2ZTEPdkvqzQTH6pidjOPERzDf45hB9Rfw0caJz19mO9LGWDzl6ROQs22pgUYj754EFQrs-6klMYH~1RA162ajPujkraCKvKh0r0mcUjI7XMbtJbH-2ptHFjVwbT~hks7v2WXWY1Z6BEdkBWFQc~9QcksRpdD6GlpdFdEoTZJOqA4DAGDog8oLsf0vy3KjwBMUmBGg-Ss8l9xkK9kmewK-x3JLCSpc~edwI3zDGenxYadT7ZjTI--lBuRCuo0s8bPwHLM87XpvRXwicT~J5Sx7XZeyeJHruA85f4kB8NCxeeeQ22hFNCZeru~QGVil5SpSXnjjTo4g__","height":0,"width":0},{"testAttemptIndex":2,"size":363302,"takenAt":"2023-09-08T13:42:50.334Z","dimensions":{"width":2560,"height":1440},"multipart":false,"specName":"a.spec.js","testFailure":true,"scaled":true,"blackout":[],"duration":206,"testId":"r0","name":"screenshot","screenshotId":"QN2J2LHZGeuCrOyxOsQsG","screenshotURL":"https://fs.currents.dev/64af8eee5411be7416b22d5f/iFNMNa1CHXLX.png?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=FgBKvyaLOOmlB6HwQSaO2-2PbHgv6Ngn9Ot6zP~KoHBq6KsH3NIbq9QV97RIRwrzB5wPGCAoNj-rmxYhhO5iSFlXrjtDtHjklibsI6Qbh7BhVI7FaCizIHI7ojTS02xVaCmSTP55jHbKkosRjZo1mVFJUq2ra~Ob8O1ZbFUnVuLVyURUNPrOrfhjs~efoemi1dz6ecYIBkqGybDe82vhyhcmSFzu6G1gfpy1pv0kq4Vp4YK70VCDkG64btpLcvmMKkvpznhGjRiR7PaamYy7WL8UQriVZKD5QkAQsl8yh6Ms2T3zjnsYbQUhkVAt65D2l20uKNs4WGmnykr9tyKqnQ__","height":0,"width":0}],"exception":null,"flaky":0}},{"groupId":"run-api-smoke-2023-09-08T13:42:28.508Z","spec":"cypress/e2e/b.spec.js","instanceId":"qdTJDmiKQ2wa","claimedAt":"2023-09-08T13:42:31.904Z","completedAt":"2023-09-08T13:43:22.717Z","machineId":"yHaFqM5V6gKh","worker":null,"results":{"videoUrl":"https://fs.currents.dev/64af8eee5411be7416b22d5f/qdTJDmiKQ2wa_ukHle2cAQBAs.mp4?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=XWbRzVvrvQm9SGhIYTQB8o-8d1T841TkfYYLvY6u0qrZXgI9BzK4T96QLjbcXgJfClDuYIZKI0lsVtYWg-vVoKixOmSnY7AGLQZ5bcnVGJ93lh99aGt6v2~wnGeSZkdrBNtgc4SZ0rfidmoDnebbH3qrOP8F7pjKLosL3RMHIOGPTovJvfaUyEk07ZEPKAjj4PUrCyOs1aiIHnNItg9mamv4EVdMUfj0Gh261d4dZRJJzvliBZlqISNScpEbXCi9UGSH0PveVR~eI8bcXy3~n8~TcxkcV5cjD781XNhVNq2Ji53IWyfchaEGcwxjvkj-ZgA2~t0BsN8WCNTn~Hs--g__","stats":{"duration":18212,"endedAt":"2023-09-08T13:43:20.716Z","startedAt":"2023-09-08T13:43:02.504Z","failures":1,"passes":2,"pending":0,"skipped":0,"suites":1,"tests":3,"wallClockDuration":18212,"wallClockStartedAt":"2023-09-08T13:43:02.504Z","wallClockEndedAt":"2023-09-08T13:43:20.716Z"},"screenshots":[{"testAttemptIndex":0,"size":374162,"takenAt":"2023-09-08T13:43:06.785Z","dimensions":{"width":2560,"height":1440},"multipart":false,"specName":"b.spec.js","testFailure":true,"scaled":true,"blackout":[],"duration":258,"testId":"r0","name":"screenshot","screenshotId":"QgWsuBQRTZMQOrhnro_ro","screenshotURL":"https://fs.currents.dev/64af8eee5411be7416b22d5f/horBZ3DuOMGr.png?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=ZvoziF30y~hzSoHR3rVM2YPJbz0KtwR1XInqIWj2aD5Q4HBalxHaWj9VmNc9Y3nGEMzaNIDkDg7mfeH73eZ5f9xZUQj9jPVjELojuHTlNaDNv98M7Ov1btFE2wS7zYWJh7wVmbTWh-MR8kNzRBHt5mWrO3PnZMY8RQDjt02MdIOPk-tYjUeQaVHcEHBbapy3G1gbwLcp-XinsZ32mO24Fqnh6avxHfBR4E1I1Flh25Uiyy0wQcgWgg6eXnqXcYjEmaLHqPsI0Amg8Purk5duteZUF0Cu9WTVwL6TI7UawutSwQFWHvEs3hQc4zEtYGD7OS7teBRQhs8cJJ7sA6UoHw__","height":0,"width":0},{"testAttemptIndex":1,"size":377830,"takenAt":"2023-09-08T13:43:11.191Z","dimensions":{"width":2560,"height":1440},"multipart":false,"specName":"b.spec.js","testFailure":true,"scaled":true,"blackout":[],"duration":281,"testId":"r0","name":"screenshot","screenshotId":"iYc4VXEvkJ5Z8TbRDWsZd","screenshotURL":"https://fs.currents.dev/64af8eee5411be7416b22d5f/3w09MBuTuFWG.png?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=LX5yKw0zEy3w66Ca1DC1QY-Twoler9DEp6qffW18oXeZj0Q-ke9G8z~BFXpErD5QTXJLgHjWaqPm6JObfcoQ7K4~8W9Bx6tjaPeVDcE~x9ZQ5wajx48eJE-eifCIpb6xzPufyfSMx7b~qqKgvo6dv0v1bke700UQ7cMotm1r-ky~JzF4Mtfbo-opuBhbMki0y7tAiDQqOAk1lk9~17pdYDK1NF17ycKmj~gMzAWaroMwX-HiOdV-6NAFNnm0ZFUZ-7aoIEiD3U8PocmQvnKdhu40Ptvv3pHkyyp8OQuzyG12JASt5~jcV4NmYvqIMu92u57a~cl8kkmyUyAL~3gK8Q__","height":0,"width":0},{"testAttemptIndex":2,"size":375503,"takenAt":"2023-09-08T13:43:15.688Z","dimensions":{"width":2560,"height":1440},"multipart":false,"specName":"b.spec.js","testFailure":true,"scaled":true,"blackout":[],"duration":233,"testId":"r0","name":"screenshot","screenshotId":"N7VLnS5SoBxKEdpk7dOYp","screenshotURL":"https://fs.currents.dev/64af8eee5411be7416b22d5f/u22UHn3JZ0Gc.png?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=PbufeWtMwI0KP6iJwJrQJFvAtk9kbpXibGoSIQJ8u9sD2W3w~mYf0UdhcwdHjVeqjgujG-z58GKOpMTpYsSdvu3VGEFl7j4b4bmWUBNNmmwU80mEEN-CNbWzUj70cxX5ZjX7RsTNBDptiFL03aAMBV1RjoRa4d6oKVHNqFVwrMgPzZKaiRKU7lSsWjxb89r3m4PEWluDidKmP-vQ2sdv3qorZa~BGumiIRI9-aPdORky6~QlTH3lNNcfw-uVQs3w51mq1IZjO7shnlhs-HLYXAv3tgGqsYJRAbCvpzvngtUEnqHKYCVMTy7-V0ZJuAB0XawyABIH~ds9MJgRL5CqaA__","height":0,"width":0},{"testAttemptIndex":0,"size":378982,"takenAt":"2023-09-08T13:43:20.250Z","dimensions":{"width":2560,"height":1440},"multipart":false,"specName":"b.spec.js","testFailure":true,"scaled":true,"blackout":[],"duration":209,"testId":"r2","name":"screenshot","screenshotId":"0FcsNENWstOxPmSzmcyDS","screenshotURL":"https://fs.currents.dev/64af8eee5411be7416b22d5f/bVytB6veRTaW.png?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=I5kRRfAu~qmfL6u8z8hrzz95Neg5Rhy8FyvPeMnp99ApitxlcL6sb-CWX3PJiijxWKb19RwDUHGJ7FINKEHSZSQJNG1KONd641odmnrArHoiijAnVFqK-lc5xbrA7R~YfrxgxTl~ScohGSi7zFcqG0jt~NCcNq4smTdrLwb8zVt-M9MXuNvXHX9spjw1cAz3UrDjEtp6~oeDfTudKGgIwWZdyUidv0UecyOoyWWP3KwDgckyRMXJrz79nLzeah87IVyzCrU4K~JDimNQzpWV58tPJ2JUbrKhyNSepn~swbk7f0haPWjhzJFQe7DMX4Md7CFYhLJVVy-k0ZERLkwwgQ__","height":0,"width":0}],"exception":null,"flaky":1}},{"groupId":"run-api-smoke-2023-09-08T13:42:28.508Z","spec":"cypress/e2e/c.spec.js","instanceId":"YJXj6GeHgi9Z","claimedAt":"2023-09-08T13:42:31.898Z","completedAt":"2023-09-08T13:43:01.593Z","machineId":"yHaFqM5V6gKh","worker":null,"results":{"videoUrl":"https://fs.currents.dev/64af8eee5411be7416b22d5f/YJXj6GeHgi9Z_3az7f5VZ0AUl.mp4?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=r31DkISnKQrj5va9ZeXFwGvdeDV~T~P4FKSzBWyeaNSrH-3wzAb7Xk6ZcT8w7X~2Up0EVIy6FuPWX6Xytnix-LL3tuZZtiMRE61LvnoZtPcURzln-HTJc-jQWKjO8aamqwTaZC~imnkudC-SMLjMUrbAgNtdNiHBgamQdVujmmH2-PIEUMHwmcqsqsXR4nt6~qDLXdrf~xpWLN6p1aTjbCAJ4alkvo13vEN6i2rJXyKWqr~XZbvuNQBL~Pkhq7fkPBZTZcOBbIm4ztXh3AUEPHLA8i8Bbf6Di6DejJsUeov2vvkA5H9y9NJfuwlKmKWoOO9iRsc3LqapF0t8JCux8Q__","stats":{"duration":67,"endedAt":"2023-09-08T13:43:00.018Z","startedAt":"2023-09-08T13:42:59.951Z","failures":0,"passes":0,"pending":1,"skipped":0,"suites":1,"tests":1,"wallClockDuration":67,"wallClockStartedAt":"2023-09-08T13:42:59.951Z","wallClockEndedAt":"2023-09-08T13:43:00.018Z"},"screenshots":[],"exception":null,"flaky":0}},{"groupId":"run-api-smoke-2023-09-08T13:42:28.508Z","spec":"cypress/e2e/d.spec.js","instanceId":"Mzlt7nMYwIdn","claimedAt":"2023-09-08T13:42:31.892Z","completedAt":"2023-09-08T13:42:58.850Z","machineId":"yHaFqM5V6gKh","worker":null,"results":{"videoUrl":"https://fs.currents.dev/64af8eee5411be7416b22d5f/Mzlt7nMYwIdn_KLFYL9MMJTat.mp4?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=InSU3itmKaS4fua2YC1Tz8FLKmg-~ilVxh5LzwiTzHKdgADnAIz35i8xz-pA-GB9fncy6C3zprnrJuBgtDSLi-F25bcUTJsRV~yYZbV9tIaJJDqdM~Ny3a4DGDEIUP~5-MbsuZg~r5F7QPMQ95QV88l1Dv-VJnDnGNeo~IlvkaZOWicdkuh1A0NKnPLuxiXRnpnWQM2roQeICgngp75Y9bxzbeQ2wyd4~Vp0Yj0VOBN~NPY-7cUZZ9Z60Hy2MF0HK3fZrYJf~1CvLm7Rn0XLCL2qrgqR5dT~jrVyz5znCXacjD5JpbYe7uVFOFDaOcKnisrtyaaRLiAG2ppW4GHxXg__","stats":{"duration":1108,"endedAt":"2023-09-08T13:42:57.027Z","startedAt":"2023-09-08T13:42:55.919Z","failures":0,"passes":2,"pending":0,"skipped":0,"suites":1,"tests":2,"wallClockDuration":1108,"wallClockStartedAt":"2023-09-08T13:42:55.919Z","wallClockEndedAt":"2023-09-08T13:42:57.027Z"},"screenshots":[{"testAttemptIndex":0,"size":234576,"takenAt":"2023-09-08T13:42:56.156Z","dimensions":{"width":2000,"height":1320},"multipart":true,"pixelRatio":2,"name":"internal-assert-screenshot","specName":"d.spec.js","scaled":false,"blackout":[],"duration":732,"testId":"r0","screenshotId":"x2G5Ca6DBxNl3zzjv9EDG","screenshotURL":"https://fs.currents.dev/64af8eee5411be7416b22d5f/7Dehjgyld5N2.png?Expires=1694439797&Key-Pair-Id=K1ZNDCCIZ3P4FU&Signature=vh~0BbcrzKw6aFw36cTA5LBHwkxgprOw0ChNElg69NFNlrZI2nIVEORd4ykB4~U8OpAnRUTg4Qr1Xtyk2vWgSyuf7L3PDJwHnCaz7v~DMVqCBemdr0MllYEIMc6lFjsEu3-5HS6~lN3SFzmboZj4~ea3fJ4AF2SWu3nChI7TacHXEEUMW26A5B8A-qrnNFfngf8vcYoKiNL2pTsuPQbU0kyZEBWRybcySBnsV4NimJYX45jK-5LgBuVtS3ZADyM~NF1NzAC8-ZwG55J6Z5B45Zrtj0pyLNPXhR6jnGSxJMAOhT9XWYGeWo453OVEfe-3pzZBLKj3A7M6ai2AcToIRg__","height":0,"width":0}],"exception":null,"flaky":0}}],"completionState":"COMPLETE","status":"FAILED"}}