Run E2E tests in shards (chromium & firefox) (#1662)

.github/workflows/e2e-tests.yml

@@ -0,0 +1,42 @@
+name: Run E2E tests
+
+on:
+  workflow_call:
+
+jobs:
+  e2e-tests:
+    name: ${{matrix.browser}} (${{ matrix.shard }}/${{ matrix.shards }})
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [chromium, firefox]
+        shard: [1, 2, 3, 4, 5, 6]
+        shards: [6]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      -
+        uses: actions/checkout@v3
+      -
+        name: Playwright E2E Tests
+        run: make e2e-tests-ci browser=${{ matrix.browser }} shard="${{ matrix.shard }}/${{ matrix.shards }}"
+      -
+        name: Upload Playwright test results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-${{ matrix.browser }}-${{ matrix.shard }}
+          path: test/e2e/test-results
+          if-no-files-found: error
+
+  e2e-tests-result:
+    if: always()
+    runs-on: ubuntu-latest
+    needs: e2e-tests
+    steps:
+      - name: Check aggregated E2E result
+        if: ${{ needs.e2e-tests.result != 'success' }}
+        run: |
+          echo "Some E2E tests failed"
+          exit 1

.github/workflows/integrate-and-deploy.yml

@@ -74,17 +74,6 @@ jobs:
       -
         name: Unit Tests
         run: make unit-tests-ci
-      -
-        name: Playwright E2E Tests
-        run: make e2e-tests-ci
-      -
-        name: Upload Playwright test results
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results
-          path: test/e2e/test-results
-          if-no-files-found: error
       -
         name: Log in to Docker Hub
         uses: docker/login-action@v2
@@ -109,10 +98,13 @@ jobs:
       IMAGE_NEXT_APP: ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG_NEXT_APP }}
       IMAGE_LFMERGE: ${{ steps.image.outputs.LFMERGE_NAMESPACE }}:${{ steps.image.outputs.TAG_LFMERGE }}
 
+  e2e-tests:
+    uses: ./.github/workflows/e2e-tests.yml
+
   deploy:
     runs-on: [self-hosted, languageforge]
 
-    needs: integrate
+    needs: [integrate, e2e-tests]
 
     steps:
       -

.github/workflows/pull-request.yml

@@ -54,45 +54,8 @@ jobs:
           github_token: ${{ github.token }}
           junit_files: PhpUnitTests.xml
 
-  build-app-run-e2e-tests:
-    runs-on: ubuntu-latest
-
-    steps:
-      -
-        uses: actions/checkout@v3
-      -
-        uses: actions/setup-node@v3
-        with:
-          node-version: '16.14.0'
-          cache: 'npm'
-      -
-        name: Get installed Playwright version
-        id: playwright-version
-        run: echo -n "version=$(npm ls @playwright/test --json | jq --raw-output '.version')" >> $GITHUB_OUTPUT
-      -
-        name: Cache playwright browsers
-        uses: actions/cache@v3
-        id: playwright-cache
-        with:
-          path: '~/.cache/ms-playwright'
-          # Make each playwright version use its own cache in case it uses different browser versions
-          # Cache entries are deleted if not accessed for 7 days
-          # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy
-          key: 'playwright-${{ steps.playwright-version.outputs.version }}'
-
-      -
-        name: Playwright E2E Tests
-        # see https://playwright.dev/docs/ci#github-actions
-        run: make e2e-tests-ci
-
-      -
-        name: Upload Playwright test results
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results
-          path: test/e2e/test-results
-          if-no-files-found: error
+  e2e-tests:
+    uses: ./.github/workflows/e2e-tests.yml
 
   check-code-formatting:
     runs-on: ubuntu-latest

Makefile

@@ -16,14 +16,14 @@ ui-builder:
 e2e-tests-ci:
 	npm ci
 	$(MAKE) e2e-app
-	npx playwright install chromium
-	npx playwright test -c ./test/e2e/playwright.config.ts
+	npx playwright install ${browser} --with-deps
+	npx playwright test -c ./test/e2e/playwright.config.ts --project=${browser} --shard=${shard}
 
 .PHONY: e2e-tests
 e2e-tests: ui-builder
 	npm install
 	$(MAKE) e2e-app
-	npx playwright install chromium
+	npx playwright install chromium firefox
 	npx playwright test -c ./test/e2e/playwright.config.ts $(params)
 
 .PHONY: e2e-app

test/e2e/fixtures.ts

@@ -2,6 +2,7 @@ import type { Browser, Page } from '@playwright/test';
 import { test as base, APIRequestContext } from '@playwright/test';
 import { users } from './constants';
 import { getStorageStatePath, Project, ProjectTestService, UserDetails, UserTestService } from './utils';
+import { TestProject } from './utils/types';
 
 export type Tab = Page;
 export type E2EUsername = keyof typeof users;
@@ -75,3 +76,20 @@ export function projectPerTest(lazy?: boolean): () => Project | Promise<Project>
     return () => currentTestProject;
   }
 }
+
+export function defaultProject(): TestProject {
+
+  let project: Project;
+  let entryIds: string[];
+
+  test.beforeAll(async ({ projectService }, testInfo) => {
+    const defaultProject = await projectService.createDefaultProject(testInfo);
+    project = defaultProject.project();
+    entryIds = defaultProject.entryIds();
+  });
+
+  return {
+    project: () => project,
+    entryIds: () => entryIds,
+  };
+}

test/e2e/global-setup.ts

@@ -1,4 +1,4 @@
-import { chromium, firefox, FullConfig, Page, webkit } from '@playwright/test';
+import { BrowserType, chromium, firefox, FullConfig, Page, webkit } from '@playwright/test';
 import * as fs from 'fs';
 import { appUrl, users } from './constants';
 import { getStorageStatePath, login, UserDetails, UserTestService } from './utils';
@@ -20,22 +20,37 @@ async function initE2EUser(page: Page, user: UserDetails) {
   await context.storageState({ path });
 }
 
+/**
+ * @returns The first project browser that is installed (different CI jobs use/install different browsers)
+ */
+function findInstalledBrowser(config: FullConfig): BrowserType {
+  const browserTypes = config.projects.map((project) => {
+    const browserType = { chromium, firefox, webkit }[project.use.defaultBrowserType];
+    return {
+      browserType,
+      installed: fs.existsSync(browserType.executablePath()),
+    }
+  });
+
+  return browserTypes.find((browser) => browser.installed).browserType;
+}
+
 export default async function globalSetup(config: FullConfig) {
   console.log('Starting global setup\n');
   console.time('Global setup took');
 
   try {
-    const use = config.projects[0].use;
-    const browserType = { chromium, firefox, webkit }[use.defaultBrowserType];
+    const browserType = findInstalledBrowser(config);
     const browser = await browserType.launch();
+
     for (const user of Object.values(users)) {
       const context = await browser.newContext({ baseURL: appUrl });
       const page = await context.newPage();
       await initE2EUser(page, user);
       await context.close();
     }
   } catch (error) {
-    console.warn(`Error in Playwright global setup: ${error}.`);
+    console.error(`Error in Playwright global setup: ${error}.`);
     throw error;
   }
 

test/e2e/pages/base-page.ts

@@ -67,10 +67,8 @@ export abstract class BasePage
    * @returns the page for convenience/chaining
    */
   async reload(): Promise<this> {
-    await Promise.all([
-      this.page.reload(),
-      this.waitFor(),
-    ]).catch(error => {
+    await this.page.reload();
+    await this.waitFor().catch(error => {
       console.error(error);
       throw error;
     });

test/e2e/pages/configuration-fields.tab.ts

@@ -17,7 +17,7 @@ export class ConfigurationPageFieldsTab extends ConfigurationPage {
     return row.locator('css=td,th').nth(columnIndex).locator('input');
   }
 
-  async toggleField(tableTitle: string, field: string): Promise<void> {
+  async toggleFieldExpanded(tableTitle: string, field: string): Promise<void> {
     const row: Locator = this.getRow(this.getTable(tableTitle), field);
     await row.locator('.field-specific-btn').click();
   }

test/e2e/pages/editor.page.ts

@@ -28,11 +28,6 @@ export class EditorPage extends BasePage {
   };
   readonly renderedDivs = this.locator('.dc-rendered-entryContainer');
 
-  readonly search = {
-    searchInput: this.locator('#editor-entry-search-entries'),
-    matchCount: this.locator('#totalNumberOfEntries >> span')
-  };
-
   readonly noEntries = this.locator('.no-entries');
   readonly entryCard = this.locator('.entry-card');
   readonly senseCard = this.locator('.dc-sense.card');
@@ -178,20 +173,16 @@ export class EditorPage extends BasePage {
     return card.locator('[data-ng-switch-when="pictures"]');
   }
 
-  // returns the locator to the picture or undefined if 0 or more than one pictures with this filename are found
-  async getPicture(card: Locator, filename: string): Promise<Locator> {
-    const picture: Locator = card.locator(`img[src$="${filename}"]`);
-    return (await picture.count() == 1 ? picture : undefined);
+  picture(filename: string, parent = this.senseCard): Locator {
+    return parent.locator(`img[src$="${filename}"]`);
   }
 
-  async getPictureDeleteButton(card: Locator, pictureFilename: string): Promise<Locator> {
-    const button = card.locator(`[data-ng-repeat="picture in $ctrl.pictures"]:has(img[src$="${pictureFilename}"]) >> [title="Delete Picture"]`);
-    return (await button.count() == 1 ? button : undefined);
+  deletePictureButton(pictureFilename: string, parent = this.senseCard): Locator {
+    return parent.locator(`[data-ng-repeat="picture in $ctrl.pictures"]:has(img[src$="${pictureFilename}"]) >> [title="Delete Picture"]`);
   }
 
-  async getPictureCaption(picture: Locator, languageCode: string = "en"): Promise<Locator> {
-    const caption = picture.locator(`xpath=..//.. >> div.input-group:has-Text("${languageCode}") >> textarea`);
-    return (await caption.count() == 1 ? caption : undefined);
+  caption(picture: Locator, languageCode: string = "en"): Locator {
+    return picture.locator(`xpath=..//.. >> div.input-group:has-Text("${languageCode}") >> textarea`);
   }
 
   getCancelDropboxButton(card: Locator, uploadType: UploadType): Locator {

test/e2e/pages/entry-list.page.ts

@@ -3,11 +3,13 @@ import { Project } from '../utils';
 import { BasePage } from './base-page';
 
 export class EntryListPage extends BasePage {
-  readonly totalNumberOfEntries = this.locator('#totalNumberOfEntries');
+  readonly entries = this.locator('.lexiconItemListContainer').locator('.lexiconListItem, .lexiconListItemCompact');
   readonly filterInput = this.locator('[placeholder="Search"]');
   readonly filterInputClearButton = this.locator('.clear-search-button');
   readonly matchCount = this.locator('#totalNumberOfEntries >> span');
-  readonly createNewWordButton = this.locator('#newWord:visible, #noEntriesNewWord:visible');
+  readonly createNewWordButton = this.locator('#newWord:visible, #noEntriesNewWord:visible, #editorNewWordBtn:visible');
+
+  private readonly totalNumberOfEntries = this.locator('#totalNumberOfEntries');
 
   entry(lexeme: string): Locator {
     return this.locator(`.lexiconListItem:visible:has(span:has-text("${lexeme}"))`);

test/e2e/playwright.config.ts

@@ -34,8 +34,9 @@ const config: PlaywrightTestConfig = {
   globalSetup: require.resolve('./global-setup'),
   /* Retry on CI only */
   retries: process.env.CI ? 1 : 0,
-  /* Opt out of parallel tests. Our current state management prevents a user from working on different projects simultaneously. */
-  workers: process.env.CI ? 1 : 1,
+  /* Our current state management prevents an LF-user from working on different projects simultaneously.
+  Instead, we use Playwright's sharding feature to parallelize tests accross multiple environments. */
+  workers: 1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   outputDir: 'test-results', // referenced in pull-request.yml
   reporter: process.env.CI
@@ -70,12 +71,12 @@ const config: PlaywrightTestConfig = {
       },
     },
 
-    // {
-    //   name: 'firefox',
-    //   use: {
-    //     ...devices['Desktop Firefox'],
-    //   },
-    // },
+    {
+      name: 'firefox',
+      use: {
+        ...devices['Desktop Firefox'],
+      },
+    },
 
     // {
     //   name: 'webkit',