(#490) Add an automated production sanity test (#792)

* Add playwright production sanity test

* Post-review updates

README.md

@@ -41,6 +41,7 @@ Check out the project's respective readmes:
 
 * [Backend Readme](backend/README.md)
 * [Frontend Readme](frontend/README.md)
+* [E2E Tests Readme](test/e2e/README.md)
 
 ### Localization
 

frontend/src/components/NavBar.vue

@@ -67,7 +67,6 @@ const isNavEntryActive = (item: string) => {
           :active="isNavEntryActive(item)"
           :label="t(`label.${item}`)"
           :link-name="item"
-          :data-testid="'nav-bar-' + item + '-btn'"
         />
       </div>
       <drop-down class="self-center">

frontend/src/views/LoginView.vue

@@ -214,9 +214,9 @@ const onEnter = () => {
       </div>
       <div class="form-body">
         <form v-if="loginStep !== LoginSteps.SignUpConfirm" class="form" ref="formRef" autocomplete="off" @submit.prevent @keyup.enter="() => onEnter()">
-          <text-input name="email" v-model="email" :required="true">{{ t('login.form.email') }}</text-input>
-          <text-input v-if="isPasswordAuth" name="password" v-model="password" :required="true" type="password">{{ t('label.password') }}</text-input>
-          <text-input v-if="loginStep === LoginSteps.SignUp && !hideInviteField" name="inviteCode" v-model="inviteCode" :help="t('login.form.no-invite-code')">{{ t('label.inviteCode') }}</text-input>
+          <text-input name="email" v-model="email" :required="true" data-testid="login-email-input">{{ t('login.form.email') }}</text-input>
+          <text-input v-if="isPasswordAuth" name="password" v-model="password" :required="true" type="password" data-testid="login-password-input">{{ t('label.password') }}</text-input>
+          <text-input v-if="loginStep === LoginSteps.SignUp && !hideInviteField" name="inviteCode" v-model="inviteCode" :help="t('login.form.no-invite-code')" data-testid="login-invite-code-input">{{ t('label.inviteCode') }}</text-input>
         </form>
       </div>
       <template v-slot:actions>
@@ -226,6 +226,7 @@ const onEnter = () => {
           :disabled="isLoading"
           @click="onEnter()"
           v-if="loginStep !== LoginSteps.SignUpConfirm"
+          data-testid="login-continue-btn"
         >
           {{ t('label.continue') }}
         </primary-button>
@@ -235,6 +236,7 @@ const onEnter = () => {
           :disabled="isLoading"
           @click="router.push({name: 'home'})"
           v-else
+          data-testid="login-close-btn"
         >
           {{ t('label.close') }}
         </primary-button>

test/e2e/.env.example

@@ -0,0 +1,8 @@
+# Appointment E2E Test Configuration
+
+# URLs
+APPT_PROD_URL=https://appointment.day/
+
+# Production sign-in (FxA) credentials
+APPT_PROD_LOGIN_EMAIL=
+APPT_PROD_LOGIN_PWORD=

test/e2e/.gitignore

@@ -0,0 +1,6 @@
+.env
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

test/e2e/README.md

@@ -0,0 +1,86 @@
+# Thunderbird Appointment E2E Tests
+
+Guide for running the Thunderbird Appointment E2E tests.
+
+## Prerequisite
+
+You must have a pre-existing Appointment user test account (using FxA credentials) on the platform where you are running the tests. ie. For the production sanity test you must have an Appointment test account on production (using production FxA credentials) already set up.
+
+## Installation
+
+First install the E2E suite (includes Playwright):
+
+```bash
+cd test/e2e
+npm install
+```
+
+Next install the Playwright browsers (Playwright uses it's own bundled browers) still in `test/e2e`:
+
+```bash
+npx playwright install
+```
+
+## Running Locally
+
+The E2E tests require credentials for an existing Appointment (FxA) account and reads these from your local env vars. First copy over the provided `.example.env` to a local `.env`:
+
+```bash
+cd test/e2e
+cp .env.example .env
+```
+
+Then edit your local `.env` file and provide the credentials for your Appointment test account:
+```dotenv
+APPT_PROD_LOGIN_EMAIL=<existing-test-FxA-user-email>
+APPT_PROD_LOGIN_PWORD=<exisiting-test-FxA-user-password>
+```
+
+To run the production sanity test headless (still in `test/e2e`):
+
+```bash
+npm run prod-sanity-test
+```
+
+To run the production sanity test with a UI so you can watch the tests run (still in `test/e2e`):
+
+```bash
+npm run prod-sanity-test-ui
+```
+
+To run the production sanity test in debug mode (still in `test/e2e`):
+
+```bash
+npm run prod-sanity-test-debug
+```
+
+## Running on BrowserStack
+
+You can run the E2E tests from your local machine but against browsers provided in the BrowserStack Automate cloud.
+
+<b>For security reasons when running the tests on BrowserStack I recommend that you use a dedicated test Appointment FxA account / credentials (NOT your own personal Appointment (FxA) credentials).</b>
+
+Once you have credentials for an existing Appointemnt test account, edit your local `.env` file and add the credentials:
+
+```dotenv
+APPT_PROD_LOGIN_EMAIL=<existing-test-FxA-user-email>
+APPT_PROD_LOGIN_PWORD=<exisiting-test-FxA-user-password>
+```
+
+Also in order to run on BrowserStack you need to provide your BrowserStack credentials. Sign into your BrowserStack account and navigate to your `User Profile` and find your auth username and access key. In your local terminal export the following env vars to set the BrowserStack credentials that the tests will use:
+
+```bash
+export BROWSERSTACK_USERNAME=<your-browserstack-user-name>
+```
+
+```bash
+export BROWSERSTACK_ACCESS_KEY=<your-browserstack-access-key>
+```
+
+To run the production sanity test on BrowserStack (still in `test/e2e`):
+
+```bash
+npm run prod-sanity-test-browserstack
+```
+
+After the tests finish in your local console you'll see a link to the BrowserStack test session; when signed into your BrowserStack account you'll be able to use that link to see the test session results including video playback.

test/e2e/browserstack.yml

@@ -0,0 +1,75 @@
+# =============================
+# Set BrowserStack Credentials
+# =============================
+# In the terminal export BROWSERSTACK_USERNAME=<secret>
+# In the terminal export BROWSERSTACK_ACCESS_KEY=<secret>
+
+# ======================
+# BrowserStack Reporting
+# ======================
+# The following capabilities are used to set up reporting on BrowserStack:
+# Set 'projectName' to the name of your project. Example, Marketing Website
+projectName: Thunderbird Appointment
+# Set `buildName` as the name of the job / testsuite being run
+buildName: Build
+# `buildIdentifier` is a unique id to differentiate every execution that gets appended to
+# buildName. Choose your buildIdentifier format from the available expressions:
+# ${BUILD_NUMBER} (Default): Generates an incremental counter with every execution
+# ${DATE_TIME}: Generates a Timestamp with every execution. Eg. 05-Nov-19:30
+# Read more about buildIdentifiers here -> https://www.browserstack.com/docs/automate/selenium/organize-tests
+buildIdentifier: '#${BUILD_NUMBER}' # Supports strings along with either/both ${expression}
+
+# =======================================
+# Platforms (Browsers / Devices to test)
+# =======================================
+# Platforms object contains all the browser / device combinations you want to test on.
+# Entire list available here -> (https://www.browserstack.com/list-of-browsers-and-platforms/automate)
+platforms:
+  - os: OS X
+    osVersion: Sequoia
+    browserName: playwright-firefox
+    browserVersion: latest
+  - os: OS X
+    osVersion: Sequoia
+    browserName: playwright-chromium
+    browserVersion: latest
+
+# =======================
+# Parallels per Platform
+# =======================
+# The number of parallel threads to be used for each platform set.
+# BrowserStack's SDK runner will select the best strategy based on the configured value
+#
+# Example 1 - If you have configured 3 platforms and set `parallelsPerPlatform` as 2, a total of 6 (2 * 3) parallel threads will be used on BrowserStack
+#
+# Example 2 - If you have configured 1 platform and set `parallelsPerPlatform` as 5, a total of 5 (1 * 5) parallel threads will be used on BrowserStack
+parallelsPerPlatform: 1
+
+# ==========================================
+# BrowserStack Local
+# (For localhost, staging/private websites)
+# ==========================================
+# Set browserStackLocal to true if your website under test is not accessible publicly over the internet
+# Learn more about how BrowserStack Local works here -> https://www.browserstack.com/docs/automate/selenium/local-testing-introduction
+browserstackLocal: false # <boolean> (Default false)
+# browserStackLocalOptions:
+# Options to be passed to BrowserStack local in-case of advanced configurations
+  # localIdentifier: # <string> (Default: null) Needed if you need to run multiple instances of local.
+  # forceLocal: true  # <boolean> (Default: false) Set to true if you need to resolve all your traffic via BrowserStack Local tunnel.
+  # Entire list of arguments available here -> https://www.browserstack.com/docs/automate/selenium/manage-incoming-connections
+
+# ===================
+# Debugging features
+# ===================
+debug: false # <boolean> # Set to true if you need screenshots for every selenium command ran
+networkLogs: false # <boolean> Set to true to enable HAR logs capturing; off as may contain sensitive info like login API requests
+consoleLogs: errors # <string> Remote browser's console debug levels to be printed (Default: errors)
+# Available options are `disable`, `errors`, `warnings`, `info`, `verbose` (Default: errors)
+framework: playwright
+browserstack.playwrightLogs: false # disable playwright logs appearing on browserstack builds as may contain sensitive info like credentials
+# CUSTOM_TAG_<INT>: # <string> (Default: parent folder name of the test file) Custom tag for your test suite
+browserstack.maskBasicAuth: true # mask username and passwords from browserstack session logs
+browserstack.maskCommands: sendType, sendPress, setHTTPCredentials, setStorageState, setWebAuthnCredentials # prevent sensitive info appearing in browserstack session logs
+# Test Observability is an intelligent test reporting & debugging product. It collects data using the SDK. Read more about what data is collected at https://www.browserstack.com/docs/test-observability/references/terms-and-conditions
+# Visit observability.browserstack.com to see your test reports and insights. To disable test observability, specify `testObservability: false` in the key below.
+testObservability: false

test/e2e/const/constants.ts

@@ -0,0 +1,10 @@
+// appointment urls
+export const APPT_PROD_URL = process.env.APPT_PROD_URL;
+
+// page titles
+export const APPT_PAGE_TITLE = 'Thunderbird Appointment';
+export const FXA_PAGE_TITLE = 'Mozilla accounts';
+
+// production sign-in credentials
+export const PROD_LOGIN_EMAIL = process.env.APPT_PROD_LOGIN_EMAIL;
+export const PROD_LOGIN_PWORD = process.env.APPT_PROD_LOGIN_PWORD;