Skip to content

Commit

Permalink
feat(web): restore test-timeout scaling for remote testing
Browse files Browse the repository at this point in the history
  • Loading branch information
jahorton committed Aug 13, 2024
1 parent a5c1766 commit 77d075a
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 28 deletions.
9 changes: 9 additions & 0 deletions common/test/resources/test-timeouts-remote.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import * as baseTimeouts from './test-timeouts.mjs';

// General pattern, in case more are defined.
const scaledTimeouts = {
...baseTimeouts
};
Object.keys(baseTimeouts).forEach((key) => scaledTimeouts[key] *= 10);

export const DEFAULT_BROWSER_TIMEOUT = scaledTimeouts.DEFAULT_BROWSER_TIMEOUT; //ms
104 changes: 76 additions & 28 deletions common/test/resources/wtr-browserstack-config.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { browserstackLauncher } from '@web/test-runner-browserstack';
import { legacyPlugin } from '@web/dev-server-legacy';
import named from '@keymanapp/common-test-resources/test-runner-rename-browser.mjs';
import { importMapsPlugin } from '@web/dev-server-import-maps';

// TODO: always rebuild KEYMAN_VERSION before running this; we want it to show
// the absolute current version in the BrowserStack logs, even if run locally.
Expand All @@ -26,6 +27,7 @@ const sharedCapabilities = {

/**
* @param {import('@web/test-runner').TestRunnerConfig} config
* @returns {import('@web/test-runner').TestRunnerConfig}
* */
export function buildLegacyTestingConfig(config, projectName) {
return {
Expand Down Expand Up @@ -55,48 +57,83 @@ export function buildLegacyTestingConfig(config, projectName) {
},
}),

// macOS... seems to just... time out some of the time.
named(browserstackLauncher({
capabilities: {
...sharedCapabilities,
name: projectName || `${KEYMAN_VERSION.VERSION_WITH_TAG}`,
browserName: 'Safari',
// '13.4' does not appear to work, nor does '14'.
// '15', however, does!
os_version: '15',
deviceName: 'iPhone SE 2022'
// os_version: '13.4',
// deviceName: 'iPhone SE 2020' // 2022 for Safari 15.
os: "OS X",
// // ... none of these is actually connecting for remote testing via BrowserStack.
// // No such issue when running the tests locally, though.
os_version: "Catalina",
browser_version: '13.1'
// os_version: "Mojave",
// browser_version: "12.1"
// os_version: "Monterey",
// browser_version: "15.6"
// os_version: "Big Sur",
// browser_version: '14.1'
},
}), 'Safari for iOS'),
}), 'Safari'),

// // was S7, 6.0 - but the API gave us a 'deprecated' report and errored there.
// // S8, 7.0 ... seems that the test-process outright crashes for no clearly stated reason?
// // named(browserstackLauncher({
// // capabilities: {
// // ...sharedCapabilities,
// name: projectName || `${KEYMAN_VERSION.VERSION_WITH_TAG}`,
// // browserName: 'Chrome',
// // os_version: '7.0',
// // deviceName: 'Samsung Galaxy S8'
// // },
// // }), 'Chrome for Android'),
// // iOS devices seem to flake regularly when I attempt to use 'em.
// named(browserstackLauncher({
// capabilities: {
// ...sharedCapabilities,
// name: projectName || `${KEYMAN_VERSION.VERSION_WITH_TAG}`,
// browserName: 'Safari',
// // '13.4' does not appear to work, nor does '14'.
// // '15', however, does!
// os_version: '15',
// deviceName: 'iPhone SE 2022'
// // os_version: '13.4',
// // deviceName: 'iPhone SE 2020' // 2022 for Safari 15.
// },
// }), 'Safari for iOS'),

browserstackLauncher({
// was S7, 6.0 - but the API gave us a 'deprecated' report and errored there.
// S8, 7.0 ... seems that the test-process outright crashes for no clearly stated reason?
named(browserstackLauncher({
capabilities: {
...sharedCapabilities,
name: projectName || `${KEYMAN_VERSION.VERSION_WITH_TAG}`,
browserName: 'Firefox',
// 'latest' would work, but not 90.
// '100' does work, at least.
browser_version: '100',
os: 'Windows',
os_version: '10',
name: projectName || `${KEYMAN_VERSION.VERSION_WITH_TAG}`,
browserName: 'Chrome',
os_version: '7.0',
deviceName: 'Samsung Galaxy S8'
},
}),
}), 'Chrome for Android'),

// // And sometimes the Firefox test targets flake. Why!?
// browserstackLauncher({
// capabilities: {
// ...sharedCapabilities,
// name: projectName || `${KEYMAN_VERSION.VERSION_WITH_TAG}`,
// browserName: 'Firefox',
// // 'latest' would work, but not 90.
// // '100' does work, at least.
// browser_version: '100',
// os: 'Windows',
// os_version: '10',
// },
// }),
],
plugins: [
...(config.plugins ?? []),
legacyPlugin()
legacyPlugin(),
// Redirects our timeout-defining import to one with more generous
// settings in order to compensate for delays due to serving files
// remotely.
importMapsPlugin({
inject: {
importMap: {
imports: {
// Note: this cannot resolve a node-style import on the right-hand side.
'@keymanapp/common-test-resources/test-timeouts.mjs': '/common/test/resources/test-timeouts-remote.mjs'
}
}
}
})
],

// @web/test-runner uses these settings in their BrowserStack auto-testing.
Expand All @@ -105,6 +142,17 @@ export function buildLegacyTestingConfig(config, projectName) {
testsFinishTimeout: 1000 * 60 * 2,

reporters: [...config.reporters, summaryReporter, defaultReporter],

testFramework: {
...(config.testFramework ?? {}),
config: {
...(config.testFramework?.config ?? {}),
/* remote tunneling for BrowserStack tends to add serious delays to script loads */
// 2000ms = the Mocha default.
timeout: (config.testFramework?.config?.timeout ?? 2000) * 10
}
},

debug: true
};
};

0 comments on commit 77d075a

Please sign in to comment.