perf: increase html-reporter performance

lib/merge-reports/index.js

@@ -1,13 +1,17 @@
 'use strict';
 
+const axios = require('axios');
+const {get} = require('lodash');
 const serverUtils = require('../server-utils');
 
 module.exports = async (pluginConfig, hermione, srcPaths, {destination: destPath}) => {
     validateOpts(srcPaths, destPath);
 
+    const resolvedUrls = await tryResolveUrls(srcPaths);
+
     await Promise.all([
         serverUtils.saveStaticFilesToReportDir(hermione, pluginConfig, destPath),
-        serverUtils.writeDatabaseUrlsFile(destPath, srcPaths)
+        serverUtils.writeDatabaseUrlsFile(destPath, resolvedUrls)
     ]);
 
     await hermione.htmlReporter.emitAsync(hermione.htmlReporter.events.REPORT_SAVED, {reportPath: destPath});
@@ -22,3 +26,42 @@ function validateOpts(srcPaths, destPath) {
         throw new Error(`Destination report path: ${destPath}, exists in source report paths`);
     }
 }
+
+async function tryResolveUrls(urls) {
+    const resolvedUrls = [];
+    const results = await Promise.all(urls.map(tryResolveUrl));
+
+    results.forEach(({jsonUrls, dbUrls}) => {
+        resolvedUrls.push(...jsonUrls.concat(dbUrls));
+    });
+
+    return resolvedUrls;
+}
+
+async function tryResolveUrl(url) {
+    const jsonUrls = [];
+    const dbUrls = [];
+
+    if (serverUtils.isDbUrl(url)) {
+        dbUrls.push(url);
+    } else if (serverUtils.isJsonUrl(url)) {
+        try {
+            const {data} = await axios.get(url);
+            const currentDbUrls = get(data, 'dbUrls', []);
+            const currentJsonUrls = get(data, 'jsonUrls', []);
+
+            const responses = await Promise.all(currentJsonUrls.map(tryResolveUrl));
+
+            dbUrls.push(...currentDbUrls);
+
+            responses.forEach(response => {
+                dbUrls.push(...response.dbUrls);
+                jsonUrls.push(...response.jsonUrls);
+            });
+        } catch (e) {
+            jsonUrls.push(url);
+        }
+    }
+
+    return {jsonUrls, dbUrls};
+}

lib/server-utils.ts

@@ -131,7 +131,8 @@ export async function saveStaticFilesToReportDir(htmlReporter: HtmlReporter, plu
         copyToReportDir(destPath, ['report.min.js', 'report.min.css'], staticFolder),
         fs.copy(path.resolve(staticFolder, 'index.html'), path.resolve(destPath, 'index.html')),
         fs.copy(path.resolve(staticFolder, 'icons'), path.resolve(destPath, 'icons')),
-        fs.copy(require.resolve('@gemini-testing/sql.js'), path.resolve(destPath, 'sql-wasm.js')),
+        fs.copy(require.resolve('@gemini-testing/sql.js/dist/sql-wasm.js'), path.resolve(destPath, 'sql-wasm.js')),
+        fs.copy(require.resolve('@gemini-testing/sql.js/dist/sql-wasm.wasm'), path.resolve(destPath, 'sql-wasm.wasm')),
         copyPlugins(pluginConfig, destPath)
     ]);
 }
@@ -168,10 +169,17 @@ export function urlPathNameEndsWith(currentUrl: string, searchString: string): b
     }
 }
 
-export async function writeDatabaseUrlsFile(destPath: string, srcPaths: string[]): Promise<void> {
-    const jsonUrls = srcPaths.filter(p => urlPathNameEndsWith(p, '.json'));
-    const dbUrls = srcPaths.filter(p => urlPathNameEndsWith(p, '.db'));
+export function isJsonUrl(url: string): boolean {
+    return urlPathNameEndsWith(url, '.json');
+}
 
+export function isDbUrl(url: string): boolean {
+    return urlPathNameEndsWith(url, '.db');
+}
+
+export async function writeDatabaseUrlsFile(destPath: string, srcPaths: string[]): Promise<void> {
+    const jsonUrls = srcPaths.filter(isJsonUrl);
+    const dbUrls = srcPaths.filter(isDbUrl);
     const data = {
         dbUrls,
         jsonUrls

lib/static/modules/actions.js

@@ -66,6 +66,8 @@ export const initStaticReport = () => {
             const mainDatabaseUrls = new URL('databaseUrls.json', window.location.href);
             const fetchDbResponses = await fetchDataFromDatabases([mainDatabaseUrls.href]);
 
+            plugins.preloadAll(dataFromStaticFile.config);
+
             fetchDbDetails = fetchDbResponses.map(({url, status, data}) => ({url, status, success: !!data}));
 
             const dataForDbs = fetchDbResponses.map(({data}) => data).filter(data => data);

lib/static/modules/load-plugin.js

@@ -49,17 +49,21 @@ const whitelistedDeps = {
 // - an array with the string list of required dependencies and a function as the last item.
 //   The function will be called with the dependencies as arguments plus `options` arg.
 
+const loadingPlugins = {};
 const pendingPlugins = {};
 
-export default async function loadPlugin(pluginName, pluginConfig) {
+export function preloadPlugin(pluginName) {
+    loadingPlugins[pluginName] = loadingPlugins[pluginName] || getScriptText(pluginName);
+}
+
+export async function loadPlugin(pluginName, pluginConfig) {
     if (pendingPlugins[pluginName]) {
         return pendingPlugins[pluginName];
     }
 
-    const pluginScriptPath = `plugins/${encodeURIComponent(pluginName)}/plugin.js`;
+    const scriptTextPromise = loadingPlugins[pluginName] || getScriptText(pluginName);
 
-    return pendingPlugins[pluginName] = Promise.resolve(pluginScriptPath)
-        .then(getScriptText)
+    return pendingPlugins[pluginName] = scriptTextPromise
         .then(executePluginCode)
         .then(plugin => initPlugin(plugin, pluginName, pluginConfig))
         .then(null, err => {
@@ -103,7 +107,13 @@ function executePluginCode(code) {
     return exec();
 }
 
-async function getScriptText(scriptUrl) {
-    const result = await axios.get(scriptUrl);
-    return result.data;
+function getPluginScriptPath(pluginName) {
+    return `plugins/${encodeURIComponent(pluginName)}/plugin.js`;
+}
+
+async function getScriptText(pluginName) {
+    const scriptUrl = getPluginScriptPath(pluginName);
+    const {data} = await axios.get(scriptUrl);
+
+    return data;
 }

lib/static/modules/plugins.js

@@ -1,8 +1,16 @@
-import loadPlugin from './load-plugin';
+import {loadPlugin, preloadPlugin} from './load-plugin';
 
 const plugins = Object.create(null);
 const loadedPluginConfigs = [];
 
+function preloadAll(config) {
+    if (!config || !config.pluginsEnabled || !Array.isArray(config.plugins)) {
+        return;
+    }
+
+    config.plugins.forEach(plugin => preloadPlugin(plugin.name));
+}
+
 async function loadAll(config) {
     // if plugins are disabled, act like there are no plugins defined
     if (!config || !config.pluginsEnabled || !Array.isArray(config.plugins)) {
@@ -49,4 +57,4 @@ function getLoadedConfigs() {
     return loadedPluginConfigs;
 }
 
-module.exports = {loadAll, getLoadedConfigs, forEach, get};
+module.exports = {preloadAll, loadAll, getLoadedConfigs, forEach, get};

lib/static/modules/reducers/api-values.js

@@ -1,25 +1,16 @@
 import actionNames from '../action-names';
+import {applyStateUpdate} from '../utils';
 
 export default (state, action) => {
     switch (action.type) {
         case actionNames.INIT_GUI_REPORT:
         case actionNames.INIT_STATIC_REPORT: {
             const {apiValues} = action.payload;
 
-            return applyChanges(state, apiValues);
+            return applyStateUpdate(state, {apiValues});
         }
 
         default:
             return state;
     }
 };
-
-function applyChanges(state, apiValues) {
-    return {
-        ...state,
-        apiValues: {
-            ...state.apiValues,
-            ...apiValues
-        }
-    };
-}

lib/static/modules/reducers/bottom-progress-bar.js

@@ -1,16 +1,14 @@
 import actionNames from '../action-names';
-import {produce} from 'immer';
-import {set} from 'lodash';
+import {applyStateUpdate} from '../utils';
 
-export default produce((draft, action) => {
+export default ((state, action) => {
     switch (action.type) {
         case actionNames.UPDATE_BOTTOM_PROGRESS_BAR: {
             const {currentRootSuiteId} = action.payload;
-
-            return set(draft, 'progressBar.currentRootSuiteId', currentRootSuiteId);
+            return applyStateUpdate(state, {progressBar: {currentRootSuiteId}});
         }
 
         default:
-            return draft;
+            return state;
     }
 });

lib/static/modules/reducers/config.js

@@ -1,6 +1,7 @@
 import {cloneDeep} from 'lodash';
 import {CONTROL_TYPE_RADIOBUTTON} from '../../../gui/constants/custom-gui-control-types';
 import actionNames from '../action-names';
+import {applyStateUpdate} from '../utils';
 
 export default (state, action) => {
     switch (action.type) {
@@ -10,7 +11,7 @@ export default (state, action) => {
 
             config.errorPatterns = formatErrorPatterns(config.errorPatterns);
 
-            return applyChanges(state, config);
+            return applyStateUpdate(state, {config});
         }
 
         case actionNames.RUN_CUSTOM_GUI_ACTION: {
@@ -22,7 +23,7 @@ export default (state, action) => {
             if (type === CONTROL_TYPE_RADIOBUTTON) {
                 controls.forEach((control, i) => control.active = (controlIndex === i));
 
-                return applyChanges(state, {customGui});
+                return applyStateUpdate(state, {config: {customGui}});
             }
 
             return state;
@@ -36,13 +37,3 @@ export default (state, action) => {
 function formatErrorPatterns(errorPatterns) {
     return errorPatterns.map((patternInfo) => ({...patternInfo, regexp: new RegExp(patternInfo.pattern)}));
 }
-
-function applyChanges(state, config) {
-    return {
-        ...state,
-        config: {
-            ...state.config,
-            ...config
-        }
-    };
-}

lib/static/modules/reducers/grouped-tests/by/meta.js

@@ -1,9 +1,10 @@
 import {handleActiveResults, addGroupItem, sortGroupValues} from '../helpers';
+import {ensureDiffProperty} from '../../../utils';
 
-export function groupMeta({group, groupKey, ...restArgs}) {
+export function groupMeta({group, groupKey, diff = group, ...restArgs}) {
     const metaKeys = new Set();
     if (groupKey) {
-        group.byKey[groupKey] = {};
+        ensureDiffProperty(diff, ['byKey', groupKey]);
     }
 
     const resultCb = (result) => {
@@ -18,14 +19,19 @@ export function groupMeta({group, groupKey, ...restArgs}) {
                 continue;
             }
 
-            addGroupItem({group: group.byKey[groupKey], result, value});
+            addGroupItem({
+                group: group.byKey[groupKey],
+                result,
+                value,
+                diff: diff.byKey[groupKey]
+            });
         }
     };
 
     handleActiveResults({...restArgs, resultCb});
 
-    group.allKeys = [...metaKeys].sort();
+    diff.allKeys = [...metaKeys].sort();
     if (groupKey) {
-        group.byKey[groupKey] = sortGroupValues(group.byKey[groupKey]);
+        diff.byKey[groupKey] = sortGroupValues(diff.byKey[groupKey]);
     }
 }

lib/static/modules/reducers/grouped-tests/by/result.js

@@ -1,6 +1,6 @@
 import {get} from 'lodash';
 import {handleActiveResults, addGroupItem, sortGroupValues} from '../helpers';
-import {isAssertViewError} from '../../../utils';
+import {ensureDiffProperty, isAssertViewError} from '../../../utils';
 import {ERROR_KEY, RESULT_KEYS} from '../../../../../constants/group-tests';
 
 const imageComparisonErrorMessage = 'image comparison failed';
@@ -13,21 +13,27 @@ export function groupResult(args) {
     throw new Error(`Group key must be one of ${RESULT_KEYS.join(', ')}, but got ${args.groupKey}`);
 }
 
-function groupErrors({tree, group, groupKey, errorPatterns = [], ...viewArgs}) {
-    group.byKey[groupKey] = {};
+function groupErrors({tree, group, groupKey, errorPatterns = [], diff = group, ...viewArgs}) {
+    ensureDiffProperty(diff, ['byKey', groupKey]);
 
     const resultCb = (result) => {
         const images = result.imageIds.map((imageId) => tree.images.byId[imageId]);
         const errors = extractErrors(result, images);
 
         for (const errorText of errors) {
-            addGroupItem({group: group.byKey[groupKey], result, value: errorText, patterns: errorPatterns});
+            addGroupItem({
+                group: group.byKey[groupKey],
+                result,
+                value: errorText,
+                patterns: errorPatterns,
+                diff: diff.byKey[groupKey]
+            });
         }
     };
 
     handleActiveResults({tree, ...viewArgs, resultCb});
 
-    group.byKey[groupKey] = sortGroupValues(group.byKey[groupKey]);
+    diff.byKey[groupKey] = sortGroupValues(diff.byKey[groupKey]);
 }
 
 function extractErrors(result, images) {

lib/static/modules/reducers/grouped-tests/helpers.js

@@ -26,13 +26,13 @@ export function handleActiveResults({tree = {}, viewMode = ViewMode.ALL, filtere
     }
 }
 
-export function addGroupItem({group, result, value, patterns = []}) {
+export function addGroupItem({group, result, value, patterns = [], diff = group}) {
     const stringifiedValue = stringify(value);
     const {pattern, name} = matchGroupWithPatterns(stringifiedValue, patterns);
 
     // eslint-disable-next-line no-prototype-builtins
-    if (!group.hasOwnProperty(name)) {
-        group[name] = {
+    if ((!group || !group.hasOwnProperty(name)) && !diff.hasOwnProperty(name)) {
+        diff[name] = {
             pattern,
             name,
             browserIds: [result.parentId],
@@ -44,7 +44,7 @@ export function addGroupItem({group, result, value, patterns = []}) {
         return;
     }
 
-    const groupItem = group[name];
+    const groupItem = diff[name];
 
     if (!groupItem.browserIds.includes(result.parentId)) {
         groupItem.browserIds.push(result.parentId);