Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enhancement/resource-performance-and-memory #615

Open
wants to merge 3 commits into
base: enhancement/websockets
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .env.sample
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ POOL_DESTROY_TIMEOUT = 5000
POOL_IDLE_TIMEOUT = 30000
POOL_CREATE_RETRY_INTERVAL = 200
POOL_REAPER_INTERVAL = 1000
POOL_RESOURCES_INTERVAL = 30000
POOL_BENCHMARKING = false

# LOGGING CONFIG
Expand All @@ -97,6 +98,7 @@ OTHER_LISTEN_TO_PROCESS_EXITS = true
OTHER_NO_LOGO = false
OTHER_HARD_RESET_PAGE = false
OTHER_BROWSER_SHELL_MODE = true
OTHER_CONNECTION_OVER_PIPE = false

# DEBUG CONFIG
DEBUG_ENABLE = false
Expand Down
133 changes: 132 additions & 1 deletion lib/browser.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ import { getCachePath } from './cache.js';
import { getOptions } from './config.js';
import { setupHighcharts } from './highcharts.js';
import { log, logWithStack } from './logger.js';
import { __dirname } from './utils.js';
import { __dirname, expBackoff } from './utils.js';
import { envs } from './validation.js';

import ExportError from './errors/ExportError.js';

Expand All @@ -39,6 +40,9 @@ const template = readFileSync(__dirname + '/templates/template.html', 'utf8');
// To save the browser
let browser;

// To save the WebSocket endpoint in case of a sudden disconnect
let wsEndpoint;

/**
* Retrieves the existing Puppeteer browser instance.
*
Expand Down Expand Up @@ -85,6 +89,8 @@ export async function createBrowser(puppeteerArgs = []) {
headless: other.browserShellMode ? 'shell' : true,
userDataDir: 'tmp',
args: puppeteerArgs,
// Must be disabled for debugging to work
pipe: envs.OTHER_CONNECTION_OVER_PIPE,
handleSIGINT: false,
handleSIGTERM: false,
handleSIGHUP: false,
Expand All @@ -106,6 +112,23 @@ export async function createBrowser(puppeteerArgs = []) {

// Launch the browser
browser = await puppeteer.launch(launchOptions);

// Close the initial pages if any found
const pages = await browser.pages();
if (pages) {
for (const page of pages) {
await page.close();
}
}

// Only for the WebSocket connection
if (!launchOptions.pipe) {
// Save WebSocket endpoint
wsEndpoint = browser.wsEndpoint();

// Attach the disconnected event
browser.on('disconnected', _reconnect);
}
} catch (error) {
logWithStack(
1,
Expand Down Expand Up @@ -434,6 +457,61 @@ export async function clearPageResources(page, injectedResources) {
}
}

/**
* Reconnects to the browser instance when it is disconnected. If the current
* browser connection is lost, it attempts to reconnect using the previous
* WebSocket endpoint. If the reconnection fails, it will try to close
* the browser and relaunch a new instance.
*
* @async
* @function _reconnect
*/
async function _reconnect() {
try {
// Start the reconnecting
log(3, `[browser] Restarting the browser connection.`);

// Try to reconnect the browser
if (browser && !browser.connected) {
browser = await puppeteer.connect({
browserWSEndpoint: wsEndpoint
});
}

// Save a new WebSocket endpoint
wsEndpoint = browser.wsEndpoint();

// Add the reconnect event again
browser.on('disconnected', _reconnect);

// Log the success message
log(3, `[browser] Browser reconnected successfully.`);
} catch (error) {
logWithStack(
1,
error,
'[browser] Could not restore the browser connection, attempting to relaunch.'
);

// Try to close the browser before relaunching
try {
await close();
} catch (error) {
logWithStack(
1,
error,
'[browser] Could not close the browser before relaunching (probably is already closed).'
);
}

// Try to relaunch the browser
await createBrowser(getOptions().puppeteer.args || []);

// Log the success message
log(3, `[browser] Browser relaunched successfully.`);
}
}

/**
* Sets the content for a Puppeteer Page using a predefined template
* and additional scripts. Also, sets the pageerror in order to catch
Expand Down Expand Up @@ -492,6 +570,59 @@ function _setPageEvents(poolResource) {
console.log(`[debug] ${message.text()}`);
});
}

// Add the framedetached event if the connection is over WebSocket
if (envs.OTHER_CONNECTION_OVER_PIPE === false) {
poolResource.page.on('framedetached', async (frame) => {
// Get the main frame
const mainFrame = poolResource.page.mainFrame();

// Check if a page's frame is detached and requires to be recreated
if (
frame === mainFrame &&
mainFrame.detached &&
poolResource.workCount <= pool.workLimit
) {
log(
3,
`[browser] Pool resource [${poolResource.id}] - Page's frame detached.`
);
try {
// Try to connect to a new page using exponential backoff strategy
expBackoff(
async (poolResourceId, poolResource) => {
try {
// Try to close the page with a detached frame
if (!poolResource.page.isClosed()) {
await poolResource.page.close();
}
} catch (error) {
log(
3,
`[browser] Pool resource [${poolResourceId}] - Could not close the page with a detached frame.`
);
}

// Trigger a page creation
await newPage(poolResource);
},
0,
poolResource.id,
poolResource
);
} catch (error) {
logWithStack(
3,
error,
`[browser] Pool resource [${poolResource.id}] - Could not create a new page.`
);

// Set the `workLimit` to exceeded in order to recreate the resource
poolResource.workCount = pool.workLimit + 1;
}
}
});
}
}

export default {
Expand Down
54 changes: 54 additions & 0 deletions lib/pool.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ import { createBrowser, closeBrowser, newPage, clearPage } from './browser.js';
import { getOptions } from './config.js';
import { puppeteerExport } from './export.js';
import { log, logWithStack } from './logger.js';
import { addTimer } from './timer.js';
import { getNewDateTime, measureTime } from './utils.js';
import { envs } from './validation.js';

import ExportError from './errors/ExportError.js';

Expand Down Expand Up @@ -129,6 +131,12 @@ export async function initPool(poolOptions = getOptions().pool, puppeteerArgs) {
pool.release(resource);
});

// Init the interval for checking if the minimum number of resources exist
if (envs.POOL_RESOURCES_INTERVAL) {
// Register interval for the later clearing
addTimer(_checkingResourcesInterval(envs.POOL_RESOURCES_INTERVAL));
}

log(
3,
`[pool] The pool is ready${initialResources.length ? ` with ${initialResources.length} initial resources waiting.` : '.'}`
Expand Down Expand Up @@ -161,6 +169,11 @@ export async function killPool() {
pool.release(worker.resource);
}

// Remove all attached event listeners from the pool
pool.removeAllListeners('release');
pool.removeAllListeners('destroySuccess');
pool.removeAllListeners('destroyFail');

// Destroy the pool if it is still available
if (!pool.destroyed) {
await pool.destroy();
Expand Down Expand Up @@ -575,6 +588,47 @@ function _factory(poolOptions) {
};
}

/**
* Periodically checks and ensures the minimum number of resources in the pool.
* If the total number of used, free and about to be created resources falls
* below the minimum set with the `pool.min`, it creates additional resources to
* meet the minimum requirement.
*
* @function _checkingResourcesInterval
*
* @param {number} resourceCheckInterval - The interval, in milliseconds, at
* which the pool resources are checked.
*
* @returns {NodeJS.Timeout} - Returns a timer ID that can be used to clear the
* interval later.
*/
function _checkingResourcesInterval(resourceCheckInterval) {
// Set the interval for checking the number of pool resources
return setInterval(async () => {
try {
// Get the current number of resources
let currentNumber =
pool.numUsed() + pool.numFree() + pool.numPendingCreates();

// Create missing resources
while (currentNumber++ < pool.min) {
try {
// Explicitly creating a resource
await pool._doCreate();
} catch (error) {
logWithStack(2, error, '[pool] Could not create a missing resource.');
}
}
} catch (error) {
logWithStack(
1,
error,
`[pool] Something went wrong when trying to create missing resources.`
);
}
}, resourceCheckInterval);
}

export default {
initPool,
killPool,
Expand Down
4 changes: 4 additions & 0 deletions lib/resourceRelease.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ See LICENSE file in root for details.
* proper cleanup of resources such as browsers, pages, servers, and timers.
*/

import { getBrowser } from './browser.js';
import { killPool } from './pool.js';
import { clearAllTimers } from './timer.js';
import { closeServers } from './server/server.js';
Expand All @@ -30,6 +31,9 @@ import { terminateClients } from './server/webSocket.js';
* @param {number} exitCode - An exit code for the `process.exit()` function.
*/
export async function shutdownCleanUp(exitCode) {
// Remove all attached event listeners from the browser
getBrowser().removeAllListeners('disconnected');

// Await freeing all resources
await Promise.allSettled([
// Clear all ongoing intervals
Expand Down
11 changes: 10 additions & 1 deletion lib/server/middlewares/validation.js
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,16 @@ function requestBodyMiddleware(request, _response, next) {
2,
`The request with ID ${requestId} from ${
request.headers['x-forwarded-for'] || request.connection.remoteAddress
} was incorrect. Received payload is missing correct chart data for export: ${JSON.stringify(body)}.`
} was incorrect:
Content-Type: ${request.headers['content-type']}.
Chart constructor: ${body.constr}.
Dimensions: ${body.width}x${body.height} @ ${body.scale} scale.
Type: ${body.type}.
Is SVG set? ${typeof body.svg !== 'undefined'}.
B64? ${typeof body.b64 !== 'undefined'}.
No download? ${typeof body.noDownload !== 'undefined'}.
Received payload is missing correct chart data for export: ${JSON.stringify(body)}.
`
);
throw new NoCorrectChartDataError();
}
Expand Down
Loading
Loading