Merge pull request #514 from highcharts/fix/resources-release

fix/resources-release

.env.sample

@@ -57,7 +57,6 @@ POOL_IDLE_TIMEOUT = 30000
 POOL_CREATE_RETRY_INTERVAL = 200
 POOL_REAPER_INTERVAL = 1000
 POOL_BENCHMARKING = false
-POOL_LISTEN_TO_PROCESS_EXITS = true
 
 # LOGGING CONFIG
 LOGGING_LEVEL = 4
@@ -70,4 +69,5 @@ UI_ROUTE = /
 
 # OTHER CONFIG
 OTHER_NODE_ENV = production
+OTHER_LISTEN_TO_PROCESS_EXITS = true
 OTHER_NO_LOGO = false

.gitignore

@@ -13,6 +13,8 @@ cert/
 
 tests/**/_results/
 
+resources.json
+
 **/*.png
 **/*.pdf
 **/*.svg

CHANGELOG.md

@@ -24,6 +24,12 @@ _Enhancements:_
 - Renamed the `scripts` property of the config options to `customScripts`.
 - Renamed the `initPool` function to `initExport` in the main module.
 - Renamed the `init` function to `initPool` in the pool module.
+- Moved the `listenToProcessExits` from the `pool` to the `other` section of the options.
+- Renamed the environment variable `POOL_LISTEN_TO_PROCESS_EXITS` to `OTHER_LISTEN_TO_PROCESS_EXITS`.
+- Added the `shutdownCleanUp` function for resource release (ending intervals, closing servers, destroying the pool and browser) on shutdown. It will be called in the process exit handlers.
+- Added a new process event handler for the `SIGHUP` signal.
+- Updated the `killPool` function.
+- Updates were made to the `config.js` file.
 - Replaced the temporary benchmark module with a simpler server benchmark for evaluating export time.
 - The `uncaughtException` handler now kills the pool, browser, and terminates the process with exit code 1, when enabled.
 - The browser instance should be correctly closed now when an error occurs during pool creation.
@@ -125,6 +131,7 @@ _Fixes and enhancements:_
 - Error messages are now sent back to the client instead of being displayed in rasterized output.
 - Updated NPM dependencies, removed deprecated and uneccessary dependencies.
 - Lots of smaller bugfixes and tweaks.
+- Transitioned our public server (export.highcharts.com) from HTTP to HTTPS.
 
 _New features:_
 

README.md

@@ -10,6 +10,8 @@ Significant changes have been made to the API for using the server as a Node.js
 
 An important note is that the Export Server now requires `Node.js v18.12.0` or a higher version.
 
+Additionally, with the v3 release, we transitioned from HTTP to HTTPS for export.highcharts.com, so all requests sent to our public server now must use the HTTPS protocol.
+
 ## Changelog
 
 The full change log for all versions can be viewed [here](CHANGELOG.md).
@@ -237,8 +239,7 @@ The format, along with its default values, is as follows (using the recommended
     "idleTimeout": 30000,
     "createRetryInterval": 200,
     "reaperInterval": 1000,
-    "benchmarking": false,
-    "listenToProcessExits": true
+    "benchmarking": false
   },
   "logging": {
     "level": 4,
@@ -250,6 +251,8 @@ The format, along with its default values, is as follows (using the recommended
     "route": "/"
   },
   "other": {
+    "nodeEnv": "production",
+    "listenToProcessExits": true,
     "noLogo": false
   }
 }
@@ -330,7 +333,6 @@ These variables are set in your environment and take precedence over options fro
 - `POOL_CREATE_RETRY_INTERVAL`: The duration, in milliseconds, to wait before retrying the create process in case of a failure (defaults to `200`).
 - `POOL_REAPER_INTERVAL`: The duration, in milliseconds, after which the check for idle resources to destroy is triggered (defaults to `1000`).
 - `POOL_BENCHMARKING`: Indicates whether to show statistics for the pool of resources or not (defaults to `false`).
-- `POOL_LISTEN_TO_PROCESS_EXITS`: Decides whether or not to attach _process.exit_ handlers (defaults to `true`).
 
 ### Logging Config
 
@@ -346,6 +348,7 @@ These variables are set in your environment and take precedence over options fro
 ### Other Config
 
 - `OTHER_NODE_ENV`: The type of Node.js environment. The value controls whether to include the error's stack in a response or not. Can be development or production (defaults to `production`).
+- `OTHER_LISTEN_TO_PROCESS_EXITS`: Decides whether or not to attach _process.exit_ handlers (defaults to `true`).
 - `OTHER_NO_LOGO`: Skip printing the logo on a startup. Will be replaced by a simple text (defaults to `false`).
 
 ## Command Line Arguments
@@ -372,24 +375,27 @@ _Available options:_
 - `--allowFileResources`: Controls the ability to inject resources from the filesystem. This setting has no effect when running as a server (defaults to `false`).
 - `--customCode`: Custom code to execute before chart initialization. It can be a function, code wrapped within a function, or a filename with the _.js_ extension (defaults to `false`).
 - `--callback`: JavaScript code to run during construction. It can be a function or a filename with the _.js_ extension (defaults to `false`).
-- `--resources`: Additional resources in the form of a stringified JSON. It may contain `files`, `js`, and `css` sections (defaults to `false`).
+- `--resources`: Additional resources in the form of a stringified JSON. It may contain `files` (array of JS filenames), `js` (stringified JS), and `css` (stringified CSS) sections (defaults to `false`).
 - `--loadConfig`: A file containing a pre-defined configuration to use (defaults to `false`).
 - `--createConfig`: Enables setting options through a prompt and saving them in a provided config file (defaults to `false`).
 - `--enableServer`: If set to true, the server starts on 0.0.0.0 (defaults to `false`).
 - `--host`: The hostname of the server. Additionally, it starts a server listening on the provided hostname (defaults to `0.0.0.0`).
 - `--port`: The port to be used for the server when enabled (defaults to `7801`).
 - `--serverBenchmarking`: Indicates whether to display the duration, in milliseconds, of specific actions that occur on the server while serving a request (defaults to `false`).
-- `--enableSsl`: Enables or disables the SSL protocol (defaults to `false`).
-- `--sslForced`: If set to true, the server is forced to serve only over HTTPS (defaults to `false`).
-- `--sslPort`: The port on which to run the SSL server (defaults to `443`).
-- `--certPath`: The path to the SSL certificate/key file (defaults to ``).
+- `--proxyHost`: The host of the proxy server to use, if it exists (defaults to `false`).
+- `--proxyPort`: The port of the proxy server to use, if it exists (defaults to `false`).
+- `--proxyTimeout`: The timeout for the proxy server to use, if it exists (defaults to `5000`).
 - `--enableRateLimiting`: Enables rate limiting for the server (defaults to `false`).
 - `--maxRequests`: The maximum number of requests allowed in one minute (defaults to `10`).
 - `--window`: The time window, in minutes, for the rate limiting (defaults to `1`).
 - `--delay`: The delay duration for each successive request before reaching the maximum limit (defaults to `0`).
 - `--trustProxy`: Set this to true if the server is behind a load balancer (defaults to `false`).
 - `--skipKey`: Allows bypassing the rate limiter and should be provided with the `--skipToken` argument (defaults to ``).
 - `--skipToken`: Allows bypassing the rate limiter and should be provided with the `--skipKey` argument (defaults to ``).
+- `--enableSsl`: Enables or disables the SSL protocol (defaults to `false`).
+- `--sslForce`: If set to true, the server is forced to serve only over HTTPS (defaults to `false`).
+- `--sslPort`: The port on which to run the SSL server (defaults to `443`).
+- `--certPath`: The path to the SSL certificate/key file (defaults to ``).
 - `--minWorkers`: The number of minimum and initial pool workers to spawn (defaults to `4`).
 - `--maxWorkers`: The number of maximum pool workers to spawn (defaults to `8`).
 - `--workLimit`: The number of work pieces that can be performed before restarting the worker process (defaults to `40`).
@@ -400,12 +406,13 @@ _Available options:_
 - `--createRetryInterval`: The duration, in milliseconds, to wait before retrying the create process in case of a failure (defaults to `200`).
 - `--reaperInterval`: The duration, in milliseconds, after which the check for idle resources to destroy is triggered (defaults to `1000`).
 - `--poolBenchmarking`: Indicate whether to show statistics for the pool of resources or not (defaults to `false`).
-- `--listenToProcessExits`: Decides whether or not to attach process.exit handlers (defaults to `true`).
 - `--logLevel`: The logging level to be used. Can be _0_ - silent, _1_ - error, _2_ - warning, _3_ - notice, _4_ - verbose or _5_ - benchmark (defaults to `4`).
 - `--logFile`: The name of a log file. The `--logDest` option also needs to be set to enable file logging (defaults to `highcharts-export-server.log`).
 - `--logDest`: The path to store log files. This also enables file logging (defaults to `log/`).
 - `--enableUi`: Enables or disables the user interface (UI) for the Export Server (defaults to `false`).
 - `--uiRoute`: The endpoint route to which the user interface (UI) should be attached (defaults to `/`).
+- `--nodeEnv`: The type of Node.js environment (defaults to `production`).
+- `--listenToProcessExits`: Decides whether or not to attach process.exit handlers (defaults to `true`).
 - `--noLogo`: Skip printing the logo on a startup. Will be replaced by a simple text (defaults to `false`).
 
 # HTTP Server
@@ -446,7 +453,7 @@ The server accepts the following arguments in a POST request body:
 - `scale`: The scale factor of the exported chart. Use it to improve resolution in PNG and JPEG, for example setting scale to 2 on a 600px chart will result in a 1200px output.
 - `globalOptions`: Either a JSON or a stringified JSON with global options to be passed into `Highcharts.setOptions`.
 - `themeOptions`: Either a JSON or a stringified JSON with theme options to be passed into `Highcharts.setOptions`.
-- `resources`: Additional resources in the form of a JSON or a stringified JSON. It may contain `files`, `js`, and `css` sections.
+- `resources`: Additional resources in the form of a JSON or a stringified JSON. It may contain `files` (array of JS filenames), `js` (stringified JS), and `css` (stringified CSS) sections.
 - `callback`: Stringified JavaScript function to execute in the Highcharts constructor.
 - `customCode`: Custom code to be executed before the chart initialization. This can be a function, code wrapped within a function, or a filename with the _.js_ extension. Both `allowFileResources` and `allowCodeExecution` must be set to _true_ for the option to be considered.
 - `b64`: Boolean flag, set to true to receive the chart in the _base64_ format instead of the _binary_.
@@ -547,6 +554,10 @@ This package supports both CommonJS and ES modules.
 - `server`: The server instance which offers the following functions:
   - `async startServer(serverConfig)`: The same as `startServer` describe below.
 
+  - `closeServers()`: Closes all servers associated with Express app instance.
+
+  - `getServers()`: Get all servers associated with Express app instance.
+
   - `enableRateLimiting(options)`: Enable rate limiting for the server.
     - `{Object} limitConfig`: Configuration object for rate limiting.
 
@@ -572,10 +583,6 @@ This package supports both CommonJS and ES modules.
 - `async initExport(options)`: Initializes the export process. Tasks such as configuring logging, checking cache and sources, and initializing the pool of resources happen during this stage. Function that is required to be called before trying to export charts or setting a server. The `options` is an object that contains all options.
   - `{Object} options`: All export options.
 
-- `setOptions(userOptions, args)`: Initializes and sets the general options for the server instace, keeping the principle of the options load priority. It accepts optional userOptions and args from the CLI.
-  - `{Object} userOptions`: User-provided options for customization.
-  - `{Array} args`: Command-line arguments for additional configuration (CLI usage).
-
 - `async singleExport(options)`: Starts a single export process based on the specified options. Runs the `startExport` underneath.
   - `{Object} options`: The options object containing configuration for a single export.
 
@@ -586,7 +593,12 @@ This package supports both CommonJS and ES modules.
   - `{Object} settings`: The settings object containing export configuration.
   - `{function} endCallback`: The callback function to be invoked upon finalizing work or upon error occurance of the exporting process.
 
-- `async killPool()`: Kills all workers in the pool, destroys the pool, and closes the browser instance.
+- `setOptions(userOptions, args)`: Initializes and sets the general options for the server instace, keeping the principle of the options load priority. It accepts optional userOptions and args from the CLI.
+  - `{Object} userOptions`: User-provided options for customization.
+  - `{Array} args`: Command-line arguments for additional configuration (CLI usage).
+
+- `async shutdownCleanUp(exitCode)`: Clean up function to trigger before ending process for the graceful shutdown.
+  - `{number} exitCode`: An exit code for the process.exit() function.
 
 - `log(...args)`: Logs a message. Accepts a variable amount of arguments. Arguments after `level` will be passed directly to console.log, and/or will be joined and appended to the log file.
   - `{any} args`: An array of arguments where the first is the log level and the rest are strings to build a message with.
@@ -596,10 +608,10 @@ This package supports both CommonJS and ES modules.
   - `{Error} error`: The error object.
   - `{string} customMessage`: An optional custom message to be logged along with the error.
 
-- `setLogLevel`: Sets the log level to the specified value. Log levels are (0 = no logging, 1 = error, 2 = warning, 3 = notice, 4 = verbose or 5 = benchmark).
+- `setLogLevel(newLevel)`: Sets the log level to the specified value. Log levels are (0 = no logging, 1 = error, 2 = warning, 3 = notice, 4 = verbose or 5 = benchmark).
   - `{number} newLevel`: The new log level to be set.
 
-- `enableFileLogging`: Enables file logging with the specified destination and log file.
+- `enableFileLogging(logDest, logFile)`: Enables file logging with the specified destination and log file.
   - `{string} logDest`: The destination path for log files.
   - `{string} logFile`: The log file name.
 
@@ -633,7 +645,7 @@ At some point during the transition process from the `PhantomJS` solution, certa
 Additionally, some options are now named differently due to the new structure and categorization. Here is a list of old names and their corresponding new names (`old name` -> `new name`):
 
 - `fromFile` -> `loadConfig`
-- `sslOnly` -> `force` or `sslForced`
+- `sslOnly` -> `force` or `sslForce`
 - `sslPath` -> `certPath`
 - `rateLimit` -> `maxRequests`
 - `workers` -> `maxWorkers`
@@ -665,7 +677,7 @@ Like previously mentioned, there are multiple ways to set and prioritize options
 
 ## Note about Event Listeners
 
-The Export Server attaches event listeners to `process.exit`. This is to make sure that there are no memory leaks or zombie processes if the application is unexpectedly terminated.
+The Export Server attaches event listeners to `process.exit`, `uncaughtException` and signals such as `SIGINT`, `SIGTERM` and `SIGHUP`. This is to make sure that there are no memory leaks or zombie processes if the application is unexpectedly terminated.
 
 Listeners are also attached to handle `uncaught exceptions`. If an exception occurs, the entire pool and browser instance are terminated, and the application is shut down.
 

bin/cli.js

@@ -103,11 +103,8 @@ const start = async () => {
     // Log the error with stack
     main.logWithStack(1, error);
 
-    // Kill pool and close browser if exist
-    await main.killPool();
-
-    // End process with an error code
-    process.exit(1);
+    // Gracefully shut down the process
+    await main.shutdownCleanUp(1);
   }
 };
 

lib/browser.js

@@ -199,7 +199,10 @@ export const create = async (puppeteerArgs) => {
         browser = await puppeteer.launch({
           headless: 'new',
           args: allArgs,
-          userDataDir: './tmp/'
+          userDataDir: './tmp/',
+          handleSIGINT: false,
+          handleSIGTERM: false,
+          handleSIGHUP: false
         });
       } catch (error) {
         logWithStack(
@@ -263,9 +266,8 @@ export const close = async () => {
   // Close the browser when connnected
   if (browser?.isConnected()) {
     await browser.close();
-    log(4, '[browser] Closed the browser.');
   }
-  return true;
+  log(4, '[browser] Closed the browser.');
 };
 
 export default {

lib/chart.js

@@ -219,7 +219,7 @@ export const singleExport = async (options) => {
       type !== 'svg' ? Buffer.from(info.result, 'base64') : info.result
     );
 
-    // Kill the pool
+    // Kill pool and close browser after finishing single export
     await killPool();
   });
 };

lib/envs.js

@@ -145,16 +145,21 @@ export const Config = z.object({
   SERVER_PORT: v.positiveNum(),
   SERVER_BENCHMARKING: v.boolean(),
 
+  // server proxy
   SERVER_PROXY_HOST: v.string(),
   SERVER_PROXY_PORT: v.positiveNum(),
   SERVER_PROXY_TIMEOUT: v.nonNegativeNum(),
+
+  // server rate limiting
   SERVER_RATE_LIMITING_ENABLE: v.boolean(),
   SERVER_RATE_LIMITING_MAX_REQUESTS: v.nonNegativeNum(),
   SERVER_RATE_LIMITING_WINDOW: v.nonNegativeNum(),
   SERVER_RATE_LIMITING_DELAY: v.nonNegativeNum(),
   SERVER_RATE_LIMITING_TRUST_PROXY: v.boolean(),
   SERVER_RATE_LIMITING_SKIP_KEY: v.string(),
   SERVER_RATE_LIMITING_SKIP_TOKEN: v.string(),
+
+  // server ssl
   SERVER_SSL_ENABLE: v.boolean(),
   SERVER_SSL_FORCE: v.boolean(),
   SERVER_SSL_PORT: v.positiveNum(),
@@ -171,7 +176,6 @@ export const Config = z.object({
   POOL_CREATE_RETRY_INTERVAL: v.nonNegativeNum(),
   POOL_REAPER_INTERVAL: v.nonNegativeNum(),
   POOL_BENCHMARKING: v.boolean(),
-  POOL_LISTEN_TO_PROCESS_EXITS: v.boolean(),
 
   // logger
   LOGGING_LEVEL: z
@@ -197,6 +201,7 @@ export const Config = z.object({
 
   // other
   OTHER_NODE_ENV: v.enum(['development', 'production', 'test']),
+  OTHER_LISTEN_TO_PROCESS_EXITS: v.boolean(),
   OTHER_NO_LOGO: v.boolean()
 });
 

lib/index.js

@@ -29,10 +29,49 @@ import {
   setLogLevel,
   enableFileLogging
 } from './logger.js';
-import { initPool, killPool } from './pool.js';
+import { initPool } from './pool.js';
+import { shutdownCleanUp } from './resource_release.js';
 import server, { startServer } from './server/server.js';
 import { printLogo, printUsage } from './utils.js';
 
+/**
+ * Attaches exit listeners to the process, ensuring proper cleanup of resources
+ * and termination on exit signals. Handles 'exit', 'SIGINT', 'SIGTERM', and
+ * 'uncaughtException' events.
+ */
+const attachProcessExitListeners = () => {
+  log(3, '[process] Attaching exit listeners to the process.');
+
+  // Handler for the 'exit'
+  process.on('exit', (code) => {
+    log(4, `Process exited with code ${code}.`);
+  });
+
+  // Handler for the 'SIGINT'
+  process.on('SIGINT', async (name, code) => {
+    log(4, `The ${name} event with code: ${code}.`);
+    await shutdownCleanUp(0);
+  });
+
+  // Handler for the 'SIGTERM'
+  process.on('SIGTERM', async (name, code) => {
+    log(4, `The ${name} event with code: ${code}.`);
+    await shutdownCleanUp(0);
+  });
+
+  // Handler for the 'SIGHUP'
+  process.on('SIGHUP', async (name, code) => {
+    log(4, `The ${name} event with code: ${code}.`);
+    await shutdownCleanUp(0);
+  });
+
+  // Handler for the 'uncaughtException'
+  process.on('uncaughtException', async (error, name) => {
+    logWithStack(1, error, `The ${name} error.`);
+    await shutdownCleanUp(1);
+  });
+};
+
 /**
  * Initializes the export process. Tasks such as configuring logging, checking
  * cache and sources, and initializing the pool of resources happen during
@@ -51,6 +90,11 @@ const initExport = async (options) => {
   // Init the logging
   initLogging(options.logging);
 
+  // Attach process' exit listeners
+  if (options.other.listenToProcessExits) {
+    attachProcessExitListeners();
+  }
+
   // Check if cache needs to be updated
   await checkAndUpdateCache(options);
 
@@ -71,14 +115,16 @@ export default {
   // Server
   server,
   startServer,
-  setOptions,
 
   // Exporting
   initExport,
   singleExport,
   batchExport,
   startExport,
-  killPool,
+
+  // Other
+  setOptions,
+  shutdownCleanUp,
 
   // Logs
   log,