From 1fa1c00eacefafb5e7f223bbebb6a5421675f154 Mon Sep 17 00:00:00 2001 From: Carlo <9084808+carlo-feudo@users.noreply.github.com> Date: Wed, 30 Aug 2017 14:38:58 +0100 Subject: [PATCH 1/2] exit code 111 from a worker to shut down cluster send exit code 111 from a worker (process.exit(111);) to request the Cluster to shutdown, it comes in handy for whatever reason (i.g. bind already in use). --- lib/graceful-cluster.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/graceful-cluster.js b/lib/graceful-cluster.js index 150e4a7..25707e2 100644 --- a/lib/graceful-cluster.js +++ b/lib/graceful-cluster.js @@ -159,6 +159,13 @@ GracefulCluster.start = function(options) { cluster.on('exit', function(worker, code, signal) { + // send exit code 111 from a worker (process.exit(111);) to request the Cluster to shutdown, it comes in handy for whatever reason (i.g. bind already in use). + if(code === 111) { + log('Shutdown Cluster requested, shutting down...'); + if (shutdownTimer) clearTimeout(shutdownTimer); + process.exit(code); + } + // Mark process finished. if (currentRestartingPid === worker.process.pid) { currentRestartingPid = null; From e50328ef12963ddce058a409d47a292f9db1ad09 Mon Sep 17 00:00:00 2001 From: Carlo <9084808+carlo-feudo@users.noreply.github.com> Date: Thu, 31 Aug 2017 14:29:18 +0100 Subject: [PATCH 2/2] max number of attempts There are cases where it gets generate an infinite loop in trying to re-fork other workers, this prevents the infinite loop and shuts down the cluster --- lib/graceful-cluster.js | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/graceful-cluster.js b/lib/graceful-cluster.js index 25707e2..d032925 100644 --- a/lib/graceful-cluster.js +++ b/lib/graceful-cluster.js @@ -16,6 +16,8 @@ var GracefulCluster = module.exports; - options.restartOnMemory - bytes, restart worker on memory usage. - options.restartOnTimeout - ms, restart worker by timer. - options.workersCount - workers count, if not specified `os.cpus().length` will be used. +- options.maxForksAttempts - max number of attempts of forks for dead workers + - options.timeWindow - time window to check for max number of attempts to restart a dead worker Graceful restart performed by USR2 signal: @@ -40,6 +42,11 @@ GracefulCluster.start = function(options) { var disableGraceful = options.disableGraceful; var workersCount = options.workersCount || numCPUs; + var lastDateChecked = new Date(); + var TIME_WINDOW = options.timeWindow || 60000; //milliseconds + var forksAttempts = 0; + var maxForksAttempts = options.maxForksAttempts || 100; + if (cluster.isMaster) { var currentRestartingPid = null; @@ -178,6 +185,30 @@ GracefulCluster.start = function(options) { return; } log('Cluster: worker ' + worker.process.pid + ' died (code: ' + code + '), restarting...'); + + // checks if too many attemps + + var dateNow = new Date(); + + if (dateNow - lastDateChecked < TIME_WINDOW) { + + forksAttempts++; + + if (forksAttempts >= maxForksAttempts) { + + log('Shutdown Cluster, too many forking attempts (' + forksAttempts + ') in ' + (dateNow - lastDateChecked) + ' milliseconds'); + if (shutdownTimer) clearTimeout(shutdownTimer); + process.exit(1); + + } + + } else { + + forksAttempts = 0; + lastDateChecked = dateNow; + + } + fork(); });