exit code 111 from a worker to shut down cluster

lib/graceful-cluster.js

@@ -16,6 +16,8 @@ var GracefulCluster = module.exports;
  - options.restartOnMemory       - bytes, restart worker on memory usage.
  - options.restartOnTimeout      - ms, restart worker by timer.
  - options.workersCount          - workers count, if not specified `os.cpus().length` will be used.
+- options.maxForksAttempts      - max number of attempts of forks for dead workers
+ - options.timeWindow            - time window to check for max number of attempts to restart a dead worker
 
  Graceful restart performed by USR2 signal:
 
@@ -40,6 +42,11 @@ GracefulCluster.start = function(options) {
     var disableGraceful = options.disableGraceful;
     var workersCount = options.workersCount || numCPUs;
 
+    var lastDateChecked = new Date();
+    var TIME_WINDOW = options.timeWindow || 60000; //milliseconds
+    var forksAttempts = 0;
+    var maxForksAttempts = options.maxForksAttempts || 100;
+
     if (cluster.isMaster) {
 
         var currentRestartingPid = null;
@@ -159,6 +166,13 @@ GracefulCluster.start = function(options) {
 
         cluster.on('exit', function(worker, code, signal) {
 
+            // send exit code 111 from a worker (process.exit(111);) to request the Cluster to shutdown, it comes in handy for whatever reason (i.g. bind already in use).
+            if(code === 111) {
+                log('Shutdown Cluster requested, shutting down...');
+                if (shutdownTimer) clearTimeout(shutdownTimer);
+                process.exit(code);
+            }
+
             // Mark process finished.
             if (currentRestartingPid === worker.process.pid) {
                 currentRestartingPid = null;
@@ -171,6 +185,30 @@ GracefulCluster.start = function(options) {
                 return;
             }
             log('Cluster: worker ' + worker.process.pid + ' died (code: ' + code + '), restarting...');
+
+            // checks if too many attemps
+
+            var dateNow = new Date();
+
+            if (dateNow - lastDateChecked  < TIME_WINDOW) {
+
+                forksAttempts++;
+
+                if (forksAttempts >= maxForksAttempts) {
+
+                    log('Shutdown Cluster, too many forking attempts (' + forksAttempts + ') in ' + (dateNow - lastDateChecked) + ' milliseconds');
+                    if (shutdownTimer) clearTimeout(shutdownTimer);
+                    process.exit(1);
+
+                }
+
+            } else {
+
+                forksAttempts = 0;
+                lastDateChecked = dateNow;
+
+            }
+
             fork();
         });