Initial code commit

index.js

@@ -0,0 +1,2 @@
+exports.GracefulCluster     = require('./lib/graceful-cluster');
+exports.GracefulServer      = require('./lib/graceful-server');

lib/graceful-cluster.js

@@ -0,0 +1,191 @@
+var cluster = require('cluster');
+var numCPUs = require('os').cpus().length;
+
+var GracefulCluster = module.exports;
+
+/*
+
+ Starts node.js cluster with graceful restart/shutdown.
+
+ Params:
+
+ - options.serverFunction        - required, function with worker logic.
+ - options.log                   - function, custom log function, console.log used by default.
+ - options.shutdownTimeout       - ms, force worker shutdown on SIGTERM timeout.
+ - options.disableGraceful       - disable graceful shutdown for faster debug.
+ - options.restartOnMemory       - bytes, restart worker on memory usage.
+ - options.restartOnTimeout      - ms, restart worker by timer.
+
+ Graceful restart performed by USR2 signal:
+
+ pkill -USR2 <cluster_process_name>
+
+ or
+
+ kill -s SIGUSR2 <cluster_pid>
+
+ */
+GracefulCluster.start = function(options) {
+
+    var serverFunction = options.serverFunction;
+
+    if (!serverFunction) {
+        throw new Error('Graceful cluster: `options.serverFunction` required.');
+    }
+
+    var log = options.log || console.log;
+    var shutdownTimeout = options.shutdownTimeout || 5000;
+    var disableGraceful = options.disableGraceful;
+
+    if (cluster.isMaster) {
+
+        var sigkill = false;
+        var workersCount = 0;
+        var listeningWorkersCount = 0;
+        var restartQueue = [];
+
+        // Prevent killing all workers at same time when restarting.
+        function checkRestartQueue() {
+            // Kill one worker only if maximum count are working.
+            if (restartQueue.length > 0 && listeningWorkersCount === numCPUs) {
+                var pid = restartQueue.shift();
+                try {
+                    // Send SIGTERM signal to worker. SIGTERM starts graceful shutdown of worker inside it.
+                    process.kill(pid);
+                } catch(ex) {
+                    // Fail silent on 'No such process'. May occur when kill message received after kill initiated but not finished.
+                    if (ex.code !== 'ESRCH') {
+                        throw ex;
+                    }
+                }
+            }
+        }
+
+        // Create fork with 'on restart' message event listener.
+        function fork() {
+            cluster.fork().on('message', function(message) {
+                if (message.cmd === 'restart' && message.pid && restartQueue.indexOf(message.pid) === -1) {
+                    // When worker asks to restart gracefully in cluster, then add it to restart queue.
+                    restartQueue.push(message.pid);
+                    checkRestartQueue();
+                }
+            });
+        }
+
+        // Fork workers.
+        for (var i = 0; i < numCPUs; i++) {
+            fork();
+        }
+
+        // Check if has alive workers and exit.
+        function checkIfNoWorkersAndExit() {
+            if (!workersCount) {
+                log('Cluster graceful shutdown: done.');
+                process.exit(0);
+            } else {
+                log('Cluster graceful shutdown: wait ' + workersCount + ' worker' + (workersCount > 1 ? 's' : '') + '.');
+            }
+        }
+
+        function startShutdown() {
+
+            if (disableGraceful) {
+                process.exit(0);
+                return;
+            }
+
+            // Shutdown timeout.
+            setTimeout(function() {
+                log('Cluster graceful shutdown: timeout, force exit.');
+                process.exit(0);
+            }, shutdownTimeout);
+
+            // Shutdown mode.
+            sigkill = true;
+
+            // Log how many workers alive.
+            checkIfNoWorkersAndExit();
+
+            for (var id in cluster.workers) {
+                // Send SIGTERM signal to all workers. SIGTERM starts graceful shutdown of worker inside it.
+                process.kill(cluster.workers[id].process.pid);
+            }
+        }
+        process.on('SIGTERM',startShutdown);
+        process.on('SIGINT',startShutdown);
+
+        // Gracefuly restart with 'kill -s SIGUSR2 <pid>'.
+        process.on('SIGUSR2',function() {
+            for (var id in cluster.workers) {
+                // Push all workers to restart queue.
+                restartQueue.push(cluster.workers[id].process.pid);
+            }
+            checkRestartQueue();
+        });
+
+        cluster.on('fork', function(worker) {
+            workersCount++;
+            worker.on('listening', function() {
+                listeningWorkersCount++;
+                // New worker online, maybe all online, try restart other.
+                checkRestartQueue();
+            });
+            log('Cluster: worker ' + worker.process.pid + ' started.');
+        });
+
+        cluster.on('exit', function(worker, code, signal) {
+            workersCount--;
+            listeningWorkersCount--;
+            if (sigkill) {
+                checkIfNoWorkersAndExit();
+                return;
+            }
+            log('Cluster: worker ' + worker.process.pid + ' died (code: ' + code + '), restarting...');
+            fork();
+        });
+
+        process.on('uncaughtException', function(err) {
+            if (disableGraceful) {
+                log('Cluster error:', err.stack);
+            } else {
+                log('Cluster error:', err.message);
+            }
+        });
+
+    } else {
+
+        // Start worker.
+        serverFunction();
+
+        // Self restart logic.
+
+        if (options.restartOnMemory) {
+            setInterval(function() {
+                var mem = process.memoryUsage().rss;
+                if (mem > options.restartOnMemory) {
+                    log('Cluster: worker ' + process.pid + ' used too much memory (' + Math.round(mem / (1024*1024)) + ' MB), restarting...');
+                    GracefulCluster.gracefullyRestartCurrentWorker();
+                }
+
+            }, 1000);
+        }
+
+        if (options.restartOnTimeout) {
+
+            setInterval(function() {
+
+                log('Cluster: worker ' + process.pid + ' restarting by timer...');
+                GracefulCluster.gracefullyRestartCurrentWorker();
+
+            }, options.restartOnTimeout);
+        }
+    }
+};
+
+GracefulCluster.gracefullyRestartCurrentWorker = function() {
+    // Perform restart by cluster to prevent all workers offline.
+    process.send({
+        cmd: 'restart',
+        pid: process.pid
+    });
+};

lib/graceful-server.js

@@ -0,0 +1,83 @@
+var EventEmitter = require('events').EventEmitter;
+
+var GracefulServer = function(options) {
+
+    this.server = options.server;
+
+    if (!this.server) {
+        throw new Error('Graceful shutdown: `options.server` required.');
+    }
+
+    this.log = options.log || console.log;
+    this.shutdownTimeout = options.shutdownTimeout || 5000;
+
+    // Solution got from: https://github.com/nodejs/node-v0.x-archive/issues/9066#issuecomment-124210576
+
+    var state = this.state = new EventEmitter;
+    state.setMaxListeners(0);
+    state.shutdown = false;
+
+    this.REQUESTS_COUNT = 0;
+
+    var that = this;
+
+    this.server.on('connection', function (socket) {
+        function destroy() {
+            if (socket._GS_HAS_OPEN_REQUESTS === 0) socket.destroy();
+        }
+        socket._GS_HAS_OPEN_REQUESTS = 0;
+        state.once('shutdown', destroy);
+        socket.once('close', function () {
+            state.removeListener('shutdown', destroy);
+        });
+    });
+
+    this.server.on('request', function (req, res) {
+        var socket = req.connection;
+        socket._GS_HAS_OPEN_REQUESTS++;
+        that.REQUESTS_COUNT++;
+        res.on('finish', function () {
+            that.REQUESTS_COUNT--;
+            if (state.shutdown) that.logShutdown();
+            socket._GS_HAS_OPEN_REQUESTS--;
+            if (state.shutdown && socket._GS_HAS_OPEN_REQUESTS === 0) socket.destroy();
+        });
+    });
+
+    function shutdown() {
+        that.shutdown();
+    }
+    process.on('SIGTERM', shutdown);
+    process.on('SIGINT', shutdown);
+};
+
+GracefulServer.prototype.logShutdown = function() {
+    this.log('pid:' + process.pid + ' graceful stutdown: ' + (this.REQUESTS_COUNT ? 'wait ' + this.REQUESTS_COUNT + ' request' + (this.REQUESTS_COUNT > 1 ? 's': '') + ' to finish.' : 'no active connections.'));
+};
+
+GracefulServer.prototype.shutdown = function() {
+
+    this.logShutdown();
+
+    if (this.state.shutdown) {
+        // Prevent repeat shutdown.
+        return;
+    }
+
+    var that = this;
+
+    setTimeout(function() {
+        that.log('pid:' + process.pid + ' graceful stutdown: timeout, force exit.');
+        process.exit(0);
+    }, this.shutdownTimeout);
+
+    this.server.close(function() {
+        that.log('pid:' + process.pid + ' graceful stutdown: done.');
+        process.exit(0);
+    });
+
+    this.state.shutdown = true;
+    this.state.emit('shutdown');
+};
+
+module.exports = GracefulServer;

package.json

@@ -0,0 +1,15 @@
+{
+  "name": "graceful-cluster",
+  "version": "0.0.1",
+  "description": "Gracefully restart node.js http server or cluster with zero downtime. Shutdown server without active inbound connections reset.",
+  "main": "index.js",
+  "keywords": [
+    "http",
+    "server",
+    "cluster",
+    "restart",
+    "shutdown",
+    "gracefully"
+  ],
+  "license": "MIT"
+}