Added listener to exit process when db connection fails

configs/.env.development

@@ -2,6 +2,7 @@ ASSERT_HTTPS=false
 AWS_ACCESS_KEY_ID=AKIAJXF6CNCWBWNX7JZQ
 AWS_SECRET_ACCESS_KEY=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
 CONTAINER_STOP_LIMIT=10
+DB_CONNECTION_TIMEOUT=10000
 DOCKER_IMAGE_BUILDER_CACHE=/git-cache
 DOMAIN=localhost:3001
 FULL_API_DOMAIN=http://localhost:3030

configs/.env.production-delta

@@ -1,4 +1,5 @@
 CONTAINER_STOP_LIMIT=10
+DB_CONNECTION_TIMEOUT=10000
 DOCKER_IMAGE_BUILDER_CACHE=/git-cache
 DOCKER_IMAGE_BUILDER_LAYER_CACHE=/layer-cache
 DOMAIN=runnable.io

configs/.env.production-epsilon

@@ -1,6 +1,7 @@
 ALLOW_ALL_CORS=true
 BUNYAN_LOG_USE_SRC=true
 CONTAINER_STOP_LIMIT=10
+DB_CONNECTION_TIMEOUT=10000
 DOCKER_IMAGE_BUILDER_CACHE=/git-cache
 DOCKER_IMAGE_BUILDER_LAYER_CACHE=/layer-cache
 DOMAIN=runnable-beta.com

configs/.env.production-gamma

@@ -1,6 +1,7 @@
 ALLOW_ALL_CORS=true
 BUNYAN_LOG_USE_SRC=true
 CONTAINER_STOP_LIMIT=10
+DB_CONNECTION_TIMEOUT=10000
 DOCKER_IMAGE_BUILDER_CACHE=/git-cache
 DOCKER_IMAGE_BUILDER_LAYER_CACHE=/layer-cache
 DOMAIN=runnable-gamma.com

configs/.env.staging

@@ -3,6 +3,7 @@ AWS_ACCESS_KEY_ID=AKIAJXF6CNCWBWNX7JZQ
 AWS_SECRET_ACCESS_KEY=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
 BUNYAN_BATCH_LOG_COUNT=5
 DATADOG_HOST=datadog-staging-codenow.runnableapp.com
+DB_CONNECTION_TIMEOUT=10000
 DEBUG=runnable-api:docker*,api:worker*,hermes*
 DOCKER_IMAGE_BUILDER_CACHE=/git-cache
 DOCKER_IMAGE_BUILDER_LAYER_CACHE=/layer-cache

configs/.env.test

@@ -9,6 +9,7 @@ BUNYAN_LOG_USE_SRC=true
 CONTAINER_STOP_LIMIT=10
 COOKIE_SECRET=\$up3r,$3<r3t
 CREAM_HOST=localhost:9879
+DB_CONNECTION_TIMEOUT=60000
 DEFAULT_PAGE_LIMIT=25
 DOMAIN=runnable.io
 FULL_API_DOMAIN=http://localhost:3031

lib/models/mongo/mongoose-control.js

@@ -52,6 +52,14 @@ mongooseControl.start = function (cb) {
   }
 
   mongoose.connect(process.env.MONGO, mongooseOptions, cb)
+
+  mongoose.connection.on('disconnected', function () {
+    if (!mongoose.connection._hasOpened) {
+      mongooseControl._exitIfFailedToOpen()
+    } else {
+      mongooseControl._exitIfFailedToReconnect()
+    }
+  })
 }
 
 mongooseControl.stop = function (cb) {
@@ -63,3 +71,18 @@ mongooseControl.stop = function (cb) {
     })
   })
 }
+
+mongooseControl._exitIfFailedToOpen = function () {
+  log.fatal({message: 'Failed to connect to ' + process.env.MONGO + ' failed to establish a connection to mongodb'})
+  process.exit(1)
+}
+
+mongooseControl._exitIfFailedToReconnect = function () {
+  log.error({message: 'Failed to connect to ' + process.env.MONGO + ' failed to establish a connection to mongodb'})
+  setTimeout(function () {
+    if (!mongoose.connection.readyState) {
+      log.fatal({message: 'Exiting nodejs process due to mongodb connection failure'})
+      process.exit(1)
+    }
+  }, process.env.DB_CONNECTION_TIMEOUT)
+}

unit/models/mongo/mongoose-control.js

@@ -11,6 +11,7 @@ var Code = require('code')
 var fs = require('fs')
 var mongoose = require('mongoose')
 var sinon = require('sinon')
+var clock
 
 var mongooseControl = require('models/mongo/mongoose-control')
 
@@ -103,5 +104,73 @@ describe('mongoose-control', function () {
         done()
       })
     })
+
+    describe('handling mongodb disconnect events', function () {
+      beforeEach(function (done) {
+        sinon.stub(mongoose.connection, 'on').yields()
+        sinon.stub(process, 'exit')
+        sinon.stub(mongooseControl, '_exitIfFailedToReconnect')
+        sinon.stub(mongooseControl, '_exitIfFailedToOpen')
+        done()
+      })
+
+      afterEach(function (done) {
+        mongooseControl._exitIfFailedToReconnect.restore()
+        mongooseControl._exitIfFailedToOpen.restore()
+        mongoose.connection.on.restore()
+        process.exit.restore()
+        done()
+      })
+
+      it('should exit if it cannot connect', function (done) {
+        mongooseControl.start(function (err) {
+          expect(err).to.not.exist()
+          sinon.assert.notCalled(mongooseControl._exitIfFailedToReconnect)
+          sinon.assert.calledOnce(mongooseControl._exitIfFailedToOpen)
+          done()
+        })
+      })
+
+      it('should attempt a retry if connection existed', function (done) {
+        mongoose.connection._hasOpened = true
+        mongooseControl.start(function (err) {
+          expect(err).to.not.exist()
+          sinon.assert.notCalled(mongooseControl._exitIfFailedToOpen)
+          sinon.assert.calledOnce(mongooseControl._exitIfFailedToReconnect)
+          done()
+        })
+      })
+    })
+
+    describe('exiting node process when db disconnects', function () {
+      beforeEach(function (done) {
+        clock = sinon.useFakeTimers()
+        sinon.stub(mongoose.connection, 'on').yields()
+        sinon.stub(process, 'exit')
+        done()
+      })
+
+      afterEach(function (done) {
+        mongoose.connection.on.restore()
+        process.exit.restore()
+        clock.restore()
+        done()
+      })
+
+      it('should exit immediately if it cannot connect', function (done) {
+        mongooseControl._exitIfFailedToOpen()
+        sinon.assert.calledOnce(process.exit)
+        done()
+      })
+
+      it('should attempt to reconnect when it was connected once', function (done) {
+        mongooseControl._exitIfFailedToReconnect()
+        clock.tick(1000)
+        sinon.assert.notCalled(process.exit)
+        clock.tick(60000)
+        sinon.assert.calledOnce(process.exit)
+        done()
+      })
+    })
   })
 })