adding logging

index.js

@@ -1,5 +1,6 @@
 const context = require('./context')
 const modules = require('./lib/modules')
+const log = require('./lib/log')
 
 const {
   start,
@@ -23,6 +24,6 @@ start(context)
   .then(directPosts)
   .then(end)
   .catch(function(err) {
-    console.error(err)
-    process.exit(1)
+    log.error(err)
+    process.exit(0)
   })

lib/log.js

@@ -0,0 +1,15 @@
+const bunyan = require('bunyan')
+
+module.exports = bunyan.createLogger({
+  name: 'mm-etl',
+  streams: [
+    {
+      level: 'info',
+      stream: process.stdout
+    },
+    {
+      level: 'debug',
+      path: 'mm-etl.log'
+    }
+  ]
+})

lib/modules/channels.js

@@ -3,19 +3,27 @@ const slug = require('slug')
 const XML = require('xmldoc')
 const Factory = require('../factory')
 
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'channels'
+})
+
 module.exports = function(context) {
   //
   // Select all of the rooms
   //
   return context.jabber.fetch(
-    'SELECT * FROM dbo.tc_rooms'
+    'SELECT room_jid, subject, config FROM dbo.tc_rooms'
   )
     //
     // The convert them to channels and
     // write them to the datafile
     //
     .then(function(results) {
-      console.log(`channels: ${results.recordset.length} records`)
+      log.info(`${results.recordset.length} records found`)
       //
       // Define a map to hold the channels
       // info for downstream modules
@@ -26,6 +34,7 @@ module.exports = function(context) {
       // create a channel for each room
       //
       results.recordset.forEach(function(room) {
+        log.debug(room)
         //
         // Define the channel and add
         // it to the context
@@ -42,7 +51,7 @@ module.exports = function(context) {
         // Write the channel data to the
         // output
         //
-        console.log(`channels: ... writing ${channel.name}`)
+        log.info(`... writing ${channel.name}`)
         context.output.write(
           Factory.channel(channel)
         )
@@ -58,7 +67,6 @@ module.exports = function(context) {
     })
 }
 
-
 //
 // Parses the name from a room jid
 //
@@ -90,7 +98,8 @@ const toType = function (xml='<x/>') {
   //
   var config = new XML.XmlDocument(xml)
   //
-  // Extract the value
+  // Extract the value. If it does not exist,
+  // we default to '1' - private
   //
   var value = _.get(
     config.childWithAttribute('var', 'members-only'),

lib/modules/directChannels.js

@@ -2,9 +2,17 @@ const Factory = require('../factory')
 const transform = require('./transform')
 const Utils = require('./utils')
 
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'directChannels'
+})
+
 module.exports = function(context) {
   return new Promise(function(resolve /*, reject */) {
-    console.log('direct channels: streaming records')
+    log.info('streaming records')
     //
     // Array to accumulate the channel
     // member pairs
@@ -46,6 +54,8 @@ module.exports = function(context) {
         var key = `${members[0]}|${members}[1]`
         if(!channels[key] && members[0] !== members[1]) {
           channels[key] = members
+
+          log.info(`... writing ${members}`)
           context.output.write(
             Factory.directChannel({
               members
@@ -63,6 +73,7 @@ module.exports = function(context) {
       // on finish
       //
       function() {
+        log.info('... finished')
         resolve(context)
       })
     )

lib/modules/directPosts.js

@@ -2,9 +2,22 @@ const Factory = require('../factory')
 const transform = require('./transform')
 const Utils = require('./utils')
 
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'directPosts'
+})
+
 module.exports = function(context) {
   return new Promise(function(resolve /*, reject */) {
-    console.log('direct posts: streaming records')
+    log.info('streaming records')
+    //
+    // Keep track of the number of posts
+    // written for logging
+    //
+    var written = 0
     //
     // Query messages from Jabber and pipe
     // through the post transform and
@@ -41,6 +54,13 @@ module.exports = function(context) {
           })
         )
         //
+        // Log progress periodically
+        //
+        written += 1
+        if(written % 1000 == 0) {
+          log.info(`... wrote ${written} posts`)
+        }
+        //
         // Invoke the call to mark that we are
         // done with the chunk
         //
@@ -51,6 +71,7 @@ module.exports = function(context) {
       // on finish
       //
       function() {
+        log.info(`... finished writing ${written} posts`)
         resolve(context)
       })
     )

lib/modules/end.js

@@ -1,4 +1,12 @@
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'end'
+})
+
 module.exports = function(context) {
-  console.log('end')
+  log.info('end')
   context.output.end()
 }

lib/modules/posts.js

@@ -2,9 +2,22 @@ const Factory = require('../factory')
 const transform = require('./transform')
 const Utils = require('./utils')
 
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'posts'
+})
+
 module.exports = function(context) {
   return new Promise(function(resolve /*, reject */) {
-    console.log('posts: streaming records')
+    log.info('streaming records')
+    //
+    // Keep track of the number of posts
+    // written for logging
+    //
+    var written = 0
     //
     // Query messages from Jabber and pipe
     // through the post transform and
@@ -27,7 +40,8 @@ module.exports = function(context) {
       //
       // Define the tranform
       //
-      transform('direct channels', function(message, encoding, callback) {
+      transform('posts', function(message, encoding, callback) {
+        log.debug(message)
         //
         // Lookup the channel
         //
@@ -49,6 +63,10 @@ module.exports = function(context) {
         // post to the output
         //
         if(channel && username && body) {
+          //
+          // Write the post object to the
+          // output
+          //
           context.output.write(
             Factory.post({
               team: context.values.team.name,
@@ -58,8 +76,15 @@ module.exports = function(context) {
               create_at: new Date(message.sent_date).getTime()
             })
           )
+          //
+          // Log progress periodically
+          //
+          written += 1
+          if(written % 1000 == 0) {
+            log.info(`... wrote ${written} posts`)
+          }
         } else {
-          console.log(`posts: ... [!] skipping message ${message.msg_id}`)
+          log.warn(`... [!] skipping message ${message.msg_id}`)
         }
         //
         // Invoke the call to mark that we are
@@ -72,6 +97,7 @@ module.exports = function(context) {
       // on finish
       //
       function() {
+        log.info(`... finished writing ${written} posts`)
         resolve(context)
       })
     )

lib/modules/start.js

@@ -1,13 +1,21 @@
 const datafile = require('../datafile')
 
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'start'
+})
+
 module.exports = function(context) {
-  console.log('start: connecting to jabber')
+  log.info('connecting to jabber')
   //
   // Connect to the jabber database
   //
   return context.jabber.connect(context.config)
     .then(function() {
-      console.log(`start: creating file '${context.config.target.filename}'`)
+      log.info(`creating file '${context.config.target.filename}'`)
       //
       // Create the datafile and add
       // it to the context

lib/modules/team.js

@@ -1,7 +1,15 @@
 const Factory = require('../factory')
 
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'team'
+})
+
 module.exports = function(context) {
-  console.log(`team: writing team '${context.config.define.team.name}'`)
+  log.info(`writing team '${context.config.define.team.name}'`)
   //
   // Store the team object for use
   // by other modules

lib/modules/transform.js

@@ -5,8 +5,5 @@ module.exports = function(name, transform, callback) {
     readableObjectMode: true,
     writableObjectMode: true,
     transform,
-  }).on('finish', function() {
-    console.log(`${name} ... finished`)
-    callback()
-  })
+  }).on('finish', callback)
 }

lib/modules/users.js

@@ -2,6 +2,14 @@ const _ = require('lodash')
 const Factory = require('../factory')
 const Utils = require('./utils')
 
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'users'
+})
+
 module.exports = function(context) {
   //
   // Select all of the users
@@ -21,7 +29,7 @@ module.exports = function(context) {
     // write them to the output
     //
     .then(function(results) {
-      console.log(`users: ${results.recordset.length} records`)
+      log.info(`${results.recordset.length} records found`)
       //
       // Map of users
       //
@@ -32,6 +40,7 @@ module.exports = function(context) {
       // assemble the user objects
       //
       results.recordset.forEach(function(record) {
+        log.debug(record)
         //
         // Clean the real_jid to ensure we don't
         // have duplicates with /<string> suffixes
@@ -70,7 +79,7 @@ module.exports = function(context) {
       // them to the output
       //
       _.forEach(users, function(user) {
-        console.log(`users: ... writing ${user.username}`)
+        log.info(`... writing ${user.username}`)
         context.output.write(
           Factory.user(user)
         )

lib/mssql.js

@@ -1,4 +1,5 @@
 const SQL = require('mssql')
+const log = require('./log')
 
 //
 // Declare internals
@@ -51,5 +52,6 @@ module.exports.pipe = function(query, writable) {
 // Set up an error handler
 //
 SQL.on('error', function(err) {
-  console.error(err)
+  log.error(err)
+  throw(err)
 })

package.json

@@ -13,6 +13,7 @@
     "url": "https://github.com/Brightscout/mattermost-etl/issues"
   },
   "dependencies": {
+    "bunyan": "^1.8.10",
     "joi": "^10.6.0",
     "lodash": "^4.17.4",
     "mssql": "^4.0.4",
@@ -33,6 +34,6 @@
     "test": "npm run lint && mocha lib --recursive",
     "test-watch": "npm run test -- --watch",
     "test-coverage": "istanbul cover _mocha lib -- --recursive",
-    "start": "node index.js"
+    "start": "node index.js | ./node_modules/.bin/bunyan"
   }
 }