Add dedup logic for users and user channels

lib/modules/posts.js

@@ -54,7 +54,9 @@ module.exports = function(context) {
     //
     // Query messages from Jabber and pipe
     // through the post transform and
-    // then to the output
+    // then to the output. We use pipe to
+    // handle very large data sets using
+    // streams
     //
     context.jabber
       .pipe(

lib/modules/users.js

@@ -1,67 +1,75 @@
 const _ = require('lodash')
 const Factory = require('../factory')
+const Utils = require('./utils')
 
 module.exports = function(context) {
   //
-  // Select all of the rooms
+  // Select all of the users
   //
   return context.jabber.fetch(
     'select * from dbo.tc_users'
   )
     //
-    // The convert them to channels and
-    // write them to the datafile
+    // Build up the user objects and then
+    // write them to the output
     //
     .then(function(results) {
-      console.log(`users: processing ${results.recordset.length} users`)
+      console.log(`users: processing ${results.recordset.length} records`)
       //
-      // Convenience references
-      //
-      var values = context.values
-      var define = context.config.define
-
-      //
-      // Map of userss
+      // Map of users
       //
       var users = {}
 
       //
       // Iterate over the record set and
-      // create a channel for each room
+      // assemble the user objects
       //
       results.recordset.forEach(function(record) {
+        //
+        // Clean the real_jid to ensure we don't
+        // have duplicates with /<string> suffixes
+        //
+        var real_jid = Utils.realJID(record.real_jid)
         //
         // Return a reference to the user in the user map
         // or add one if it doesn't yet exist
         //
-        var user = users[record.real_jid] = _.get(users, record.real_jid, {
-          username: toUsername(record.real_jid),
-          email: toEmail(record.real_jid),
-          auth_service: define.user.auth_service,
+        var user = users[real_jid] = _.get(users, real_jid, {
+          username: toUsername(real_jid),
+          email: real_jid,
+          auth_service: context.config.define.user.auth_service,
           teams: [{
-            name: values.team.name,
+            name: context.values.team.name,
             channels: []
           }]
         })
+
         //
-        // Look up the channel and add it to the
-        // user object if it exists
+        // Look up the channel based on the
+        // room id
         //
-        var channel = values.channels[record.room_jid]
-        if(channel) {
-          user.teams[0].channels.push({
-            name: channel.name
-          })
-        }
+        var channel = context.values.channels[record.room_jid]
         //
-        // Write the user data to the
-        // output
+        // Add it to the user
         //
+        if (channel) {
+          user.teams[0].channels = _.unionBy(user.teams[0].channels, [{
+            name: channel.name
+          }], 'name')
+        }
+      })
+
+      //
+      // Now that the users are assembled, write
+      // them to the output
+      //
+      _.forEach(users, function(user) {
         console.log(`users: ... writing ${user.username}`)
         context.output.write(
           Factory.user(user)
         )
       })
+
       //
       // Add the users map to the context
       //
@@ -73,15 +81,9 @@ module.exports = function(context) {
     })
 }
 
-//
-// Parse the email
-//
-const toEmail = function (jid='') {
-  return jid.split('/')[0]
-}
 //
 // Parse the username
 //
 const toUsername = function (jid='') {
-  return jid.split('@')[0] // jid.substr(0, jid.indexOf('@'))
+  return jid.split('@')[0]
 }

lib/modules/utils.js

@@ -0,0 +1,11 @@
+module.exports = {
+  //
+  // Removes trailing /<string> suffixes that
+  // may exist in the user ids. This happens
+  // if a user logs in to jabber more than
+  // once at the same time
+  //
+  realJID: function (jid='') {
+    return jid.split('/')[0]
+  }
+}