-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
6 changed files
with
175 additions
and
48 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| const { Transform } = require('stream') | ||
| const Factory = require('../factory') | ||
| const Utils = require('./utils') | ||
|
|
||
| module.exports = function(context) { | ||
| return new Promise(function(resolve /*, reject */) { | ||
| // | ||
| // Set up the transform stream | ||
| // | ||
| const toDirectPost = new Transform({ | ||
| readableObjectMode: true, | ||
| writableObjectMode: true, | ||
| transform: function(message, encoding, callback) { | ||
| // | ||
| // Write the direct post to the | ||
| // output | ||
| // | ||
| context.output.write( | ||
| Factory.directPost({ | ||
| channel_members: Utils.members( | ||
| context.values.users, | ||
| message.to_jid, | ||
| message.from_jid | ||
| ), | ||
| user: Utils.username(context.values.users, message.from_jid), | ||
| message: Utils.body(message), | ||
| create_at: Utils.millis(message.sent_date) | ||
| }) | ||
| ) | ||
| // | ||
| // Invoke the call to mark that we are | ||
| // done with the chunk | ||
| // | ||
| return callback() | ||
| } | ||
| }).on('finish', function() { | ||
| console.log('direct posts: ... finished') | ||
| resolve(context) | ||
| }) | ||
|
|
||
| console.log('direct posts: streaming records') | ||
|
|
||
| // | ||
| // Query messages from Jabber and pipe | ||
| // through the post transform and | ||
| // then to the output. We use pipe to | ||
| // handle very large data sets using | ||
| // streams | ||
| // | ||
| context.jabber | ||
| .pipe(` | ||
| SELECT to_jid, from_jid, sent_date, body_string, body_text FROM dbo.jm | ||
| WHERE msg_type = 'c' | ||
| AND direction = 'I' | ||
| AND (body_string != '' or datalength(body_text) > 0)`, | ||
| toDirectPost | ||
| ) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| const { Transform } = require('stream') | ||
|
|
||
| module.exports = function(name, transform, callback) { | ||
| return new Transform({ | ||
| readableObjectMode: true, | ||
| writableObjectMode: true, | ||
| transform, | ||
| }).on('finish', function() { | ||
| console.log(`${name} ... finished`) | ||
| callback(context) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,11 +1,72 @@ | ||
| module.exports = { | ||
| // | ||
| // Removes trailing /<string> suffixes that | ||
| // may exist in the user ids. This happens | ||
| // if a user logs in to jabber more than | ||
| // once at the same time | ||
| // | ||
| realJID: function (jid='') { | ||
| return jid.split('/')[0].replace(/\\20/g, '.') | ||
| const _ = require('lodash') | ||
|
|
||
| // | ||
| // Declare utils object | ||
| // | ||
| const utils = {} | ||
|
|
||
| // | ||
| // Removes trailing /<string> suffixes that | ||
| // may exist in the user ids. This happens | ||
| // if a user logs in to jabber more than | ||
| // once at the same time | ||
| // | ||
| utils.realJID = function (jid='') { | ||
| return jid.split('/')[0].replace(/\\20/g, '.') | ||
| } | ||
|
|
||
| // | ||
| // Lookup values | ||
| // | ||
| utils.lookup = function (type, map, key) { | ||
| var found = map[key] | ||
|
|
||
| if(!found) { | ||
| console.log(`... ${type} ${key} not found`) | ||
| } | ||
|
|
||
| return found | ||
| } | ||
|
|
||
| // | ||
| // Obtain the username from a jid | ||
| // | ||
| utils.username = function (users, jid) { | ||
| return utils.lookup('user', users, utils.realJID(jid)).username | ||
| } | ||
|
|
||
| // | ||
| // Find the message body | ||
| // | ||
| utils.body = function (message) { | ||
| var body = message.body_string || message.body_text | ||
|
|
||
| if(!body) { | ||
| console.log('... message body is empty') | ||
| } | ||
|
|
||
| return body | ||
| } | ||
|
|
||
| // | ||
| // Coverts the to / from JIDs to a members | ||
| // array | ||
| // | ||
| utils.members = function(users, to, from) { | ||
| return _.sortBy([ | ||
| utils.username(users, to), | ||
| utils.username(users, from), | ||
| ]) | ||
| } | ||
|
|
||
| // | ||
| // Convert ISO to millis | ||
| // | ||
| utils.millis = function(date) { | ||
| return new Date(date).getTime() | ||
| } | ||
|
|
||
| // | ||
| // Export the functions | ||
| // | ||
| module.exports = utils |