From 4d3a6610e789838447ef01b96bb7259d5a2379c1 Mon Sep 17 00:00:00 2001 From: Bob Date: Mon, 22 Feb 2016 18:25:54 +1100 Subject: [PATCH 1/8] WebDriver stuff --- package.json | 5 ++- run-headless-webdriver.js | 73 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 run-headless-webdriver.js diff --git a/package.json b/package.json index 097c102..9ca8c27 100644 --- a/package.json +++ b/package.json @@ -2,10 +2,9 @@ "name": "so-chatbot", "version": "0.2.0", "description": "A chatbot for StackOverflow.", - "main": "run-headless.js", + "main": "run-headless-webdriver.js", "dependencies": { - "nightmare": "^1.8.1" - }, + "webdriverio": "3.4.0" "devDependencies": { "browserify": "^13.0.0", "uglify-js": "^2.4.15" diff --git a/run-headless-webdriver.js b/run-headless-webdriver.js new file mode 100644 index 0000000..03a24c1 --- /dev/null +++ b/run-headless-webdriver.js @@ -0,0 +1,73 @@ +var config = require('./run-headless.config.json'); + +var webdriverio = require('webdriverio'); + +var client = webdriverio.remote({ + desiredCapabilities: { + browserName: 'firefox' + } + }); + +client + .init() + .url(config.loginUrl) + .getUrl() + .then(function(url) { + console.log('Loaded ' + url); + console.log('Logging in...'); + }) + .execute(function() { + openid.signin('stack_exchange'); + }) + .waitForExist('#affiliate-signin-iframe', 5000) + .frame('affiliate-signin-iframe') + .waitForExist('#email', 5000) + .setValue('#email', config.email) + .setValue('#password', config.password) + .submitForm('.login-form form') + .frame() + .getUrl() + .then(function(url) { + console.log('Login submitted; loaded ' + url); + }) + .url(config.roomUrl) + .getUrl() + .then(function(url) { + console.log('Loaded chatroom ' + url); + }) + .execute(function() { + var script = document.createElement('script'); + //script.src = 'http://localhost/master.js'; + script.src = 'http://localhost:8080/TestMimeMap/master.js'; + script.onload = function() { + //bot.activateDevMode(); + console.log('Loaded bot'); + bot.adapter.out.add('I have just been restarted! This happens daily automatically, or when my owner restarts me. Ready for commands.'); + }; + document.head.appendChild(script); + }) + .then(function() { + console.log('Chatbot injected'); + }) + .then(function() { + var readline = require('readline'); + var repl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + + console.log('You are now in a REPL with the remote page. Have fun!'); + + repl.on('line', function(data) { + client.execute(function(code) { + try { + return eval(code); + } catch (e) { + return e.message; + } + }).then(function(ret) { + console.log('$', ret); + repl.prompt(); + }); + }); + }); \ No newline at end of file From 7d4c4c5978f3754e36859c1c8b45ddcb18c9481c Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 23 Feb 2016 13:36:50 +1100 Subject: [PATCH 2/8] Working REPL, cleanup on exit, dump browser logs --- run-headless-webdriver.js | 98 ++++++++++++++++++++++++++++----- run-headless.config.json.SAMPLE | 9 ++- 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/run-headless-webdriver.js b/run-headless-webdriver.js index 03a24c1..aba87db 100644 --- a/run-headless-webdriver.js +++ b/run-headless-webdriver.js @@ -1,12 +1,71 @@ +var cleanupAndExit = function(exitCode) { + var exit = function() { + console.log('Exiting.'); + process.exit(exitCode); + }; + + console.log('Cleaning up...'); + setTimeout(function() { + console.warn('Failed to clean up in 5000ms. Some browser instances may remain.'); + exit(); + }, 5000); + if (client) { + if (dumpBrowserConsoleLoop) { + clearInterval(dumpBrowserConsoleLoop); + } + client.end().then(function() { + console.log('Cleaned up.'); + exit(); + }).catch(function(err) { + console.error(err.name, err.message); + console.warn('Cleanup failed. Some browser instances may remain.'); + exit(); + }); + } else { + exit(); + } +}; + +var signals = { + 'SIGHUP': 1, + 'SIGINT': 2, + 'SIGTERM': 15, + 'SIGBREAK': 21 +}; +for (var signal in signals) { + process.on(signal, function() { + console.log(signal + ' caught'); + cleanupAndExit(128 + signals[signal]); // posix signal 128 + signal number + }); +} +process.on('uncaughtException', function(err) { + console.err(err); + cleanupAndExit(1); +}); + var config = require('./run-headless.config.json'); var webdriverio = require('webdriverio'); -var client = webdriverio.remote({ - desiredCapabilities: { - browserName: 'firefox' - } +var client = webdriverio.remote(config.driverOptions); + +var logLevels = { + 'SEVERE': console.error, + 'WARNING': console.warn, + 'INFO': console.info, + 'DEBUG': console.debug +}; +var dumpBrowserConsole = function(limit) { + client.log('browser').then(function(logs) { + (limit ? logs.value.slice(-limit) : logs.value).forEach(function(log) { + (logLevels[log.value] || console.log).call(console, log.message); + }); }); +}; +var dumpBrowserConsoleLoop = null; +// phantomjs does not clear browser logs correctly +// dumping in a loop floods console cause they repeat +//var dumpBrowserConsoleLoop = setInterval(dumpBrowserConsole, 5000); client .init() @@ -19,9 +78,9 @@ client .execute(function() { openid.signin('stack_exchange'); }) - .waitForExist('#affiliate-signin-iframe', 5000) + .waitForExist('#affiliate-signin-iframe', 10000) .frame('affiliate-signin-iframe') - .waitForExist('#email', 5000) + .waitForExist('#email', 10000) .setValue('#email', config.email) .setValue('#password', config.password) .submitForm('.login-form form') @@ -55,19 +114,30 @@ client input: process.stdin, output: process.stdout }); + repl.on('SIGINT', function() { + process.emit('SIGINT'); + }); + repl.setPrompt('> '); console.log('You are now in a REPL with the remote page. Have fun!'); + var ready = true; repl.on('line', function(data) { - client.execute(function(code) { - try { - return eval(code); - } catch (e) { - return e.message; - } - }).then(function(ret) { - console.log('$', ret); + console.log('Attempting to run "' + data + '"'); + ready = false; + client.executeAsync(function(code) { + return eval(code); + }, data).then(function(ret) { + console.log('$', ret.value); + }).catch(function(err) { + console.log('!', err.name, err.message); + }).finally(function() { + // because phantomjs does not clear browser logs correctly + // we only display the last 5 to avoid flooding + dumpBrowserConsole(5); repl.prompt(); }); }); + + repl.prompt(); }); \ No newline at end of file diff --git a/run-headless.config.json.SAMPLE b/run-headless.config.json.SAMPLE index 46773e1..a1b8d86 100644 --- a/run-headless.config.json.SAMPLE +++ b/run-headless.config.json.SAMPLE @@ -3,5 +3,12 @@ "password": "what these are", "siteUrl": "https://stackoverflow.com", - "roomUrl": "https://chat.stackoverflow.com/rooms/1" + "roomUrl": "https://chat.stackoverflow.com/rooms/1", + "loginUrl": "https://stackexchange.com/users/login", + + "driverOptions": { + "desiredCapabilities": { + "browserName": "phantomjs" + } + } } \ No newline at end of file From 40551c4d6a9a64920eb09149e11b3422a905b6a9 Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 23 Feb 2016 18:35:14 +1100 Subject: [PATCH 3/8] Try to detect already logged in (if only pjs cookies worked...) --- run-headless-webdriver.js | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/run-headless-webdriver.js b/run-headless-webdriver.js index aba87db..54d830a 100644 --- a/run-headless-webdriver.js +++ b/run-headless-webdriver.js @@ -73,21 +73,27 @@ client .getUrl() .then(function(url) { console.log('Loaded ' + url); - console.log('Logging in...'); - }) - .execute(function() { - openid.signin('stack_exchange'); }) - .waitForExist('#affiliate-signin-iframe', 10000) - .frame('affiliate-signin-iframe') - .waitForExist('#email', 10000) - .setValue('#email', config.email) - .setValue('#password', config.password) - .submitForm('.login-form form') - .frame() - .getUrl() - .then(function(url) { - console.log('Login submitted; loaded ' + url); + .getText('=log in') + .then(function() { + console.log('Logging in...'); + return client + .execute(function() { + openid.signin('stack_exchange'); + }) + .waitForExist('#affiliate-signin-iframe', 10000) + .frame('affiliate-signin-iframe') + .waitForExist('#email', 10000) + .setValue('#email', config.email) + .setValue('#password', config.password) + .submitForm('.login-form form') + .frame() + .getUrl() + .then(function(url) { + console.log('Login submitted; loaded ' + url); + }); + }, function(err) { + console.log('Already logged in; skipping login'); }) .url(config.roomUrl) .getUrl() From 3564a6b86005200768b93b1c049c1d56e079b8a6 Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 23 Feb 2016 18:37:28 +1100 Subject: [PATCH 4/8] typo --- run-headless-webdriver.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-headless-webdriver.js b/run-headless-webdriver.js index 54d830a..738891b 100644 --- a/run-headless-webdriver.js +++ b/run-headless-webdriver.js @@ -39,7 +39,7 @@ for (var signal in signals) { }); } process.on('uncaughtException', function(err) { - console.err(err); + console.error(err); cleanupAndExit(1); }); From b16a71c9ce280edd0f7d50fbbc96c464887fd63d Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 23 Feb 2016 18:43:07 +1100 Subject: [PATCH 5/8] script url configurable now --- run-headless-webdriver.js | 7 +++---- run-headless.config.json.SAMPLE | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/run-headless-webdriver.js b/run-headless-webdriver.js index 738891b..11f8e2d 100644 --- a/run-headless-webdriver.js +++ b/run-headless-webdriver.js @@ -100,17 +100,16 @@ client .then(function(url) { console.log('Loaded chatroom ' + url); }) - .execute(function() { + .execute(function(scriptUrl) { var script = document.createElement('script'); - //script.src = 'http://localhost/master.js'; - script.src = 'http://localhost:8080/TestMimeMap/master.js'; + script.src = scriptUrl; script.onload = function() { //bot.activateDevMode(); console.log('Loaded bot'); bot.adapter.out.add('I have just been restarted! This happens daily automatically, or when my owner restarts me. Ready for commands.'); }; document.head.appendChild(script); - }) + }, config.scriptUrl) .then(function() { console.log('Chatbot injected'); }) diff --git a/run-headless.config.json.SAMPLE b/run-headless.config.json.SAMPLE index a1b8d86..6c2e77d 100644 --- a/run-headless.config.json.SAMPLE +++ b/run-headless.config.json.SAMPLE @@ -5,6 +5,8 @@ "siteUrl": "https://stackoverflow.com", "roomUrl": "https://chat.stackoverflow.com/rooms/1", "loginUrl": "https://stackexchange.com/users/login", + + "scriptUrl": "http://localhost/master.js", "driverOptions": { "desiredCapabilities": { From 18d6cbacc3572739e939190cf6c0cfb428180e2c Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 24 Feb 2016 11:18:46 +1100 Subject: [PATCH 6/8] Remove extra/leftover code --- run-headless-webdriver.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/run-headless-webdriver.js b/run-headless-webdriver.js index 11f8e2d..f5def7b 100644 --- a/run-headless-webdriver.js +++ b/run-headless-webdriver.js @@ -126,10 +126,8 @@ client console.log('You are now in a REPL with the remote page. Have fun!'); - var ready = true; repl.on('line', function(data) { console.log('Attempting to run "' + data + '"'); - ready = false; client.executeAsync(function(code) { return eval(code); }, data).then(function(ret) { From c8402d22b68ff2388a3a137e6204c7ee291f1e93 Mon Sep 17 00:00:00 2001 From: Bob Date: Thu, 3 Mar 2016 11:21:16 +1100 Subject: [PATCH 7/8] Update instructions in README.md --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5587fe8..34cb75e 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,15 @@ The bot is currently a big dangle-on script running in your browser. **Run `book If you wish, you can also run it headlessly on top of phantomjs and node: -* Install [phantomjs 2](http://phantomjs.org/) (yes, it has to be 2 and above). Differs from platform to platform. -* Install nightmare: `npm install nightmare` -* Edit your credentials into `run-headless.js` -* Hit the road: `env DEBUG=nightmare node run-headless.js` +1. Install [phantomjs 2](http://phantomjs.org/) (yes, it has to be 2 and above). Differs from platform to platform. +2. Install webdriverio: `npm install webdriverio@3.4.0` (4.x is broken) +3. Copy `run-headless.config.json.SAMPLE` into `run-headless.config.json` (update if necessary), and edit your credentials into it. +4. Hit the road: + + ```sh +screen -mdS pjs-server phantomjs --webdriver=4444 +screen -mdS bot node run-headless-webdriver.js +``` ###Building### From a7c96d4ccd97e6fe307d598f7ea7374fdd7781d7 Mon Sep 17 00:00:00 2001 From: Bob Date: Thu, 3 Mar 2016 11:23:45 +1100 Subject: [PATCH 8/8] gtfo tabs --- run-headless.config.json.SAMPLE | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/run-headless.config.json.SAMPLE b/run-headless.config.json.SAMPLE index 6c2e77d..7548278 100644 --- a/run-headless.config.json.SAMPLE +++ b/run-headless.config.json.SAMPLE @@ -1,13 +1,13 @@ { - "email": "you can guess", - "password": "what these are", + "email": "you can guess", + "password": "what these are", - "siteUrl": "https://stackoverflow.com", - "roomUrl": "https://chat.stackoverflow.com/rooms/1", + "siteUrl": "https://stackoverflow.com", + "roomUrl": "https://chat.stackoverflow.com/rooms/1", "loginUrl": "https://stackexchange.com/users/login", - + "scriptUrl": "http://localhost/master.js", - + "driverOptions": { "desiredCapabilities": { "browserName": "phantomjs"