From 9fe2bde45a53560cbe354fe2c77c976401a9d63c Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 10:17:38 +0200 Subject: [PATCH 01/62] firebase script is included --- logger.js | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/logger.js b/logger.js index 063abc2..ce29c87 100644 --- a/logger.js +++ b/logger.js @@ -1,8 +1,25 @@ -var firebase_reference = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); +var isActive; + +window.onfocus = function() { + isActive = true; +}; + +window.onblur = function() { + isActive = false; +}; +// Include Firebase library +var firebase_script = document.createElement('script'); firebase_script.src = 'https://cdn.firebase.com/js/client/2.2.9/firebase.js'; document.head.appendChild(firebase_script); +var firebase_reference = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); +// Log event to Firebase bucket firebase_reference.push({ - pathname: document.location.pathname, - search: document.location.search, - hostname: document.location.hostname, - hash: document.location.hash + pathname: document.location.pathname, + search: document.location.search, + hostname: document.location.hostname, + hash: document.location.hash }); + +// test +setInterval(function() { + console.log(window.isActive ? 'active' : 'inactive'); +}, 1000); \ No newline at end of file From 2bd0aacb5dcc203f91dbd041aa3c383596773dec Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 10:30:30 +0200 Subject: [PATCH 02/62] firebase init on script load --- logger.js | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/logger.js b/logger.js index ce29c87..3a00255 100644 --- a/logger.js +++ b/logger.js @@ -8,18 +8,23 @@ window.onblur = function() { isActive = false; }; // Include Firebase library -var firebase_script = document.createElement('script'); firebase_script.src = 'https://cdn.firebase.com/js/client/2.2.9/firebase.js'; document.head.appendChild(firebase_script); +var firebase_script = document.createElement('script'); +firebase_script.src = 'https://cdn.firebase.com/js/client/2.2.9/firebase.js'; +document.getElementsByTagName('head')[0].appendChild(firebase_script); -var firebase_reference = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); -// Log event to Firebase bucket -firebase_reference.push({ - pathname: document.location.pathname, - search: document.location.search, - hostname: document.location.hostname, - hash: document.location.hash -}); +// when Firebase library is loaded - initialize and use it +firebase_script.onload = function() { + var firebase_reference = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); + // Log event to Firebase bucket + firebase_reference.push({ + pathname: document.location.pathname, + search: document.location.search, + hostname: document.location.hostname, + hash: document.location.hash + }); -// test -setInterval(function() { - console.log(window.isActive ? 'active' : 'inactive'); -}, 1000); \ No newline at end of file + // test + setInterval(function() { + console.log(window.isActive ? 'active' : 'inactive'); + }, 1000); +}; From cfc221bd22f2e37d0eb13dfb0fa3c788e6d4a30d Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 10:36:10 +0200 Subject: [PATCH 03/62] LOGGER object keeping settings --- logger.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/logger.js b/logger.js index 3a00255..c655228 100644 --- a/logger.js +++ b/logger.js @@ -1,3 +1,18 @@ +// settings +var LOGGER = { + 'firebase':{ + 'base': "https://cdn.firebase.com/", + 'version':"2.2.9", + 'bucket':"crowdworker-logger", + getScriptURL: function(){ + return LOGGER.firebase.base+"js/client/"+LOGGER.firebase.version+"/firebase.js"; + }, + getBucketURL: function(){ + return "https://"+LOGGER.firebase.bucket+".firebaseio.com/"; + } + } +} + var isActive; window.onfocus = function() { @@ -9,12 +24,12 @@ window.onblur = function() { }; // Include Firebase library var firebase_script = document.createElement('script'); -firebase_script.src = 'https://cdn.firebase.com/js/client/2.2.9/firebase.js'; +firebase_script.src = LOGGER.firebase.getScript.getScriptURL(); document.getElementsByTagName('head')[0].appendChild(firebase_script); // when Firebase library is loaded - initialize and use it firebase_script.onload = function() { - var firebase_reference = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); + var firebase_reference = new Firebase(LOGGER.firebase.getScript.getScriptURL()+"trials"); // Log event to Firebase bucket firebase_reference.push({ pathname: document.location.pathname, From 4d4a4a60475f1ef38f030ae07e0bc61e76d3d36b Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 10:38:16 +0200 Subject: [PATCH 04/62] getScript removed --- logger.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logger.js b/logger.js index c655228..8adcbe9 100644 --- a/logger.js +++ b/logger.js @@ -24,12 +24,12 @@ window.onblur = function() { }; // Include Firebase library var firebase_script = document.createElement('script'); -firebase_script.src = LOGGER.firebase.getScript.getScriptURL(); +firebase_script.src = LOGGER.firebase.getScriptURL(); document.getElementsByTagName('head')[0].appendChild(firebase_script); // when Firebase library is loaded - initialize and use it firebase_script.onload = function() { - var firebase_reference = new Firebase(LOGGER.firebase.getScript.getScriptURL()+"trials"); + var firebase_reference = new Firebase(LOGGER.firebase.getScriptURL()+"trials"); // Log event to Firebase bucket firebase_reference.push({ pathname: document.location.pathname, From 0c196b8c3c7455653da2c0305d974d8f9fb17d24 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 10:42:05 +0200 Subject: [PATCH 05/62] scriptURL - > bucketURL --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index 8adcbe9..aa4e625 100644 --- a/logger.js +++ b/logger.js @@ -29,7 +29,7 @@ document.getElementsByTagName('head')[0].appendChild(firebase_script); // when Firebase library is loaded - initialize and use it firebase_script.onload = function() { - var firebase_reference = new Firebase(LOGGER.firebase.getScriptURL()+"trials"); + var firebase_reference = new Firebase(LOGGER.firebase.getBucketURL()+"trials"); // Log event to Firebase bucket firebase_reference.push({ pathname: document.location.pathname, From 4e3392e7a9bc0773508b724f85906d13ff8f189c Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 11:30:57 +0200 Subject: [PATCH 06/62] restructured everything to EDA_LOGGER --- logger.js | 83 +++++++++++++++++++++++++------------------------------ 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/logger.js b/logger.js index aa4e625..f73fc16 100644 --- a/logger.js +++ b/logger.js @@ -1,45 +1,38 @@ -// settings -var LOGGER = { - 'firebase':{ - 'base': "https://cdn.firebase.com/", - 'version':"2.2.9", - 'bucket':"crowdworker-logger", - getScriptURL: function(){ - return LOGGER.firebase.base+"js/client/"+LOGGER.firebase.version+"/firebase.js"; - }, - getBucketURL: function(){ - return "https://"+LOGGER.firebase.bucket+".firebaseio.com/"; - } - } -} - -var isActive; - -window.onfocus = function() { - isActive = true; -}; - -window.onblur = function() { - isActive = false; -}; -// Include Firebase library -var firebase_script = document.createElement('script'); -firebase_script.src = LOGGER.firebase.getScriptURL(); -document.getElementsByTagName('head')[0].appendChild(firebase_script); - -// when Firebase library is loaded - initialize and use it -firebase_script.onload = function() { - var firebase_reference = new Firebase(LOGGER.firebase.getBucketURL()+"trials"); - // Log event to Firebase bucket - firebase_reference.push({ - pathname: document.location.pathname, - search: document.location.search, - hostname: document.location.hostname, - hash: document.location.hash - }); - - // test - setInterval(function() { - console.log(window.isActive ? 'active' : 'inactive'); - }, 1000); -}; +var EDA_LOGGER = EDA_LOGGER || (function() { + var _args = { + key_name: "test_name", + key_value: "test_value" + }; // private + return { + init: function(Args) { + _args = Args; + // Include Firebase library + connect_firebase(); + // some other initializing + // when Firebase library is loaded - initialize and use it + firebase_script.onload = function() { + var firebase_reference = new Firebase(LOGGER.firebase.getBucketURL() + "trials"); + // Log event to Firebase bucket + }; + }, + init_firebase: function() { + var firebase_script = document.createElement('script'); + firebase_script.src = "https://cdn.firebase.com/js/client/2.2.9/firebase.js"; + document.getElementsByTagName('head')[0].appendChild(firebase_script); + firebase_script.onload = function() { + _args["firebase_instance"] = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); + log_event(_args["firebase_instance"]); + }; + }, + log_event: function(firebase_reference) { + firebase_reference.push({ + pathname: document.location.pathname, + search: document.location.search, + hostname: document.location.hostname, + hash: document.location.hash, + key_name: _args.key_name, + key_value: _args.key_value + }); + } + }; +}()); \ No newline at end of file From 6c12a4e57c9e457a792777a08629b3d61c712147 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 11:32:16 +0200 Subject: [PATCH 07/62] no message --- logger.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/logger.js b/logger.js index f73fc16..5d848ab 100644 --- a/logger.js +++ b/logger.js @@ -7,13 +7,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { init: function(Args) { _args = Args; // Include Firebase library - connect_firebase(); - // some other initializing - // when Firebase library is loaded - initialize and use it - firebase_script.onload = function() { - var firebase_reference = new Firebase(LOGGER.firebase.getBucketURL() + "trials"); - // Log event to Firebase bucket - }; + init_firebase(); }, init_firebase: function() { var firebase_script = document.createElement('script'); From 2a1d8aeb62c20c4a46053264d9da63b7c04621a7 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 11:34:09 +0200 Subject: [PATCH 08/62] debug --- logger.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/logger.js b/logger.js index 5d848ab..9a896cb 100644 --- a/logger.js +++ b/logger.js @@ -12,6 +12,8 @@ var EDA_LOGGER = EDA_LOGGER || (function() { init_firebase: function() { var firebase_script = document.createElement('script'); firebase_script.src = "https://cdn.firebase.com/js/client/2.2.9/firebase.js"; + console.log(firebase_script.src); + console.log(document.getElementsByTagName('head')[0]); document.getElementsByTagName('head')[0].appendChild(firebase_script); firebase_script.onload = function() { _args["firebase_instance"] = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); From dc60bb7fc46d0470a3d3b5696fee946696fafba4 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 5 Oct 2015 11:38:03 +0200 Subject: [PATCH 09/62] "this" added --- logger.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logger.js b/logger.js index 9a896cb..1d3dc08 100644 --- a/logger.js +++ b/logger.js @@ -7,7 +7,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { init: function(Args) { _args = Args; // Include Firebase library - init_firebase(); + this.init_firebase(); }, init_firebase: function() { var firebase_script = document.createElement('script'); @@ -17,7 +17,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { document.getElementsByTagName('head')[0].appendChild(firebase_script); firebase_script.onload = function() { _args["firebase_instance"] = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); - log_event(_args["firebase_instance"]); + this.log_event(_args["firebase_instance"]); }; }, log_event: function(firebase_reference) { From 657432f711dc2dbbcc52f79bfbfae473290ea7c2 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 6 Oct 2015 14:18:13 +0200 Subject: [PATCH 10/62] logs tree. --- index.html | 13 +++++++++++-- logger.js | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/index.html b/index.html index e6ba478..a55e350 100644 --- a/index.html +++ b/index.html @@ -1,8 +1,16 @@ + - - + + +
@@ -18,4 +26,5 @@

CrowdFlower Launcher

+ diff --git a/logger.js b/logger.js index 1d3dc08..bf0e597 100644 --- a/logger.js +++ b/logger.js @@ -1,7 +1,22 @@ +// Logs path: +/* +base_url/platform_name/task_id/assignment_code/ + +{ + key_name: "test_name", + key_value: "test_value", + worker_id: 0, + logs: [ + ] +} + +*/ + var EDA_LOGGER = EDA_LOGGER || (function() { var _args = { key_name: "test_name", - key_value: "test_value" + key_value: "test_value", + task_id: 0 }; // private return { init: function(Args) { @@ -15,20 +30,23 @@ var EDA_LOGGER = EDA_LOGGER || (function() { console.log(firebase_script.src); console.log(document.getElementsByTagName('head')[0]); document.getElementsByTagName('head')[0].appendChild(firebase_script); + var logger = this; firebase_script.onload = function() { - _args["firebase_instance"] = new Firebase("https://crowdworker-logger.firebaseio.com/trials"); - this.log_event(_args["firebase_instance"]); + var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/-1"),document.location.pathname.length); + console.log(assignment_code); + _args["firebase_assignment"] = new Firebase("https://crowdworker-logger.firebaseio.com/"+document.location.hostname+"/"+_args["task_id"]+"/"+assignment_code); + _args["firebase_assignment"].update({ + key_name: _args.key_name, + key_value: _args.key_value, + }); + _args["firebase_logs"] = _args["firebase_assignment"].child('logs'); + logger.log_event(_args["firebase_logs"]); }; }, log_event: function(firebase_reference) { firebase_reference.push({ - pathname: document.location.pathname, - search: document.location.search, - hostname: document.location.hostname, - hash: document.location.hash, - key_name: _args.key_name, - key_value: _args.key_value + status: 1 }); } }; -}()); \ No newline at end of file +}()); From 8b7a0c73078cc176825fea8960139c1efa5db1fe Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 6 Oct 2015 14:21:55 +0200 Subject: [PATCH 11/62] no message --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index bf0e597..f74e10e 100644 --- a/logger.js +++ b/logger.js @@ -32,7 +32,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { document.getElementsByTagName('head')[0].appendChild(firebase_script); var logger = this; firebase_script.onload = function() { - var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/-1"),document.location.pathname.length); + var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"),document.location.pathname.length).replace(/-/g,''); console.log(assignment_code); _args["firebase_assignment"] = new Firebase("https://crowdworker-logger.firebaseio.com/"+document.location.hostname+"/"+_args["task_id"]+"/"+assignment_code); _args["firebase_assignment"].update({ From 410cec4a5497da24b4de3d593796007db9eed777 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 6 Oct 2015 14:25:47 +0200 Subject: [PATCH 12/62] no message --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index f74e10e..745130a 100644 --- a/logger.js +++ b/logger.js @@ -34,7 +34,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { firebase_script.onload = function() { var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"),document.location.pathname.length).replace(/-/g,''); console.log(assignment_code); - _args["firebase_assignment"] = new Firebase("https://crowdworker-logger.firebaseio.com/"+document.location.hostname+"/"+_args["task_id"]+"/"+assignment_code); + _args["firebase_assignment"] = new Firebase("https://crowdworker-logger.firebaseio.com/"+document.location.hostname+"/"+_args["task_id"]+assignment_code); _args["firebase_assignment"].update({ key_name: _args.key_name, key_value: _args.key_value, From 2beef500cd10c9331628504819383d9aa51fb195 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 6 Oct 2015 14:37:27 +0200 Subject: [PATCH 13/62] periods are removed from domain name --- logger.js | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/logger.js b/logger.js index 745130a..295d5a8 100644 --- a/logger.js +++ b/logger.js @@ -13,6 +13,8 @@ base_url/platform_name/task_id/assignment_code/ */ var EDA_LOGGER = EDA_LOGGER || (function() { + var debug = true; + var _args = { key_name: "test_name", key_value: "test_value", @@ -24,17 +26,26 @@ var EDA_LOGGER = EDA_LOGGER || (function() { // Include Firebase library this.init_firebase(); }, + console: function(message){if (debug) console.log("[EDA_LOGGER] LOG: "+message);}, init_firebase: function() { var firebase_script = document.createElement('script'); firebase_script.src = "https://cdn.firebase.com/js/client/2.2.9/firebase.js"; - console.log(firebase_script.src); - console.log(document.getElementsByTagName('head')[0]); + + this.console(firebase_script.src); + document.getElementsByTagName('head')[0].appendChild(firebase_script); var logger = this; firebase_script.onload = function() { - var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"),document.location.pathname.length).replace(/-/g,''); - console.log(assignment_code); - _args["firebase_assignment"] = new Firebase("https://crowdworker-logger.firebaseio.com/"+document.location.hostname+"/"+_args["task_id"]+assignment_code); + // get the assignment code from the url + var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"),document.location.pathname.length); + logger.console(assignment_code); + // get the platform code from the url + var platform_code = document.location.hostname.replace(/./g,'') + // form the firebase endpoint url + var firebase_endpoint_url = "https://crowdworker-logger.firebaseio.com/"+platform_code+"/"+_args["task_id"]+assignment_code; + logger.console(firebase_endpoint_url); + + _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); _args["firebase_assignment"].update({ key_name: _args.key_name, key_value: _args.key_value, From 1222c6069b611ed6e7e39c5ceae118c2c0890b61 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 6 Oct 2015 14:39:06 +0200 Subject: [PATCH 14/62] log function went up --- logger.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/logger.js b/logger.js index 295d5a8..4c60631 100644 --- a/logger.js +++ b/logger.js @@ -13,8 +13,8 @@ base_url/platform_name/task_id/assignment_code/ */ var EDA_LOGGER = EDA_LOGGER || (function() { - var debug = true; - + var debug = false; + var log = function(message){if (debug) console.log("[EDA_LOGGER] LOG: "+message);}; var _args = { key_name: "test_name", key_value: "test_value", @@ -26,24 +26,23 @@ var EDA_LOGGER = EDA_LOGGER || (function() { // Include Firebase library this.init_firebase(); }, - console: function(message){if (debug) console.log("[EDA_LOGGER] LOG: "+message);}, init_firebase: function() { var firebase_script = document.createElement('script'); firebase_script.src = "https://cdn.firebase.com/js/client/2.2.9/firebase.js"; - this.console(firebase_script.src); - + log(firebase_script.src); + document.getElementsByTagName('head')[0].appendChild(firebase_script); var logger = this; firebase_script.onload = function() { // get the assignment code from the url var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"),document.location.pathname.length); - logger.console(assignment_code); + log(assignment_code); // get the platform code from the url var platform_code = document.location.hostname.replace(/./g,'') // form the firebase endpoint url var firebase_endpoint_url = "https://crowdworker-logger.firebaseio.com/"+platform_code+"/"+_args["task_id"]+assignment_code; - logger.console(firebase_endpoint_url); + log(firebase_endpoint_url); _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); _args["firebase_assignment"].update({ From a8259fee90bf3abf1875253d92ae4c83e0240271 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 6 Oct 2015 14:40:32 +0200 Subject: [PATCH 15/62] "/./" -> "." --- logger.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logger.js b/logger.js index 4c60631..47d84d5 100644 --- a/logger.js +++ b/logger.js @@ -13,7 +13,7 @@ base_url/platform_name/task_id/assignment_code/ */ var EDA_LOGGER = EDA_LOGGER || (function() { - var debug = false; + var debug = true; var log = function(message){if (debug) console.log("[EDA_LOGGER] LOG: "+message);}; var _args = { key_name: "test_name", @@ -39,7 +39,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"),document.location.pathname.length); log(assignment_code); // get the platform code from the url - var platform_code = document.location.hostname.replace(/./g,'') + var platform_code = document.location.hostname.replace(".",'') // form the firebase endpoint url var firebase_endpoint_url = "https://crowdworker-logger.firebaseio.com/"+platform_code+"/"+_args["task_id"]+assignment_code; log(firebase_endpoint_url); From 2d6b701bb012a881cc9af260e344508d3b99ac2a Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 6 Oct 2015 14:41:55 +0200 Subject: [PATCH 16/62] "." ->/\./g --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index 47d84d5..6902029 100644 --- a/logger.js +++ b/logger.js @@ -39,7 +39,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"),document.location.pathname.length); log(assignment_code); // get the platform code from the url - var platform_code = document.location.hostname.replace(".",'') + var platform_code = document.location.hostname.replace(/\./g,''); // form the firebase endpoint url var firebase_endpoint_url = "https://crowdworker-logger.firebaseio.com/"+platform_code+"/"+_args["task_id"]+assignment_code; log(firebase_endpoint_url); From 8499a9b5997f3dbb21b34bfe32a838556f6902af Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 6 Oct 2015 15:58:20 +0200 Subject: [PATCH 17/62] listener first commit --- listener.js | 9 ++++++ logger.js | 81 +++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 listener.js diff --git a/listener.js b/listener.js new file mode 100644 index 0000000..53faaf6 --- /dev/null +++ b/listener.js @@ -0,0 +1,9 @@ +var Firebase = require('firebase'); +var Ref = new Firebase('https://crowdworker-logger.firebaseio.com/'+"localhost"+'/'); +Ref.on('child_changed', function(task_snapshot, prevchildname) { + var task_reference = Ref.child(task_snapshot.key()+'/logs'); + task_reference.on('child_changed', function(assignment_snapshot, prevchildname){ + console.log(assignment_snapshot.key()); + }); + +}); diff --git a/logger.js b/logger.js index 6902029..d054ead 100644 --- a/logger.js +++ b/logger.js @@ -1,7 +1,6 @@ // Logs path: /* base_url/platform_name/task_id/assignment_code/ - { key_name: "test_name", key_value: "test_value", @@ -9,12 +8,24 @@ base_url/platform_name/task_id/assignment_code/ logs: [ ] } - */ var EDA_LOGGER = EDA_LOGGER || (function() { - var debug = true; - var log = function(message){if (debug) console.log("[EDA_LOGGER] LOG: "+message);}; + var settings = { + debug: true, + firebase: { + bucket: "crowdworker-logger" + } + } + // A log function which can be easily turned of using debug variable + var log = function(message) { + if (settings.debug) console.log("[EDA_LOGGER] LOG: " + message); + }; + + function isPageHidden() { + return document.hidden || document.msHidden || document.webkitHidden || document.mozHidden; + } + var _args = { key_name: "test_name", key_value: "test_value", @@ -24,9 +35,12 @@ var EDA_LOGGER = EDA_LOGGER || (function() { init: function(Args) { _args = Args; // Include Firebase library - this.init_firebase(); + var logger = this; + logger.init_firebase(function() { + logger.init_events_capturing(); + }); }, - init_firebase: function() { + init_firebase: function(callback) { var firebase_script = document.createElement('script'); firebase_script.src = "https://cdn.firebase.com/js/client/2.2.9/firebase.js"; @@ -36,12 +50,12 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var logger = this; firebase_script.onload = function() { // get the assignment code from the url - var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"),document.location.pathname.length); + var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"), document.location.pathname.length); log(assignment_code); // get the platform code from the url - var platform_code = document.location.hostname.replace(/\./g,''); + var platform_code = document.location.hostname.replace(/\./g, ''); // form the firebase endpoint url - var firebase_endpoint_url = "https://crowdworker-logger.firebaseio.com/"+platform_code+"/"+_args["task_id"]+assignment_code; + var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + assignment_code; log(firebase_endpoint_url); _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); @@ -50,12 +64,55 @@ var EDA_LOGGER = EDA_LOGGER || (function() { key_value: _args.key_value, }); _args["firebase_logs"] = _args["firebase_assignment"].child('logs'); - logger.log_event(_args["firebase_logs"]); + callback(); + }; + }, + init_events_capturing: function() { + var logger = this; + // Log the task was opened by the worker + logger.log_event(_args["firebase_logs"], "opened"); + // Log the task page was closed by the worker + window.onbeforeunload = function() { + logger.log_event(_args["firebase_logs"], "closed"); }; + logger.init_visibility_changes(); + }, + init_visibility_changes: function() { + var hidden, visibilityChange; + if (typeof document.hidden !== "undefined") { // Opera 12.10 and Firefox 18 and later support + hidden = "hidden"; + visibilityChange = "visibilitychange"; + } else if (typeof document.mozHidden !== "undefined") { + hidden = "mozHidden"; + visibilityChange = "mozvisibilitychange"; + } else if (typeof document.msHidden !== "undefined") { + hidden = "msHidden"; + visibilityChange = "msvisibilitychange"; + } else if (typeof document.webkitHidden !== "undefined") { + hidden = "webkitHidden"; + visibilityChange = "webkitvisibilitychange"; + } + var logger = this; + + function handleVisibilityChange() { + if (document[hidden]) { + logger.log_event(_args["firebase_logs"], "hidden"); + } else { + logger.log_event(_args["firebase_logs"], "active"); + } + } + // Warn if the browser doesn't support addEventListener or the Page Visibility API + if (typeof document.addEventListener === "undefined" || + typeof document[hidden] === "undefined") { + alert("This demo requires a browser, such as Google Chrome or Firefox, that supports the Page Visibility API."); + } else { + // Handle page visibility change + document.addEventListener(visibilityChange, handleVisibilityChange, false); + } }, - log_event: function(firebase_reference) { + log_event: function(firebase_reference, status) { firebase_reference.push({ - status: 1 + status: status }); } }; From 35e0e3b8de7952239622843122b2d669b7181384 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Thu, 22 Oct 2015 17:20:24 +0200 Subject: [PATCH 18/62] listener --- listener.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/listener.js b/listener.js index 53faaf6..a5757ba 100644 --- a/listener.js +++ b/listener.js @@ -1,9 +1,9 @@ var Firebase = require('firebase'); var Ref = new Firebase('https://crowdworker-logger.firebaseio.com/'+"localhost"+'/'); Ref.on('child_changed', function(task_snapshot, prevchildname) { - var task_reference = Ref.child(task_snapshot.key()+'/logs'); - task_reference.on('child_changed', function(assignment_snapshot, prevchildname){ + console.log(task_snapshot.key()); + var task_reference = Ref.child(task_snapshot.key()); + task_reference.on('child_added', function(assignment_snapshot, prevchildname){ console.log(assignment_snapshot.key()); }); - }); From 37c674e0c020d28c39fcd3fd5717ccb30cfe5db8 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Thu, 22 Oct 2015 17:53:26 +0200 Subject: [PATCH 19/62] runtime unit_id updates --- listener.js | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/listener.js b/listener.js index a5757ba..6802f95 100644 --- a/listener.js +++ b/listener.js @@ -1,9 +1,45 @@ var Firebase = require('firebase'); -var Ref = new Firebase('https://crowdworker-logger.firebaseio.com/'+"localhost"+'/'); +var requestify = require('requestify'); +var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/'; +var Ref = new Firebase(firebase_base_url); +var CROWDFLOWER_API_KEY = process.env.CROWDFLOWER_API_KEY; + Ref.on('child_changed', function(task_snapshot, prevchildname) { - console.log(task_snapshot.key()); - var task_reference = Ref.child(task_snapshot.key()); - task_reference.on('child_added', function(assignment_snapshot, prevchildname){ - console.log(assignment_snapshot.key()); - }); + var task_id = task_snapshot.key(); + console.log(task_id); + var task_reference = Ref.child(task_id); + + task_reference.on('child_added', function(assignment_snapshot, prevchildname) { + var assignment_details = assignment_snapshot.val(); + var unit_id = assignment_details['unit_id']; + if (unit_id == undefined) { + var key_name = assignment_details['key_name']; + var unit_key = { + 'name': assignment_details['key_name'], + 'value': assignment_details['key_value'] + }; + console.log(unit_key); + var units_url = "http://api.crowdflower.com/v1/jobs/" + task_id + "/units/?key=" + CROWDFLOWER_API_KEY; + console.log(units_url); + + requestify.get(units_url, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var units = response.getBody(); + //console.log(units); + for (var key in units) { + if (units[key][unit_key["name"]] == unit_key["value"]) { + task_reference.child(assignment_snapshot.key()).update({ + unit_id: key + }); + } + // do something with key + } + }); + + } + }); }); From 9bfc759293bcfd9f5c0a7efff3daff39c289beec Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Thu, 5 Nov 2015 15:01:23 +0100 Subject: [PATCH 20/62] Heroku deploy --- README.md | 2 ++ app.json | 13 +++++++++++++ logger.js | 2 +- package.json | 20 ++++++++++++++++++++ procfile | 1 + listener.js => unit_references_maker.js | 0 6 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 app.json create mode 100644 package.json create mode 100644 procfile rename listener.js => unit_references_maker.js (100%) diff --git a/README.md b/README.md index d86fd53..eeea184 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ # worker-activity-logger + +[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) \ No newline at end of file diff --git a/app.json b/app.json new file mode 100644 index 0000000..08b3403 --- /dev/null +++ b/app.json @@ -0,0 +1,13 @@ +{ + "name": "Relauncher-Logger-ReferenceMaker", + "description": "Server side application listening to a given Firebase bucket and fixing unit references.", + "repository": "https://github.com/ReLauncher/worker-activity-logger", + "logo": "https://avatars3.githubusercontent.com/u/14817691?v=3&s=200", + "keywords": ["relauncher", "crowdflower","crowdsourcing","microtasks"], + "env": { + "HEROKU_URL": { + "description": "The URL of this Heroku app.", + "value": "http://YOURAPPNAME.herokuapp.com" + } + } +} \ No newline at end of file diff --git a/logger.js b/logger.js index d054ead..e645fa1 100644 --- a/logger.js +++ b/logger.js @@ -104,7 +104,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { // Warn if the browser doesn't support addEventListener or the Page Visibility API if (typeof document.addEventListener === "undefined" || typeof document[hidden] === "undefined") { - alert("This demo requires a browser, such as Google Chrome or Firefox, that supports the Page Visibility API."); + //alert("This demo requires a browser, such as Google Chrome or Firefox, that supports the Page Visibility API."); } else { // Handle page visibility change document.addEventListener(visibilityChange, handleVisibilityChange, false); diff --git a/package.json b/package.json new file mode 100644 index 0000000..8579f97 --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "name": "Relauncher-Logger-ReferenceMaker", + "version": "0.0.1", + "description": "", + "main": "unit_references_maker.js", + "engines": { + "node": "0.10.29" + }, + "scripts": { + "start": "node unit_references_maker.js" + }, + "repository": { + "type": "git", + "url": "https://github.com/ReLauncher/worker-activity-logger.git" + }, + "dependencies": { + "firebase":"2.3.1", + "requestify":"0.1.17" + } +} \ No newline at end of file diff --git a/procfile b/procfile new file mode 100644 index 0000000..104c7e4 --- /dev/null +++ b/procfile @@ -0,0 +1 @@ +web: node unit_references_maker.js \ No newline at end of file diff --git a/listener.js b/unit_references_maker.js similarity index 100% rename from listener.js rename to unit_references_maker.js From ac793ee8407ca76f348448a75ec79f4357979b8d Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Thu, 5 Nov 2015 15:03:57 +0100 Subject: [PATCH 21/62] simplified app.son --- app.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app.json b/app.json index 08b3403..3138987 100644 --- a/app.json +++ b/app.json @@ -1,13 +1,13 @@ { - "name": "Relauncher-Logger-ReferenceMaker", - "description": "Server side application listening to a given Firebase bucket and fixing unit references.", - "repository": "https://github.com/ReLauncher/worker-activity-logger", - "logo": "https://avatars3.githubusercontent.com/u/14817691?v=3&s=200", - "keywords": ["relauncher", "crowdflower","crowdsourcing","microtasks"], - "env": { - "HEROKU_URL": { - "description": "The URL of this Heroku app.", - "value": "http://YOURAPPNAME.herokuapp.com" + "name": "Relauncher Logger ReferenceMaker", + "description": "Aapplication listening to a given Firebase bucket and fixing unit references.", + "repository": "https://github.com/ReLauncher/worker-activity-logger", + "logo": "https://avatars3.githubusercontent.com/u/14817691?v=3&s=200", + "keywords": ["relauncher", "crowdflower", "crowdsourcing", "microtasks"], + "env": { + "HEROKU_URL": { + "description": "The URL of this Heroku app.", + "value": "http://YOURAPPNAME.herokuapp.com" + } } - } -} \ No newline at end of file +} From 2ea59e6bd7461ab8d2319641f48261bb301c616b Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Thu, 5 Nov 2015 15:06:20 +0100 Subject: [PATCH 22/62] Logger is omitted in title --- app.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.json b/app.json index 3138987..119e1d7 100644 --- a/app.json +++ b/app.json @@ -1,5 +1,5 @@ { - "name": "Relauncher Logger ReferenceMaker", + "name": "Relauncher ReferenceMaker", "description": "Aapplication listening to a given Firebase bucket and fixing unit references.", "repository": "https://github.com/ReLauncher/worker-activity-logger", "logo": "https://avatars3.githubusercontent.com/u/14817691?v=3&s=200", From d7ad852c685401229319a228fdeba3b576ddaf19 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 23 Nov 2015 15:35:34 +0100 Subject: [PATCH 23/62] logger dt: Date.now() is added --- app.json | 4 ++++ logger.js | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app.json b/app.json index 119e1d7..dadc8c5 100644 --- a/app.json +++ b/app.json @@ -8,6 +8,10 @@ "HEROKU_URL": { "description": "The URL of this Heroku app.", "value": "http://YOURAPPNAME.herokuapp.com" + }, + "CROWDFLOWER_API_KEY":{ + "description" : "your API key from https://make.crowdflower.com/account/user", + "value":"" } } } diff --git a/logger.js b/logger.js index e645fa1..b0847ca 100644 --- a/logger.js +++ b/logger.js @@ -112,7 +112,8 @@ var EDA_LOGGER = EDA_LOGGER || (function() { }, log_event: function(firebase_reference, status) { firebase_reference.push({ - status: status + status: status, + dt: Date.now() }); } }; From 7ae28701dc8b081380c26a6ee376dd65f5934ee2 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 10:55:55 +0100 Subject: [PATCH 24/62] Log dates changed to Firebase.ServerValue.TIMESTAMP --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index b0847ca..453d301 100644 --- a/logger.js +++ b/logger.js @@ -113,7 +113,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { log_event: function(firebase_reference, status) { firebase_reference.push({ status: status, - dt: Date.now() + dt: Firebase.ServerValue.TIMESTAMP }); } }; From 6d2659befebb3bfc911cf754b02235686558190b Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 11:26:13 +0100 Subject: [PATCH 25/62] log worker_id --- logger.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/logger.js b/logger.js index 453d301..bb369d4 100644 --- a/logger.js +++ b/logger.js @@ -62,6 +62,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { _args["firebase_assignment"].update({ key_name: _args.key_name, key_value: _args.key_value, + worker_id: logger.utility.getCookie("ajs_user_id") }); _args["firebase_logs"] = _args["firebase_assignment"].child('logs'); callback(); @@ -115,6 +116,13 @@ var EDA_LOGGER = EDA_LOGGER || (function() { status: status, dt: Firebase.ServerValue.TIMESTAMP }); + }, + utility: { + getCookie: function(name) { + var value = "; " + document.cookie; + var parts = value.split("; " + name + "="); + if (parts.length == 2) return parts.pop().split(";").shift(); + } } }; }()); From 20862a554503dd93c0d2127cdea41a77e2200d71 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 11:40:02 +0100 Subject: [PATCH 26/62] Firebase logs for a given job to csv converter --- firebase_to_csv.js | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 firebase_to_csv.js diff --git a/firebase_to_csv.js b/firebase_to_csv.js new file mode 100644 index 0000000..a50d272 --- /dev/null +++ b/firebase_to_csv.js @@ -0,0 +1,84 @@ +var requestify = require('requestify'); +var fs = require('fs'); + + +var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/'; +var task_id = 836250; +var firebase_target_url = firebase_base_url + task_id + ".json"; +var folder = "logs/"; +var filename = folder + task_id + "_logs.csv"; + +fs.createWriteStream(filename); +fs.truncate(filename, 0, function() { + console.log('file ' + filename + ' was cleaned up.') +}); +fs.appendFile(filename, 'unit_id, assignment_id, worker_id, dt_start, dt_end, status\n', function(err) {}); + +var Logs = []; +// ------------------------------------------------------- +// Convert log object into a string +// ------------------------------------------------------- +function stringify(log_array) { + for (var i = 0; i < log_array.length; i++) { + log_array[i]['string'] = ""; + log_array[i]['string'] += log_array[i].unit_id + ", "; + log_array[i]['string'] += log_array[i].assignment_id + ", "; + log_array[i]['string'] += log_array[i].worker_id + ", "; + log_array[i]['string'] += log_array[i].dt_start + ", "; + log_array[i]['string'] += log_array[i].dt_end + ", "; + log_array[i]['string'] += log_array[i].status; + } + return log_array; +} +// ------------------------------------------------------- +// Collect data for a given Job and make a plain Array of log objects +// ------------------------------------------------------- +requestify.get(firebase_target_url, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var assignments = response.getBody(); + for (var a_id in assignments) { + var a = assignments[a_id]; + if (a_id != "editor_preview" && a.unit_id != undefined) { + for (var l_id in a.logs) { + var l = a.logs[l_id]; + var log_record = { + 'unit_id': a.unit_id, + 'assignment_id': a_id, + 'worker_id': a.worker_id, + 'dt_start': l.dt, + 'status': l.status + } + Logs.push(log_record); + } + } else { + console.log("PROBLEMATIC ASSIGNMENT", a); + } + } + // ------------------------------------------------------- + // Add end events to log objects + // ------------------------------------------------------- + for (var i = 0; i < Logs.length; i++) { + if (Logs[i].status != 'closed') { + if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id) { + Logs[i]['dt_end'] = Logs[i + 1]['dt_start']; + } + } + if (Logs[i]['dt_end'] == undefined) { + Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000; + } + + } + // ------------------------------------------------------- + // Convert each object in array into string and save it into the log file + // ------------------------------------------------------- + Logs = stringify(Logs); + for (var i = 0; i < Logs.length; i++) { + console.log(Logs[i]['string']); + fs.appendFile(filename, Logs[i]['string'] + '\n', function(err) {}); + } + console.log(firebase_target_url); + }); From 67f4d15a5738e96159a6d88eac7fd2209878e331 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 11:43:06 +0100 Subject: [PATCH 27/62] index.html is renamed into example.html --- index.html => example.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename index.html => example.html (100%) diff --git a/index.html b/example.html similarity index 100% rename from index.html rename to example.html From a626ad7064b011f1fa3aa6fa71c976e19444739d Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 13:28:27 +0100 Subject: [PATCH 28/62] Readme. Set up logs collection. --- README.md | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eeea184..ba2f666 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,30 @@ # worker-activity-logger -[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) \ No newline at end of file +#### To collect worker browser logs for your tasks: + +1. fork this repository, +2. create a bucket in [Firebase](https://www.firebase.com/) (it is free), +3. insert your firebase bucket name in [logger.js](https://github.com/ReLauncher/worker-activity-logger/blob/gh-pages/logger.js#L17): +```javascript +var settings = { + debug: true, + firebase: { + bucket: "YOUR_FIREBASE_BUCKET_NAME" + } + } +``` +4. [create a job](https://success.crowdflower.com/hc/en-us/articles/204056975-Getting-Started-on-CrowdFlower-An-Overview-to-Building-a-Job) in CrowdFlower, upload your data. Make sure you have a column, which can be used as an identificator of a given data row (unit). If you collect tags for images, it can be *image_url* +5. add the following in [CML](https://success.crowdflower.com/hc/en-us/articles/202817989-CML-CrowdFlower-Markup-Language-Overview) of your CrowdFlower job: +```html + + +``` +When you launch the job you should see logs appearing you in your firebase bucket. If you do not - check all the steps carefully. If it still does not work, create an *issue* in this repository. + +[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) From 1086af68a3ade65d694bd98dfdcad1e694207c6c Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 13:31:26 +0100 Subject: [PATCH 29/62] Readme. Formatting. --- README.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ba2f666..afbbd61 100644 --- a/README.md +++ b/README.md @@ -5,26 +5,29 @@ 1. fork this repository, 2. create a bucket in [Firebase](https://www.firebase.com/) (it is free), 3. insert your firebase bucket name in [logger.js](https://github.com/ReLauncher/worker-activity-logger/blob/gh-pages/logger.js#L17): -```javascript -var settings = { + + ```javascript + var settings = { debug: true, firebase: { bucket: "YOUR_FIREBASE_BUCKET_NAME" } } -``` + ``` 4. [create a job](https://success.crowdflower.com/hc/en-us/articles/204056975-Getting-Started-on-CrowdFlower-An-Overview-to-Building-a-Job) in CrowdFlower, upload your data. Make sure you have a column, which can be used as an identificator of a given data row (unit). If you collect tags for images, it can be *image_url* 5. add the following in [CML](https://success.crowdflower.com/hc/en-us/articles/202817989-CML-CrowdFlower-Markup-Language-Overview) of your CrowdFlower job: -```html - - -``` + + ```html + + + ``` + When you launch the job you should see logs appearing you in your firebase bucket. If you do not - check all the steps carefully. If it still does not work, create an *issue* in this repository. [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) From 8e83e63b29053f15f8d2d1ad5feddb03d6821065 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 13:32:28 +0100 Subject: [PATCH 30/62] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index afbbd61..0973f30 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # worker-activity-logger -#### To collect worker browser logs for your tasks: +#### Setup worker browser logs collection for your tasks in 5 steps: 1. fork this repository, 2. create a bucket in [Firebase](https://www.firebase.com/) (it is free), From 37106d3450980341dd3a09d36eed49b96cb6575e Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 13:33:49 +0100 Subject: [PATCH 31/62] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0973f30..2ead653 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# worker-activity-logger +# ReLauncher | Worker Activity Logger #### Setup worker browser logs collection for your tasks in 5 steps: From d70935ff35151c5425d87d0ffaf782af65439932 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 14:54:39 +0100 Subject: [PATCH 32/62] search for worker_id --- logger.js | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/logger.js b/logger.js index bb369d4..9734dbf 100644 --- a/logger.js +++ b/logger.js @@ -62,7 +62,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { _args["firebase_assignment"].update({ key_name: _args.key_name, key_value: _args.key_value, - worker_id: logger.utility.getCookie("ajs_user_id") + worker_id: logger.utility.getWorkerId() }); _args["firebase_logs"] = _args["firebase_assignment"].child('logs'); callback(); @@ -118,10 +118,24 @@ var EDA_LOGGER = EDA_LOGGER || (function() { }); }, utility: { - getCookie: function(name) { - var value = "; " + document.cookie; - var parts = value.split("; " + name + "="); - if (parts.length == 2) return parts.pop().split(";").shift(); + getCookieByMatch: function(regex) { + var cs = document.cookie.split(/;\s*/), + ret = [], + i; + for (i = 0; i < cs.length; i++) { + if (cs[i].match(regex)) { + ret.push(cs[i]); + } + } + return ret; + }, + getWorkerId: function(){ + var logger = this; + var patterns=logger.utility.getCookieByMatch(/user_id=/); + if (patterns.length>0){ + return patterns[0].split("=")[1]; + } + else return null; } } }; From 71faae69a07074ef24d5c32b0f9aa92a1574d32b Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 14:59:16 +0100 Subject: [PATCH 33/62] fix logger --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index 9734dbf..6f2151c 100644 --- a/logger.js +++ b/logger.js @@ -131,7 +131,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { }, getWorkerId: function(){ var logger = this; - var patterns=logger.utility.getCookieByMatch(/user_id=/); + var patterns=logger.getCookieByMatch(/user_id=/); if (patterns.length>0){ return patterns[0].split("=")[1]; } From f7c8aadba4f056d24b3efae3844b8f829fa9187d Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 15:06:30 +0100 Subject: [PATCH 34/62] worker_id or user_id --- logger.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/logger.js b/logger.js index 6f2151c..d31c5ba 100644 --- a/logger.js +++ b/logger.js @@ -129,13 +129,18 @@ var EDA_LOGGER = EDA_LOGGER || (function() { } return ret; }, - getWorkerId: function(){ + getWorkerId: function() { var logger = this; - var patterns=logger.getCookieByMatch(/user_id=/); - if (patterns.length>0){ + var patterns = logger.getCookieByMatch(/user_id=/); + if (patterns.length > 0) { return patterns[0].split("=")[1]; + }else{ + patterns = logger.getCookieByMatch(/worker_id=/); + if (patterns.length > 0) { + return patterns[0].split("=")[1]; + } } - else return null; + return null; } } }; From 2e3371ed6fc1d817680ec3247910ce74bbc0cc29 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Mon, 30 Nov 2015 16:16:34 +0100 Subject: [PATCH 35/62] removed worker_id logging --- logger.js | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/logger.js b/logger.js index d31c5ba..11a2ef4 100644 --- a/logger.js +++ b/logger.js @@ -61,8 +61,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); _args["firebase_assignment"].update({ key_name: _args.key_name, - key_value: _args.key_value, - worker_id: logger.utility.getWorkerId() + key_value: _args.key_value }); _args["firebase_logs"] = _args["firebase_assignment"].child('logs'); callback(); @@ -116,32 +115,6 @@ var EDA_LOGGER = EDA_LOGGER || (function() { status: status, dt: Firebase.ServerValue.TIMESTAMP }); - }, - utility: { - getCookieByMatch: function(regex) { - var cs = document.cookie.split(/;\s*/), - ret = [], - i; - for (i = 0; i < cs.length; i++) { - if (cs[i].match(regex)) { - ret.push(cs[i]); - } - } - return ret; - }, - getWorkerId: function() { - var logger = this; - var patterns = logger.getCookieByMatch(/user_id=/); - if (patterns.length > 0) { - return patterns[0].split("=")[1]; - }else{ - patterns = logger.getCookieByMatch(/worker_id=/); - if (patterns.length > 0) { - return patterns[0].split("=")[1]; - } - } - return null; - } } }; }()); From e124f8dff3686602c938209ef0bd5574c6f79525 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 1 Dec 2015 14:34:16 +0100 Subject: [PATCH 36/62] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ead653..8a50ed6 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,6 @@ ``` -When you launch the job you should see logs appearing you in your firebase bucket. If you do not - check all the steps carefully. If it still does not work, create an *issue* in this repository. +When you launch the job you should see logs appearing in your firebase bucket. If you do not - check all the steps carefully. If it still does not work, create an *issue* in this repository. [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) From 5af24893009886617532b3f6cb585928e29863b1 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 12:29:56 +0100 Subject: [PATCH 37/62] unit_id set right from logger --- logger.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/logger.js b/logger.js index 11a2ef4..b17be07 100644 --- a/logger.js +++ b/logger.js @@ -61,7 +61,8 @@ var EDA_LOGGER = EDA_LOGGER || (function() { _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); _args["firebase_assignment"].update({ key_name: _args.key_name, - key_value: _args.key_value + key_value: _args.key_value, + unit_id:_args.key_value }); _args["firebase_logs"] = _args["firebase_assignment"].child('logs'); callback(); From 664070f031842969ca6f0690f7c371442a811be4 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 13:45:35 +0100 Subject: [PATCH 38/62] tab session to logger --- logger.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/logger.js b/logger.js index b17be07..de68cec 100644 --- a/logger.js +++ b/logger.js @@ -12,6 +12,7 @@ base_url/platform_name/task_id/assignment_code/ var EDA_LOGGER = EDA_LOGGER || (function() { var settings = { + session: Math.floor(Math.random()*1000000) + 1, debug: true, firebase: { bucket: "crowdworker-logger" @@ -113,6 +114,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { }, log_event: function(firebase_reference, status) { firebase_reference.push({ + session: settings.session, status: status, dt: Firebase.ServerValue.TIMESTAMP }); From 438646fc7d964690fc08f03f8263a5423ee61332 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 16:02:24 +0100 Subject: [PATCH 39/62] automatic capture of worker_id and task_id --- logger.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/logger.js b/logger.js index de68cec..c94aabf 100644 --- a/logger.js +++ b/logger.js @@ -30,7 +30,8 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var _args = { key_name: "test_name", key_value: "test_value", - task_id: 0 + task_id: parseInt(document.getElementById("assignment-job-id").innerHTML), + worker_id : parseInt(document.getElementById("assignment-worker-id").innerHTML) }; // private return { init: function(Args) { From 8ff77198beaf526779f73bf80b6302e1dad6aaba Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 16:10:42 +0100 Subject: [PATCH 40/62] task_id --- logger.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/logger.js b/logger.js index c94aabf..1d47e86 100644 --- a/logger.js +++ b/logger.js @@ -12,7 +12,7 @@ base_url/platform_name/task_id/assignment_code/ var EDA_LOGGER = EDA_LOGGER || (function() { var settings = { - session: Math.floor(Math.random()*1000000) + 1, + session: Math.floor(Math.random() * 1000000) + 1, debug: true, firebase: { bucket: "crowdworker-logger" @@ -29,14 +29,14 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var _args = { key_name: "test_name", - key_value: "test_value", - task_id: parseInt(document.getElementById("assignment-job-id").innerHTML), - worker_id : parseInt(document.getElementById("assignment-worker-id").innerHTML) + key_value: "test_value" }; // private return { init: function(Args) { _args = Args; - // Include Firebase library + _args['task_id']: parseInt(document.getElementById("assignment-job-id").innerHTML); + _args['worker_id']: parseInt(document.getElementById("assignment-worker-id").innerHTML); + // Include Firebase library var logger = this; logger.init_firebase(function() { logger.init_events_capturing(); @@ -64,7 +64,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { _args["firebase_assignment"].update({ key_name: _args.key_name, key_value: _args.key_value, - unit_id:_args.key_value + unit_id: _args.key_value }); _args["firebase_logs"] = _args["firebase_assignment"].child('logs'); callback(); From fad590253f082cdf3944f07a8cf29dde1abf34c7 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 16:12:32 +0100 Subject: [PATCH 41/62] fixed error with ":" --- logger.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/logger.js b/logger.js index 1d47e86..dddaca9 100644 --- a/logger.js +++ b/logger.js @@ -34,9 +34,9 @@ var EDA_LOGGER = EDA_LOGGER || (function() { return { init: function(Args) { _args = Args; - _args['task_id']: parseInt(document.getElementById("assignment-job-id").innerHTML); - _args['worker_id']: parseInt(document.getElementById("assignment-worker-id").innerHTML); - // Include Firebase library + _args['task_id'] = parseInt(document.getElementById("assignment-job-id").innerHTML); + _args['worker_id'] = parseInt(document.getElementById("assignment-worker-id").innerHTML); + // Include Firebase library var logger = this; logger.init_firebase(function() { logger.init_events_capturing(); From 03b76fa2d05870f44492891abfc3a60db53912ec Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 16:15:56 +0100 Subject: [PATCH 42/62] split sessions in logs --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index dddaca9..a223398 100644 --- a/logger.js +++ b/logger.js @@ -66,7 +66,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { key_value: _args.key_value, unit_id: _args.key_value }); - _args["firebase_logs"] = _args["firebase_assignment"].child('logs'); + _args["firebase_logs"] = _args["firebase_assignment"].child(settings.session+'/logs'); callback(); }; }, From d441901a95d1d922b7c70d1f082ab322cdb8fdd5 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 16:23:57 +0100 Subject: [PATCH 43/62] logs/sessions --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index a223398..61190ce 100644 --- a/logger.js +++ b/logger.js @@ -66,7 +66,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { key_value: _args.key_value, unit_id: _args.key_value }); - _args["firebase_logs"] = _args["firebase_assignment"].child(settings.session+'/logs'); + _args["firebase_logs"] = _args["firebase_assignment"].child('logs/'+settings.session); callback(); }; }, From 348ebbe7fa526201bc6a0efe9304c3f7cfd85063 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 17:54:30 +0100 Subject: [PATCH 44/62] job_id/unit_id/assignment_code/sessions/logs --- logger.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/logger.js b/logger.js index 61190ce..7140909 100644 --- a/logger.js +++ b/logger.js @@ -57,9 +57,8 @@ var EDA_LOGGER = EDA_LOGGER || (function() { // get the platform code from the url var platform_code = document.location.hostname.replace(/\./g, ''); // form the firebase endpoint url - var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + assignment_code; + var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + "/"+_args['key_value'] + assignment_code; log(firebase_endpoint_url); - _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); _args["firebase_assignment"].update({ key_name: _args.key_name, From 89af029c0ff1d4b3ded13842bba08e124d8602da Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 17:57:57 +0100 Subject: [PATCH 45/62] no message --- logger.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logger.js b/logger.js index 7140909..f13957d 100644 --- a/logger.js +++ b/logger.js @@ -57,7 +57,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { // get the platform code from the url var platform_code = document.location.hostname.replace(/\./g, ''); // form the firebase endpoint url - var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + "/"+_args['key_value'] + assignment_code; + var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + "/units/"+_args['key_value']+"/assignments" + assignment_code; log(firebase_endpoint_url); _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); _args["firebase_assignment"].update({ @@ -65,7 +65,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { key_value: _args.key_value, unit_id: _args.key_value }); - _args["firebase_logs"] = _args["firebase_assignment"].child('logs/'+settings.session); + _args["firebase_logs"] = _args["firebase_assignment"].child('sessions/'+settings.session); callback(); }; }, From b73d752b60a89acfe823b819c76fde01392806e7 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 18:00:37 +0100 Subject: [PATCH 46/62] logs simplification --- logger.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/logger.js b/logger.js index f13957d..1cdba08 100644 --- a/logger.js +++ b/logger.js @@ -60,11 +60,11 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + "/units/"+_args['key_value']+"/assignments" + assignment_code; log(firebase_endpoint_url); _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); - _args["firebase_assignment"].update({ + /*_args["firebase_assignment"].update({ key_name: _args.key_name, key_value: _args.key_value, unit_id: _args.key_value - }); + });*/ _args["firebase_logs"] = _args["firebase_assignment"].child('sessions/'+settings.session); callback(); }; @@ -114,7 +114,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { }, log_event: function(firebase_reference, status) { firebase_reference.push({ - session: settings.session, + //session: settings.session, status: status, dt: Firebase.ServerValue.TIMESTAMP }); From f257c8989ac7d47f891c9e0ee78e5254a80e57f6 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 21:35:41 +0100 Subject: [PATCH 47/62] mouse keyboard scroll --- logger.js | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/logger.js b/logger.js index 1cdba08..a47193e 100644 --- a/logger.js +++ b/logger.js @@ -1,18 +1,7 @@ -// Logs path: -/* -base_url/platform_name/task_id/assignment_code/ -{ - key_name: "test_name", - key_value: "test_value", - worker_id: 0, - logs: [ - ] -} -*/ - var EDA_LOGGER = EDA_LOGGER || (function() { var settings = { session: Math.floor(Math.random() * 1000000) + 1, + activity_interval: 5000, debug: true, firebase: { bucket: "crowdworker-logger" @@ -57,7 +46,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { // get the platform code from the url var platform_code = document.location.hostname.replace(/\./g, ''); // form the firebase endpoint url - var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + "/units/"+_args['key_value']+"/assignments" + assignment_code; + var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + "/units/" + _args['key_value'] + "/assignments" + assignment_code; log(firebase_endpoint_url); _args["firebase_assignment"] = new Firebase(firebase_endpoint_url); /*_args["firebase_assignment"].update({ @@ -65,10 +54,40 @@ var EDA_LOGGER = EDA_LOGGER || (function() { key_value: _args.key_value, unit_id: _args.key_value });*/ - _args["firebase_logs"] = _args["firebase_assignment"].child('sessions/'+settings.session); + _args["firebase_logs"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/tab_visibilty"); + _args["firebase_activity"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/page_activity"); + callback(); }; }, + init_activity_capturing: function() { + var logger = this; + var activity_statuses = { + "keyboard": 0, + "mouse": 0, + "scroll": 0, + }; + setInterval(function() { + logger.log_event(_args["firebase_activity"], (function(a) { + activity_statuses = { + "keyboard": 0, + "mouse": 0, + "scroll": 0, + }; + return a; + }(activity_statuses))); + }, settings.activity_interval); + + document.onkeydown = function(evt) { + activity_statuses["keyboard"] = 1; + }; + document.onmousemove = function(evt) { + activity_statuses["mouse"] = 1; + }; + window.onscroll = function(evt) { + activity_statuses["mouse"] = 1; + }; + }, init_events_capturing: function() { var logger = this; // Log the task was opened by the worker From 993969ebc03395be91811b7045821b61b0208ea6 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 21:37:09 +0100 Subject: [PATCH 48/62] init activity capturing --- logger.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/logger.js b/logger.js index a47193e..df46aae 100644 --- a/logger.js +++ b/logger.js @@ -29,6 +29,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var logger = this; logger.init_firebase(function() { logger.init_events_capturing(); + logger.init_activity_capturing(); }); }, init_firebase: function(callback) { @@ -85,7 +86,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { activity_statuses["mouse"] = 1; }; window.onscroll = function(evt) { - activity_statuses["mouse"] = 1; + activity_statuses["scroll"] = 1; }; }, init_events_capturing: function() { From eb25a25553223bc77a777e4c9cdfdc79cc35b3ea Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 15 Jan 2016 21:40:48 +0100 Subject: [PATCH 49/62] no message --- logger.js | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/logger.js b/logger.js index df46aae..fa9aa4e 100644 --- a/logger.js +++ b/logger.js @@ -92,10 +92,14 @@ var EDA_LOGGER = EDA_LOGGER || (function() { init_events_capturing: function() { var logger = this; // Log the task was opened by the worker - logger.log_event(_args["firebase_logs"], "opened"); + logger.log_event(_args["firebase_logs"], { + status: "opened" + }); // Log the task page was closed by the worker window.onbeforeunload = function() { - logger.log_event(_args["firebase_logs"], "closed"); + logger.log_event(_args["firebase_logs"], { + status: "closed" + }); }; logger.init_visibility_changes(); }, @@ -118,9 +122,13 @@ var EDA_LOGGER = EDA_LOGGER || (function() { function handleVisibilityChange() { if (document[hidden]) { - logger.log_event(_args["firebase_logs"], "hidden"); + logger.log_event(_args["firebase_logs"], { + status: "hidden" + }); } else { - logger.log_event(_args["firebase_logs"], "active"); + logger.log_event(_args["firebase_logs"], { + status: "active" + }); } } // Warn if the browser doesn't support addEventListener or the Page Visibility API @@ -132,12 +140,9 @@ var EDA_LOGGER = EDA_LOGGER || (function() { document.addEventListener(visibilityChange, handleVisibilityChange, false); } }, - log_event: function(firebase_reference, status) { - firebase_reference.push({ - //session: settings.session, - status: status, - dt: Firebase.ServerValue.TIMESTAMP - }); + log_event: function(firebase_reference, data) { + data['dt'] = Firebase.ServerValue.TIMESTAMP; + firebase_reference.push(data); } }; }()); From 86bf4bee44c47e93bfbb42e72d1f45c272f116c3 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 26 Jan 2016 14:17:00 +0100 Subject: [PATCH 50/62] add key logging --- logger.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/logger.js b/logger.js index fa9aa4e..fc5c2bc 100644 --- a/logger.js +++ b/logger.js @@ -57,6 +57,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { });*/ _args["firebase_logs"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/tab_visibilty"); _args["firebase_activity"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/page_activity"); + _args["firebase_keys"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/key_pressed"); callback(); }; @@ -88,6 +89,11 @@ var EDA_LOGGER = EDA_LOGGER || (function() { window.onscroll = function(evt) { activity_statuses["scroll"] = 1; }; + + document.onkeydown = function() { + var key = event.keyCode || event.charCode; + logger.log_event(_args["firebase_activity"],{"key":key}); + }; }, init_events_capturing: function() { var logger = this; From 1955d79108d259ef91733935335bda406b69565a Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 26 Jan 2016 14:18:57 +0100 Subject: [PATCH 51/62] keys --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index fc5c2bc..9345279 100644 --- a/logger.js +++ b/logger.js @@ -92,7 +92,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { document.onkeydown = function() { var key = event.keyCode || event.charCode; - logger.log_event(_args["firebase_activity"],{"key":key}); + logger.log_event(_args["firebase_keys"],{"key":key}); }; }, init_events_capturing: function() { From 87f5e9eed13d3cddf1c86132a617cd188868a2d2 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 26 Jan 2016 14:25:37 +0100 Subject: [PATCH 52/62] 2 seconds, scroll top --- logger.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/logger.js b/logger.js index 9345279..9ee2c88 100644 --- a/logger.js +++ b/logger.js @@ -1,7 +1,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var settings = { session: Math.floor(Math.random() * 1000000) + 1, - activity_interval: 5000, + activity_interval: 2000, debug: true, firebase: { bucket: "crowdworker-logger" @@ -68,6 +68,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { "keyboard": 0, "mouse": 0, "scroll": 0, + "scroll_top":0, }; setInterval(function() { logger.log_event(_args["firebase_activity"], (function(a) { @@ -75,6 +76,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { "keyboard": 0, "mouse": 0, "scroll": 0, + "scroll_top":0 }; return a; }(activity_statuses))); @@ -88,6 +90,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { }; window.onscroll = function(evt) { activity_statuses["scroll"] = 1; + activity_statuses["scroll_top"] = document.body.scrollTop; }; document.onkeydown = function() { From 23f8ce34369e59d9d556be4d53f6f74d964a958a Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 26 Jan 2016 14:36:19 +0100 Subject: [PATCH 53/62] text selected --- logger.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/logger.js b/logger.js index 9ee2c88..96e4c6e 100644 --- a/logger.js +++ b/logger.js @@ -15,7 +15,15 @@ var EDA_LOGGER = EDA_LOGGER || (function() { function isPageHidden() { return document.hidden || document.msHidden || document.webkitHidden || document.mozHidden; } - + function getSelectedText() { + var text = ""; + if (typeof window.getSelection != "undefined") { + text = window.getSelection().toString(); + } else if (typeof document.selection != "undefined" && document.selection.type == "Text") { + text = document.selection.createRange().text; + } + return text; + } var _args = { key_name: "test_name", key_value: "test_value" @@ -69,6 +77,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { "mouse": 0, "scroll": 0, "scroll_top":0, + "text_selected":0 }; setInterval(function() { logger.log_event(_args["firebase_activity"], (function(a) { @@ -76,7 +85,8 @@ var EDA_LOGGER = EDA_LOGGER || (function() { "keyboard": 0, "mouse": 0, "scroll": 0, - "scroll_top":0 + "scroll_top":0, + "text_selected":0 }; return a; }(activity_statuses))); @@ -85,6 +95,11 @@ var EDA_LOGGER = EDA_LOGGER || (function() { document.onkeydown = function(evt) { activity_statuses["keyboard"] = 1; }; + document.onkeyup = function(evt){ + var selected_text = getSelectedText(); + if (selected_text != "") + activity_statuses["text_selected"] = 1; + } document.onmousemove = function(evt) { activity_statuses["mouse"] = 1; }; From 282b5e09787e8a52dcdd6d2a8d04c8d45c36d79b Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 26 Jan 2016 14:41:51 +0100 Subject: [PATCH 54/62] keyup -> mouse --- logger.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/logger.js b/logger.js index 96e4c6e..1d75c9f 100644 --- a/logger.js +++ b/logger.js @@ -95,9 +95,8 @@ var EDA_LOGGER = EDA_LOGGER || (function() { document.onkeydown = function(evt) { activity_statuses["keyboard"] = 1; }; - document.onkeyup = function(evt){ - var selected_text = getSelectedText(); - if (selected_text != "") + document.onmouseup = function(evt){ + if (getSelectedText() != "") activity_statuses["text_selected"] = 1; } document.onmousemove = function(evt) { From 44257855a1afdbcfb4dbfd5c82debda384731d07 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 26 Jan 2016 15:30:41 +0100 Subject: [PATCH 55/62] mouse clicks --- logger.js | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/logger.js b/logger.js index 1d75c9f..79650e1 100644 --- a/logger.js +++ b/logger.js @@ -11,7 +11,24 @@ var EDA_LOGGER = EDA_LOGGER || (function() { var log = function(message) { if (settings.debug) console.log("[EDA_LOGGER] LOG: " + message); }; - + function fullPath(el){ + var names = []; + while (el.parentNode){ + if (el.id){ + names.unshift('#'+el.id); + break; + }else{ + if (el==el.ownerDocument.documentElement) + names.unshift(el.tagName); + else{ + for (var c=1,e=el;e.previousElementSibling;e=e.previousElementSibling,c++); + names.unshift(el.tagName+":nth-child("+c+")"); + } + el=el.parentNode; + } + } + return names.join(" > "); + } function isPageHidden() { return document.hidden || document.msHidden || document.webkitHidden || document.mozHidden; } @@ -66,6 +83,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { _args["firebase_logs"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/tab_visibilty"); _args["firebase_activity"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/page_activity"); _args["firebase_keys"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/key_pressed"); + _args["firebase_clicks"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/clicks"); callback(); }; @@ -102,15 +120,20 @@ var EDA_LOGGER = EDA_LOGGER || (function() { document.onmousemove = function(evt) { activity_statuses["mouse"] = 1; }; + window.onscroll = function(evt) { activity_statuses["scroll"] = 1; activity_statuses["scroll_top"] = document.body.scrollTop; }; - document.onkeydown = function() { - var key = event.keyCode || event.charCode; + document.onkeydown = function(evt) { + var key = evt.keyCode || evt.charCode; logger.log_event(_args["firebase_keys"],{"key":key}); }; + document.onmousedown = function(evt) { + var element_path = fullPath(evt.path[0]); + logger.log_event(_args["firebase_clicks"],{"element":element_path}); + }; }, init_events_capturing: function() { var logger = this; From a8c49c1db45b7848f7870a0faa04fe2646630ae4 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 4 Mar 2016 10:35:57 -0800 Subject: [PATCH 56/62] Update README.md --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 8a50ed6..87e0ae6 100644 --- a/README.md +++ b/README.md @@ -30,4 +30,15 @@ When you launch the job you should see logs appearing in your firebase bucket. If you do not - check all the steps carefully. If it still does not work, create an *issue* in this repository. +- [ ] copy task with all units +- [ ] make sure reward is as you wanted +- [ ] change job_id in: + - [ ] unit_ids_maker.js + - [ ] queue_management.js +- [ ] run unit_ids_maker.js (once) +- [ ] run queue_management.js (keep it running during execution) +- [ ] run ReLauncher server (npm start) +- [ ] launch the job +- [ ] start ReLauncher for this job + [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) From 645982d746004ab57326e51096c8768fc8b4f088 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 4 Mar 2016 10:37:07 -0800 Subject: [PATCH 57/62] Update README.md --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 87e0ae6..9ccb672 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,7 @@ When you launch the job you should see logs appearing in your firebase bucket. If you do not - check all the steps carefully. If it still does not work, create an *issue* in this repository. -- [ ] copy task with all units -- [ ] make sure reward is as you wanted -- [ ] change job_id in: - - [ ] unit_ids_maker.js - - [ ] queue_management.js -- [ ] run unit_ids_maker.js (once) +- [ ] run unit_references_maker.js (once) - [ ] run queue_management.js (keep it running during execution) - [ ] run ReLauncher server (npm start) - [ ] launch the job From 479254c0f43e2f3305ec5c329180d5ac3b3e5339 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 4 Mar 2016 10:38:03 -0800 Subject: [PATCH 58/62] generators --- firebase_to_csv.js | 29 ++++-- generators/clicks_to_csv.js | 94 ++++++++++++++++++ generators/generate_logs.sh | 22 +++++ generators/key_presses_to_csv.js | 94 ++++++++++++++++++ generators/page_activity_to_csv.js | 103 +++++++++++++++++++ generators/tab_visibility_to_csv.js | 133 +++++++++++++++++++++++++ queue_management.js | 147 ++++++++++++++++++++++++++++ test.js | 3 + unit_ids_maker.js | 54 ++++++++++ 9 files changed, 673 insertions(+), 6 deletions(-) create mode 100644 generators/clicks_to_csv.js create mode 100644 generators/generate_logs.sh create mode 100644 generators/key_presses_to_csv.js create mode 100644 generators/page_activity_to_csv.js create mode 100644 generators/tab_visibility_to_csv.js create mode 100644 queue_management.js create mode 100644 test.js create mode 100644 unit_ids_maker.js diff --git a/firebase_to_csv.js b/firebase_to_csv.js index a50d272..088b362 100644 --- a/firebase_to_csv.js +++ b/firebase_to_csv.js @@ -3,7 +3,7 @@ var fs = require('fs'); var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/'; -var task_id = 836250; +var task_id = 837900; var firebase_target_url = firebase_base_url + task_id + ".json"; var folder = "logs/"; var filename = folder + task_id + "_logs.csv"; @@ -12,18 +12,21 @@ fs.createWriteStream(filename); fs.truncate(filename, 0, function() { console.log('file ' + filename + ' was cleaned up.') }); -fs.appendFile(filename, 'unit_id, assignment_id, worker_id, dt_start, dt_end, status\n', function(err) {}); +fs.appendFile(filename, 'unit_number, unit_id, assignment_id, dt_start, dt_end, status\n', function(err) {}); var Logs = []; +var Units = {}; +var MaxUnit = 0; // ------------------------------------------------------- // Convert log object into a string // ------------------------------------------------------- function stringify(log_array) { for (var i = 0; i < log_array.length; i++) { log_array[i]['string'] = ""; + log_array[i]['string'] += log_array[i].unit_number + ", "; log_array[i]['string'] += log_array[i].unit_id + ", "; log_array[i]['string'] += log_array[i].assignment_id + ", "; - log_array[i]['string'] += log_array[i].worker_id + ", "; + //log_array[i]['string'] += log_array[i].worker_id + ", "; log_array[i]['string'] += log_array[i].dt_start + ", "; log_array[i]['string'] += log_array[i].dt_end + ", "; log_array[i]['string'] += log_array[i].status; @@ -42,20 +45,33 @@ requestify.get(firebase_target_url, { var assignments = response.getBody(); for (var a_id in assignments) { var a = assignments[a_id]; - if (a_id != "editor_preview" && a.unit_id != undefined) { + if (a_id != "editor_preview" && a.unit_id) { + var unit_number; + if (Units[a.unit_id.toString()]) { + unit_number = Units[a.unit_id.toString()]; + } else { + MaxUnit++; + Units[a.unit_id.toString()] = MaxUnit; + unit_number = Units[a.unit_id.toString()]; + } + + + + //console.log(a); for (var l_id in a.logs) { var l = a.logs[l_id]; var log_record = { + 'unit_number': unit_number, 'unit_id': a.unit_id, 'assignment_id': a_id, - 'worker_id': a.worker_id, + //'worker_id': a.worker_id || "0", 'dt_start': l.dt, 'status': l.status } Logs.push(log_record); } } else { - console.log("PROBLEMATIC ASSIGNMENT", a); + console.log("PROBLEMATIC ASSIGNMENT", a_id, a); } } // ------------------------------------------------------- @@ -76,6 +92,7 @@ requestify.get(firebase_target_url, { // Convert each object in array into string and save it into the log file // ------------------------------------------------------- Logs = stringify(Logs); + //console.log(Logs.length); for (var i = 0; i < Logs.length; i++) { console.log(Logs[i]['string']); fs.appendFile(filename, Logs[i]['string'] + '\n', function(err) {}); diff --git a/generators/clicks_to_csv.js b/generators/clicks_to_csv.js new file mode 100644 index 0000000..103e708 --- /dev/null +++ b/generators/clicks_to_csv.js @@ -0,0 +1,94 @@ +var requestify = require('requestify'); +var fs = require('fs'); + + +var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/'; +var task_id = process.argv[2]; +var firebase_target_url = firebase_base_url + task_id + ".json"; +var folder = "../logs/"+task_id+"/"; +var filename = folder + task_id + "_clicks.csv"; + +fs.createWriteStream(filename); +fs.truncate(filename, 0, function() { + console.log('file ' + filename + ' was cleaned up.') +}); +fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start, element\n', function(err) {}); + +var LOGS = []; + +var MaxUnit = 0; +// ------------------------------------------------------- +// Convert log object into a string +// ------------------------------------------------------- +function stringify(log_array) { + for (var i = 0; i < log_array.length; i++) { + log_array[i]['string'] = ""; + log_array[i]['string'] += log_array[i].task_id + ", "; + log_array[i]['string'] += log_array[i].unit_id + ", "; + log_array[i]['string'] += log_array[i].assignment_id + ", "; + log_array[i]['string'] += log_array[i].session_id + ", "; + log_array[i]['string'] += log_array[i].dt_start + ", "; + log_array[i]['string'] += log_array[i].element; + } + return log_array; +} + +function endify(log_array) { + var Logs = log_array; + for (var i = 0; i < Logs.length; i++) { + if (Logs[i].status != 'closed') { + if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) { + Logs[i]['dt_end'] = Logs[i + 1]['dt_start']; + } + } + if (Logs[i]['dt_end'] == undefined) { + Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000; + } + + } + return Logs; +} +// ------------------------------------------------------- +// Collect data for a given Job and make a plain Array of log objects +// ------------------------------------------------------- +requestify.get(firebase_target_url, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var job_logs = response.getBody(); + var units = job_logs.units + for (var unit_id in units) { + var assignments = units[unit_id]['assignments']; + for (var assignment_id in assignments) { + var sessions = assignments[assignment_id]['sessions']; + for (var session_id in sessions) { + var logs = sessions[session_id]['clicks']; + //console.log(logs); + for (var log_id in logs) { + var log_record = { + task_id: task_id, + unit_id: unit_id, + assignment_id: assignment_id, + session_id: session_id, + dt_start: logs[log_id].dt, + element: logs[log_id].element + }; + LOGS.push(log_record); + } + } + } + } + //LOGS = endify(LOGS); + LOGS = stringify(LOGS); + console.log(LOGS.length); + var all_text = ""; + for (var i = 0; i < LOGS.length; i++) { + all_text+=LOGS[i]['string'] + '\n'; + } + fs.appendFile(filename, all_text, function(err) { + if (err) + console.log(err); + }); + }); diff --git a/generators/generate_logs.sh b/generators/generate_logs.sh new file mode 100644 index 0000000..5d787c1 --- /dev/null +++ b/generators/generate_logs.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +echo "====================================" +echo "= FIREBASE CROWDFLOWER LOGS TO CSV =" +echo "====================================" + +echo "\nNow we create a folder for $@ task" +mkdir "../logs/$@" +echo "\nNow we generate logs of key presses from firebase..." +node key_presses_to_csv.js $@ +echo "\nNow we generate logs of clicks from firebase..." +node clicks_to_csv.js $@ +echo "\nNow we generate logs of page activity from firebase..." +node page_activity_to_csv.js $@ +echo "\nNow we generate logs of tab visibility from firebase..." +node tab_visibility_to_csv.js $@ + +echo "\nNow we copy the folder with logs to stats-and-graphs folder to be further processed" +cp -r "../logs/$@" "../../stats-and-graphs/logs/$@" + +echo "\nDONE" +echo "====================================" \ No newline at end of file diff --git a/generators/key_presses_to_csv.js b/generators/key_presses_to_csv.js new file mode 100644 index 0000000..996d239 --- /dev/null +++ b/generators/key_presses_to_csv.js @@ -0,0 +1,94 @@ +var requestify = require('requestify'); +var fs = require('fs'); + + +var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/'; +var task_id = process.argv[2]; +var firebase_target_url = firebase_base_url + task_id + ".json"; +var folder = "../logs/"+task_id+"/"; +var filename = folder + task_id + "_keys.csv"; + +fs.createWriteStream(filename); +fs.truncate(filename, 0, function() { + console.log('file ' + filename + ' was cleaned up.') +}); +fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start, key\n', function(err) {}); + +var LOGS = []; + +var MaxUnit = 0; +// ------------------------------------------------------- +// Convert log object into a string +// ------------------------------------------------------- +function stringify(log_array) { + for (var i = 0; i < log_array.length; i++) { + log_array[i]['string'] = ""; + log_array[i]['string'] += log_array[i].task_id + ", "; + log_array[i]['string'] += log_array[i].unit_id + ", "; + log_array[i]['string'] += log_array[i].assignment_id + ", "; + log_array[i]['string'] += log_array[i].session_id + ", "; + log_array[i]['string'] += log_array[i].dt_start + ", "; + log_array[i]['string'] += log_array[i].key; + } + return log_array; +} + +function endify(log_array) { + var Logs = log_array; + for (var i = 0; i < Logs.length; i++) { + if (Logs[i].status != 'closed') { + if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) { + Logs[i]['dt_end'] = Logs[i + 1]['dt_start']; + } + } + if (Logs[i]['dt_end'] == undefined) { + Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000; + } + + } + return Logs; +} +// ------------------------------------------------------- +// Collect data for a given Job and make a plain Array of log objects +// ------------------------------------------------------- +requestify.get(firebase_target_url, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var job_logs = response.getBody(); + var units = job_logs.units + for (var unit_id in units) { + var assignments = units[unit_id]['assignments']; + for (var assignment_id in assignments) { + var sessions = assignments[assignment_id]['sessions']; + for (var session_id in sessions) { + var logs = sessions[session_id]['key_pressed']; + //console.log(logs); + for (var log_id in logs) { + var log_record = { + task_id: task_id, + unit_id: unit_id, + assignment_id: assignment_id, + session_id: session_id, + dt_start: logs[log_id].dt, + key: logs[log_id].key + }; + LOGS.push(log_record); + } + } + } + } + //LOGS = endify(LOGS); + LOGS = stringify(LOGS); + console.log(LOGS.length); + var all_text = ""; + for (var i = 0; i < LOGS.length; i++) { + all_text+=LOGS[i]['string'] + '\n'; + } + fs.appendFile(filename, all_text, function(err) { + if (err) + console.log(err); + }); + }); diff --git a/generators/page_activity_to_csv.js b/generators/page_activity_to_csv.js new file mode 100644 index 0000000..ef28105 --- /dev/null +++ b/generators/page_activity_to_csv.js @@ -0,0 +1,103 @@ +var requestify = require('requestify'); +var fs = require('fs'); + + +var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/'; +var task_id = process.argv[2]; +var firebase_target_url = firebase_base_url + task_id + ".json"; +var folder = "../logs/"+task_id+"/"; +var filename = folder + task_id + "_page.csv"; + +fs.createWriteStream(filename); +fs.truncate(filename, 0, function() { + console.log('file ' + filename + ' was cleaned up.') +}); +fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start,dt_end, keyboard, mouse, scroll, scroll_top, text_selected\n', function(err) {}); + +var LOGS = []; + +var MaxUnit = 0; +// ------------------------------------------------------- +// Convert log object into a string +// ------------------------------------------------------- +function stringify(log_array) { + for (var i = 0; i < log_array.length; i++) { + log_array[i]['string'] = ""; + log_array[i]['string'] += log_array[i].task_id + ", "; + log_array[i]['string'] += log_array[i].unit_id + ", "; + log_array[i]['string'] += log_array[i].assignment_id + ", "; + log_array[i]['string'] += log_array[i].session_id + ", "; + log_array[i]['string'] += log_array[i].dt_start + ", "; + log_array[i]['string'] += log_array[i].dt_end + ", "; + log_array[i]['string'] += log_array[i].keyboard + ", "; + log_array[i]['string'] += log_array[i].mouse + ", "; + log_array[i]['string'] += log_array[i].scroll + ", "; + log_array[i]['string'] += log_array[i].scroll_top + ", "; + log_array[i]['string'] += log_array[i].text_selected; + } + return log_array; +} + +function endify(log_array) { + var Logs = log_array; + for (var i = 0; i < Logs.length; i++) { + if (Logs[i].status != 'closed') { + if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) { + Logs[i]['dt_end'] = Logs[i + 1]['dt_start']; + } + } + if (Logs[i]['dt_end'] == undefined) { + Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000; + } + + } + return Logs; +} +// ------------------------------------------------------- +// Collect data for a given Job and make a plain Array of log objects +// ------------------------------------------------------- +requestify.get(firebase_target_url, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var job_logs = response.getBody(); + var units = job_logs.units + for (var unit_id in units) { + var assignments = units[unit_id]['assignments']; + for (var assignment_id in assignments) { + var sessions = assignments[assignment_id]['sessions']; + for (var session_id in sessions) { + var logs = sessions[session_id]['page_activity']; + //console.log(logs); + for (var log_id in logs) { + var log_record = { + task_id: task_id, + unit_id: unit_id, + assignment_id: assignment_id, + session_id: session_id, + dt_start: logs[log_id].dt, + keyboard: logs[log_id].keyboard, + mouse: logs[log_id].mouse, + scroll: logs[log_id].scroll, + scroll_top: Math.round(logs[log_id].scroll_top), + text_selected: logs[log_id].text_selected + }; + LOGS.push(log_record); + } + } + } + } + LOGS = endify(LOGS); + LOGS = stringify(LOGS); + console.log(LOGS.length); + var all_text = ""; + for (var i = 0; i < LOGS.length; i++) { + all_text+=LOGS[i]['string'] + '\n'; + } + fs.appendFile(filename, all_text, function(err) { + if (err) + console.log(err); + }); + }); diff --git a/generators/tab_visibility_to_csv.js b/generators/tab_visibility_to_csv.js new file mode 100644 index 0000000..0fa352d --- /dev/null +++ b/generators/tab_visibility_to_csv.js @@ -0,0 +1,133 @@ +var requestify = require('requestify'); +var fs = require('fs'); + + +var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/'; +var task_id = process.argv[2]; +var firebase_target_url = firebase_base_url + task_id + ".json"; +var folder = "../logs/"+task_id+"/"; +var filename = folder + task_id + "_tabs.csv"; + +fs.createWriteStream(filename); +fs.truncate(filename, 0, function() { + console.log('file ' + filename + ' was cleaned up.') +}); +fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start,dt_end, status\n', function(err) {}); + +var LOGS = []; + +var MaxUnit = 0; + +function dynamicSort(property) { + var sortOrder = 1; + if(property[0] === "-") { + sortOrder = -1; + property = property.substr(1); + } + return function (a,b) { + var result = (a[property] < b[property]) ? -1 : (a[property] > b[property]) ? 1 : 0; + return result * sortOrder; + } +} +function dynamicSortMultiple() { + /* + * save the arguments object as it will be overwritten + * note that arguments object is an array-like object + * consisting of the names of the properties to sort by + */ + var props = arguments; + return function (obj1, obj2) { + var i = 0, result = 0, numberOfProperties = props.length; + /* try getting a different result from 0 (equal) + * as long as we have extra properties to compare + */ + while(result === 0 && i < numberOfProperties) { + result = dynamicSort(props[i])(obj1, obj2); + i++; + } + return result; + } +} +// ------------------------------------------------------- +// Convert log object into a string +// ------------------------------------------------------- +function stringify(log_array) { + for (var i = 0; i < log_array.length; i++) { + log_array[i]['string'] = ""; + log_array[i]['string'] += log_array[i].task_id + ", "; + log_array[i]['string'] += log_array[i].unit_id + ", "; + log_array[i]['string'] += log_array[i].assignment_id + ", "; + log_array[i]['string'] += log_array[i].session_id + ", "; + log_array[i]['string'] += log_array[i].dt_start + ", "; + log_array[i]['string'] += log_array[i].dt_end + ", "; + log_array[i]['string'] += log_array[i].status; + } + return log_array; +} + +function endify(log_array) { + var Logs = log_array; + for (var i = 0; i < Logs.length; i++) { + if (Logs[i].status != 'closed') { + if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) { + Logs[i]['dt_end'] = Logs[i + 1]['dt_start']; + } + } + if (Logs[i]['dt_end'] == undefined) { + Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000; + } + + } + return Logs; +} +// ------------------------------------------------------- +// Collect data for a given Job and make a plain Array of log objects +// ------------------------------------------------------- +requestify.get(firebase_target_url, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var job_logs = response.getBody(); + var units = job_logs.units + for (var unit_id in units) { + var assignments = units[unit_id]['assignments']; + for (var assignment_id in assignments) { + var sessions = assignments[assignment_id]['sessions']; + for (var session_id in sessions) { + var logs = sessions[session_id]['tab_visibilty']; + // console.log(logs); + for (var log_id in logs) { + var log_record = { + task_id: task_id, + unit_id: parseInt(unit_id), + assignment_id: assignment_id, + session_id: parseInt(session_id), + dt_start: parseInt((logs[log_id].status != "closed")?logs[log_id].dt:logs[log_id].dt+500), + status: logs[log_id].status + }; + LOGS.push(log_record); + } + } + } + } + LOGS.sort(dynamicSortMultiple("session_id","dt_start")); + //LOGS.sort(dynamicSort("assignment_id")); + //LOGS.sort(dynamicSort("unit_id")); + + LOGS = endify(LOGS); + LOGS = stringify(LOGS); + console.log(LOGS.length); + var all_text = ""; + for (var i = 0; i < LOGS.length; i++) { + all_text+=LOGS[i]['string'] + '\n'; + } + fs.appendFile(filename, all_text, function(err) { + if (err) + console.log(err); + }); + console.log(firebase_target_url); + //console.log(firebase_target_url); + + }); diff --git a/queue_management.js b/queue_management.js new file mode 100644 index 0000000..d120d72 --- /dev/null +++ b/queue_management.js @@ -0,0 +1,147 @@ +var Firebase = require('firebase'); +var requestify = require('requestify'); + + +var JOB_TO_BE_PROCESSED = 850870; +var DELAY_QUEUE_TIME = 30000; + + +var Crowdflower = { + base: "http://api.crowdflower.com/v1/", + api_key: process.env.CROWDFLOWER_API_KEY, + getUnit: function(job_id, unit_id, callback) { + var cf = Crowdflower; + var crowdflower_unit_url = cf.base + "/jobs/" + job_id + "/units/" + unit_id + ".json?key=" + cf.api_key; + console.log(crowdflower_unit_url); + requestify.get(crowdflower_unit_url, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var crowdflower_unit = response.getBody(); + callback(crowdflower_unit); + }); + } +} +var Firebase_urls = { + base: 'https://crowdworker-logger.firebaseio.com/', + crowdflower: "taskscrowdflowercom/", + queue: function() { + var fb = Firebase_urls; + return fb.base + "queue/"; + }, + tasks: function() { + var fb = Firebase_urls; + var url = fb.base + fb.crowdflower; + return url; + }, + assignments: function(job_id) { + var fb = Firebase_urls; + var url = fb.base + fb.crowdflower + job_id + "/"; + return url; + } +} + +function getLastLog(logs) { + if (logs) { + var last_log_id = Object.keys(logs)[Object.keys(logs).length - 1]; + var last_log = logs[last_log_id]; + return last_log; + } + return false; +} + + +function addToQueue(job_id, unit_id, assignment_id) { + console.log(DELAY_QUEUE_TIME / 1000 + " seconds passed"); + var assignment_ref = new Firebase(Firebase_urls.queue()); + assignment_ref.push({ + suspect: "abandoned", + job_id: job_id, + unit_id: unit_id, + assignment_id: assignment_id, + dt: Firebase.ServerValue.TIMESTAMP + }); + console.log("event is added to the queue"); +} + +function relaunchUnit(job_id, unit_id) { + requestify.post("http://localhost:3000/jobs/" + job_id + "/units/" + unit_id + "/relaunch", { + api_key: Crowdflower.api_key + }, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var resp = response.getBody(); + console.log(resp); + }); +} + +console.log("====================\nStart Logs Monitoring\n===================="); + +var logs_ref = new Firebase(Firebase_urls.assignments(JOB_TO_BE_PROCESSED)); +logs_ref.on('child_changed', function(assignment_snapshot, prevchildname) { + var job_id = JOB_TO_BE_PROCESSED; + + var assignment_details = assignment_snapshot.val(); + var assignment_id = assignment_snapshot.key(); + + var unit_id = assignment_details['unit_id']; + var logs = assignment_details['logs']; + + if (unit_id) { + last_log = getLastLog(logs); + if (last_log && last_log.status == "closed") { + console.log(unit_id, assignment_id, assignment_details["status"]); + //add the unit to queue to be processed + setTimeout(function() { + addToQueue(job_id, unit_id, assignment_id); + }, DELAY_QUEUE_TIME); + } + } +}); + + + + + +console.log("====================\nStart Queue Monitoring\n===================="); +var ref_queue = new Firebase(Firebase_urls.queue()); +ref_queue.on('child_added', function(queue_element_snapshot, prevchildname) { + console.log("processing a queue element"); + + var element = queue_element_snapshot.val(); + var unit_id = element['unit_id']; + var job_id = element['job_id']; + var assignment_id = element['assignment_id']; + var assignment_ref = new Firebase(Firebase_urls.assignments(job_id) + assignment_id + '/logs/'); + + assignment_ref.orderByChild("dt").limitToLast(1).once("value", function(log_snapshot) { + var logs = log_snapshot.val(); + var last_log = getLastLog(logs); + console.log(last_log); + // var logs = assignment['logs']; + // getLastLog(logs, function(last_log) { + if (last_log.status == "closed") { + console.log("check unit state at CrowdFlower"); + setTimeout(function() { + console.log(DELAY_QUEUE_TIME/1000+" seconds passed"); + Crowdflower.getUnit(job_id, unit_id, function(cf_unit) { + console.log(cf_unit['state']); + if (cf_unit['state'] == "judging") { + console.log("to relaunch the unit"); + relaunchUnit(job_id, unit_id); + } + }); + }, DELAY_QUEUE_TIME); + + } + }); + // remove element from queue, because it is checked; + var queue_element_to_be_deleted = new Firebase(Firebase_urls.queue() + queue_element_snapshot.key()); + queue_element_to_be_deleted.set(null); + console.log("element " + queue_element_snapshot.key() + " is removed"); +}); diff --git a/test.js b/test.js new file mode 100644 index 0000000..f860259 --- /dev/null +++ b/test.js @@ -0,0 +1,3 @@ +process.argv.forEach(function (val, index, array) { + console.log(index + ': ' + val); +}); \ No newline at end of file diff --git a/unit_ids_maker.js b/unit_ids_maker.js new file mode 100644 index 0000000..648eecd --- /dev/null +++ b/unit_ids_maker.js @@ -0,0 +1,54 @@ +var Firebase = require('firebase'); +var requestify = require('requestify'); +//var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/'; +//var Ref = new Firebase(firebase_base_url); +var CROWDFLOWER_API_KEY = process.env.CROWDFLOWER_API_KEY; +var JOB_TO_BE_PROCESSED = 855249; + +var Crowdflower = { + base: "http://api.crowdflower.com/v1/", + api_key: process.env.CROWDFLOWER_API_KEY, + getUnits: function(job_id, callback) { + var cf = Crowdflower; + var crowdflower_units_url = cf.base + "/jobs/" + job_id + "/units.json?key=" + cf.api_key; + console.log(crowdflower_units_url); + requestify.get(crowdflower_units_url, { + headers: { + "Accept": "application/json" + } + }) + .then(function(response) { + var units = response.getBody(); + if (callback) + callback(units); + }); + }, + updateUnit: function(job_id, unit_id, data, callback) { + var cf = Crowdflower; + var crowdflower_unit_url = cf.base + "/jobs/" + job_id + "/units/" + unit_id + ".json?key=" + cf.api_key; + + var put_data = { + unit: { + data: data + } + }; + console.log(put_data); + requestify.put(crowdflower_unit_url, put_data) + .then(function(crowdflower_resp) { + var unit_info = crowdflower_resp.getBody(); + if (callback) + callback(unit_info); + }); + } +} + +Crowdflower.getUnits(JOB_TO_BE_PROCESSED, function(units) { + for (var unit_id in units) { + var unit_data = units[unit_id]; + unit_data['re_unit_id'] = unit_id; + //console.log(unit_data); + Crowdflower.updateUnit(JOB_TO_BE_PROCESSED, unit_id, unit_data, function(updated_unit) { + console.log(updated_unit); + }); + } +}) From 5f9a287725355cde977d49f1daeef6afb1a66112 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 14 Feb 2017 22:05:30 +0100 Subject: [PATCH 59/62] update firebase to 3.6.9 --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index 79650e1..7dc5cf1 100644 --- a/logger.js +++ b/logger.js @@ -59,7 +59,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { }, init_firebase: function(callback) { var firebase_script = document.createElement('script'); - firebase_script.src = "https://cdn.firebase.com/js/client/2.2.9/firebase.js"; + firebase_script.src = "https://www.gstatic.com/firebasejs/3.6.9/firebase.js"; log(firebase_script.src); From 87f15896b85518fe0862f389767e94ef4014c18b Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 14 Feb 2017 22:16:09 +0100 Subject: [PATCH 60/62] updated back to 2.2.9 --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index 7dc5cf1..79650e1 100644 --- a/logger.js +++ b/logger.js @@ -59,7 +59,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { }, init_firebase: function(callback) { var firebase_script = document.createElement('script'); - firebase_script.src = "https://www.gstatic.com/firebasejs/3.6.9/firebase.js"; + firebase_script.src = "https://cdn.firebase.com/js/client/2.2.9/firebase.js"; log(firebase_script.src); From e8d0a02a91b2d7d60a3bea9f5264de88174743cc Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Tue, 14 Feb 2017 22:45:45 +0100 Subject: [PATCH 61/62] Update logger.js --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index 79650e1..cff3e5c 100644 --- a/logger.js +++ b/logger.js @@ -4,7 +4,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { activity_interval: 2000, debug: true, firebase: { - bucket: "crowdworker-logger" + bucket: "IMAGE-TAGGING-APP" } } // A log function which can be easily turned of using debug variable From 0e33cbb0cc0d93853c379dbbb22a11f8478547f1 Mon Sep 17 00:00:00 2001 From: Pavel Kucherbaev Date: Fri, 12 May 2017 15:18:07 +0200 Subject: [PATCH 62/62] Update logger.js --- logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logger.js b/logger.js index cff3e5c..79650e1 100644 --- a/logger.js +++ b/logger.js @@ -4,7 +4,7 @@ var EDA_LOGGER = EDA_LOGGER || (function() { activity_interval: 2000, debug: true, firebase: { - bucket: "IMAGE-TAGGING-APP" + bucket: "crowdworker-logger" } } // A log function which can be easily turned of using debug variable