@@ -18,4 +26,5 @@
CrowdFlower Launcher
+
diff --git a/firebase_to_csv.js b/firebase_to_csv.js
new file mode 100644
index 0000000..088b362
--- /dev/null
+++ b/firebase_to_csv.js
@@ -0,0 +1,101 @@
+var requestify = require('requestify');
+var fs = require('fs');
+
+
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var task_id = 837900;
+var firebase_target_url = firebase_base_url + task_id + ".json";
+var folder = "logs/";
+var filename = folder + task_id + "_logs.csv";
+
+fs.createWriteStream(filename);
+fs.truncate(filename, 0, function() {
+ console.log('file ' + filename + ' was cleaned up.')
+});
+fs.appendFile(filename, 'unit_number, unit_id, assignment_id, dt_start, dt_end, status\n', function(err) {});
+
+var Logs = [];
+var Units = {};
+var MaxUnit = 0;
+// -------------------------------------------------------
+// Convert log object into a string
+// -------------------------------------------------------
+function stringify(log_array) {
+ for (var i = 0; i < log_array.length; i++) {
+ log_array[i]['string'] = "";
+ log_array[i]['string'] += log_array[i].unit_number + ", ";
+ log_array[i]['string'] += log_array[i].unit_id + ", ";
+ log_array[i]['string'] += log_array[i].assignment_id + ", ";
+ //log_array[i]['string'] += log_array[i].worker_id + ", ";
+ log_array[i]['string'] += log_array[i].dt_start + ", ";
+ log_array[i]['string'] += log_array[i].dt_end + ", ";
+ log_array[i]['string'] += log_array[i].status;
+ }
+ return log_array;
+}
+// -------------------------------------------------------
+// Collect data for a given Job and make a plain Array of log objects
+// -------------------------------------------------------
+requestify.get(firebase_target_url, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var assignments = response.getBody();
+ for (var a_id in assignments) {
+ var a = assignments[a_id];
+ if (a_id != "editor_preview" && a.unit_id) {
+ var unit_number;
+ if (Units[a.unit_id.toString()]) {
+ unit_number = Units[a.unit_id.toString()];
+ } else {
+ MaxUnit++;
+ Units[a.unit_id.toString()] = MaxUnit;
+ unit_number = Units[a.unit_id.toString()];
+ }
+
+
+
+ //console.log(a);
+ for (var l_id in a.logs) {
+ var l = a.logs[l_id];
+ var log_record = {
+ 'unit_number': unit_number,
+ 'unit_id': a.unit_id,
+ 'assignment_id': a_id,
+ //'worker_id': a.worker_id || "0",
+ 'dt_start': l.dt,
+ 'status': l.status
+ }
+ Logs.push(log_record);
+ }
+ } else {
+ console.log("PROBLEMATIC ASSIGNMENT", a_id, a);
+ }
+ }
+ // -------------------------------------------------------
+ // Add end events to log objects
+ // -------------------------------------------------------
+ for (var i = 0; i < Logs.length; i++) {
+ if (Logs[i].status != 'closed') {
+ if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id) {
+ Logs[i]['dt_end'] = Logs[i + 1]['dt_start'];
+ }
+ }
+ if (Logs[i]['dt_end'] == undefined) {
+ Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000;
+ }
+
+ }
+ // -------------------------------------------------------
+ // Convert each object in array into string and save it into the log file
+ // -------------------------------------------------------
+ Logs = stringify(Logs);
+ //console.log(Logs.length);
+ for (var i = 0; i < Logs.length; i++) {
+ console.log(Logs[i]['string']);
+ fs.appendFile(filename, Logs[i]['string'] + '\n', function(err) {});
+ }
+ console.log(firebase_target_url);
+ });
diff --git a/generators/clicks_to_csv.js b/generators/clicks_to_csv.js
new file mode 100644
index 0000000..103e708
--- /dev/null
+++ b/generators/clicks_to_csv.js
@@ -0,0 +1,94 @@
+var requestify = require('requestify');
+var fs = require('fs');
+
+
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var task_id = process.argv[2];
+var firebase_target_url = firebase_base_url + task_id + ".json";
+var folder = "../logs/"+task_id+"/";
+var filename = folder + task_id + "_clicks.csv";
+
+fs.createWriteStream(filename);
+fs.truncate(filename, 0, function() {
+ console.log('file ' + filename + ' was cleaned up.')
+});
+fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start, element\n', function(err) {});
+
+var LOGS = [];
+
+var MaxUnit = 0;
+// -------------------------------------------------------
+// Convert log object into a string
+// -------------------------------------------------------
+function stringify(log_array) {
+ for (var i = 0; i < log_array.length; i++) {
+ log_array[i]['string'] = "";
+ log_array[i]['string'] += log_array[i].task_id + ", ";
+ log_array[i]['string'] += log_array[i].unit_id + ", ";
+ log_array[i]['string'] += log_array[i].assignment_id + ", ";
+ log_array[i]['string'] += log_array[i].session_id + ", ";
+ log_array[i]['string'] += log_array[i].dt_start + ", ";
+ log_array[i]['string'] += log_array[i].element;
+ }
+ return log_array;
+}
+
+function endify(log_array) {
+ var Logs = log_array;
+ for (var i = 0; i < Logs.length; i++) {
+ if (Logs[i].status != 'closed') {
+ if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) {
+ Logs[i]['dt_end'] = Logs[i + 1]['dt_start'];
+ }
+ }
+ if (Logs[i]['dt_end'] == undefined) {
+ Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000;
+ }
+
+ }
+ return Logs;
+}
+// -------------------------------------------------------
+// Collect data for a given Job and make a plain Array of log objects
+// -------------------------------------------------------
+requestify.get(firebase_target_url, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var job_logs = response.getBody();
+ var units = job_logs.units
+ for (var unit_id in units) {
+ var assignments = units[unit_id]['assignments'];
+ for (var assignment_id in assignments) {
+ var sessions = assignments[assignment_id]['sessions'];
+ for (var session_id in sessions) {
+ var logs = sessions[session_id]['clicks'];
+ //console.log(logs);
+ for (var log_id in logs) {
+ var log_record = {
+ task_id: task_id,
+ unit_id: unit_id,
+ assignment_id: assignment_id,
+ session_id: session_id,
+ dt_start: logs[log_id].dt,
+ element: logs[log_id].element
+ };
+ LOGS.push(log_record);
+ }
+ }
+ }
+ }
+ //LOGS = endify(LOGS);
+ LOGS = stringify(LOGS);
+ console.log(LOGS.length);
+ var all_text = "";
+ for (var i = 0; i < LOGS.length; i++) {
+ all_text+=LOGS[i]['string'] + '\n';
+ }
+ fs.appendFile(filename, all_text, function(err) {
+ if (err)
+ console.log(err);
+ });
+ });
diff --git a/generators/generate_logs.sh b/generators/generate_logs.sh
new file mode 100644
index 0000000..5d787c1
--- /dev/null
+++ b/generators/generate_logs.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+echo "===================================="
+echo "= FIREBASE CROWDFLOWER LOGS TO CSV ="
+echo "===================================="
+
+echo "\nNow we create a folder for $@ task"
+mkdir "../logs/$@"
+echo "\nNow we generate logs of key presses from firebase..."
+node key_presses_to_csv.js $@
+echo "\nNow we generate logs of clicks from firebase..."
+node clicks_to_csv.js $@
+echo "\nNow we generate logs of page activity from firebase..."
+node page_activity_to_csv.js $@
+echo "\nNow we generate logs of tab visibility from firebase..."
+node tab_visibility_to_csv.js $@
+
+echo "\nNow we copy the folder with logs to stats-and-graphs folder to be further processed"
+cp -r "../logs/$@" "../../stats-and-graphs/logs/$@"
+
+echo "\nDONE"
+echo "===================================="
\ No newline at end of file
diff --git a/generators/key_presses_to_csv.js b/generators/key_presses_to_csv.js
new file mode 100644
index 0000000..996d239
--- /dev/null
+++ b/generators/key_presses_to_csv.js
@@ -0,0 +1,94 @@
+var requestify = require('requestify');
+var fs = require('fs');
+
+
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var task_id = process.argv[2];
+var firebase_target_url = firebase_base_url + task_id + ".json";
+var folder = "../logs/"+task_id+"/";
+var filename = folder + task_id + "_keys.csv";
+
+fs.createWriteStream(filename);
+fs.truncate(filename, 0, function() {
+ console.log('file ' + filename + ' was cleaned up.')
+});
+fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start, key\n', function(err) {});
+
+var LOGS = [];
+
+var MaxUnit = 0;
+// -------------------------------------------------------
+// Convert log object into a string
+// -------------------------------------------------------
+function stringify(log_array) {
+ for (var i = 0; i < log_array.length; i++) {
+ log_array[i]['string'] = "";
+ log_array[i]['string'] += log_array[i].task_id + ", ";
+ log_array[i]['string'] += log_array[i].unit_id + ", ";
+ log_array[i]['string'] += log_array[i].assignment_id + ", ";
+ log_array[i]['string'] += log_array[i].session_id + ", ";
+ log_array[i]['string'] += log_array[i].dt_start + ", ";
+ log_array[i]['string'] += log_array[i].key;
+ }
+ return log_array;
+}
+
+function endify(log_array) {
+ var Logs = log_array;
+ for (var i = 0; i < Logs.length; i++) {
+ if (Logs[i].status != 'closed') {
+ if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) {
+ Logs[i]['dt_end'] = Logs[i + 1]['dt_start'];
+ }
+ }
+ if (Logs[i]['dt_end'] == undefined) {
+ Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000;
+ }
+
+ }
+ return Logs;
+}
+// -------------------------------------------------------
+// Collect data for a given Job and make a plain Array of log objects
+// -------------------------------------------------------
+requestify.get(firebase_target_url, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var job_logs = response.getBody();
+ var units = job_logs.units
+ for (var unit_id in units) {
+ var assignments = units[unit_id]['assignments'];
+ for (var assignment_id in assignments) {
+ var sessions = assignments[assignment_id]['sessions'];
+ for (var session_id in sessions) {
+ var logs = sessions[session_id]['key_pressed'];
+ //console.log(logs);
+ for (var log_id in logs) {
+ var log_record = {
+ task_id: task_id,
+ unit_id: unit_id,
+ assignment_id: assignment_id,
+ session_id: session_id,
+ dt_start: logs[log_id].dt,
+ key: logs[log_id].key
+ };
+ LOGS.push(log_record);
+ }
+ }
+ }
+ }
+ //LOGS = endify(LOGS);
+ LOGS = stringify(LOGS);
+ console.log(LOGS.length);
+ var all_text = "";
+ for (var i = 0; i < LOGS.length; i++) {
+ all_text+=LOGS[i]['string'] + '\n';
+ }
+ fs.appendFile(filename, all_text, function(err) {
+ if (err)
+ console.log(err);
+ });
+ });
diff --git a/generators/page_activity_to_csv.js b/generators/page_activity_to_csv.js
new file mode 100644
index 0000000..ef28105
--- /dev/null
+++ b/generators/page_activity_to_csv.js
@@ -0,0 +1,103 @@
+var requestify = require('requestify');
+var fs = require('fs');
+
+
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var task_id = process.argv[2];
+var firebase_target_url = firebase_base_url + task_id + ".json";
+var folder = "../logs/"+task_id+"/";
+var filename = folder + task_id + "_page.csv";
+
+fs.createWriteStream(filename);
+fs.truncate(filename, 0, function() {
+ console.log('file ' + filename + ' was cleaned up.')
+});
+fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start,dt_end, keyboard, mouse, scroll, scroll_top, text_selected\n', function(err) {});
+
+var LOGS = [];
+
+var MaxUnit = 0;
+// -------------------------------------------------------
+// Convert log object into a string
+// -------------------------------------------------------
+function stringify(log_array) {
+ for (var i = 0; i < log_array.length; i++) {
+ log_array[i]['string'] = "";
+ log_array[i]['string'] += log_array[i].task_id + ", ";
+ log_array[i]['string'] += log_array[i].unit_id + ", ";
+ log_array[i]['string'] += log_array[i].assignment_id + ", ";
+ log_array[i]['string'] += log_array[i].session_id + ", ";
+ log_array[i]['string'] += log_array[i].dt_start + ", ";
+ log_array[i]['string'] += log_array[i].dt_end + ", ";
+ log_array[i]['string'] += log_array[i].keyboard + ", ";
+ log_array[i]['string'] += log_array[i].mouse + ", ";
+ log_array[i]['string'] += log_array[i].scroll + ", ";
+ log_array[i]['string'] += log_array[i].scroll_top + ", ";
+ log_array[i]['string'] += log_array[i].text_selected;
+ }
+ return log_array;
+}
+
+function endify(log_array) {
+ var Logs = log_array;
+ for (var i = 0; i < Logs.length; i++) {
+ if (Logs[i].status != 'closed') {
+ if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) {
+ Logs[i]['dt_end'] = Logs[i + 1]['dt_start'];
+ }
+ }
+ if (Logs[i]['dt_end'] == undefined) {
+ Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000;
+ }
+
+ }
+ return Logs;
+}
+// -------------------------------------------------------
+// Collect data for a given Job and make a plain Array of log objects
+// -------------------------------------------------------
+requestify.get(firebase_target_url, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var job_logs = response.getBody();
+ var units = job_logs.units
+ for (var unit_id in units) {
+ var assignments = units[unit_id]['assignments'];
+ for (var assignment_id in assignments) {
+ var sessions = assignments[assignment_id]['sessions'];
+ for (var session_id in sessions) {
+ var logs = sessions[session_id]['page_activity'];
+ //console.log(logs);
+ for (var log_id in logs) {
+ var log_record = {
+ task_id: task_id,
+ unit_id: unit_id,
+ assignment_id: assignment_id,
+ session_id: session_id,
+ dt_start: logs[log_id].dt,
+ keyboard: logs[log_id].keyboard,
+ mouse: logs[log_id].mouse,
+ scroll: logs[log_id].scroll,
+ scroll_top: Math.round(logs[log_id].scroll_top),
+ text_selected: logs[log_id].text_selected
+ };
+ LOGS.push(log_record);
+ }
+ }
+ }
+ }
+ LOGS = endify(LOGS);
+ LOGS = stringify(LOGS);
+ console.log(LOGS.length);
+ var all_text = "";
+ for (var i = 0; i < LOGS.length; i++) {
+ all_text+=LOGS[i]['string'] + '\n';
+ }
+ fs.appendFile(filename, all_text, function(err) {
+ if (err)
+ console.log(err);
+ });
+ });
diff --git a/generators/tab_visibility_to_csv.js b/generators/tab_visibility_to_csv.js
new file mode 100644
index 0000000..0fa352d
--- /dev/null
+++ b/generators/tab_visibility_to_csv.js
@@ -0,0 +1,133 @@
+var requestify = require('requestify');
+var fs = require('fs');
+
+
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var task_id = process.argv[2];
+var firebase_target_url = firebase_base_url + task_id + ".json";
+var folder = "../logs/"+task_id+"/";
+var filename = folder + task_id + "_tabs.csv";
+
+fs.createWriteStream(filename);
+fs.truncate(filename, 0, function() {
+ console.log('file ' + filename + ' was cleaned up.')
+});
+fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start,dt_end, status\n', function(err) {});
+
+var LOGS = [];
+
+var MaxUnit = 0;
+
+function dynamicSort(property) {
+ var sortOrder = 1;
+ if(property[0] === "-") {
+ sortOrder = -1;
+ property = property.substr(1);
+ }
+ return function (a,b) {
+ var result = (a[property] < b[property]) ? -1 : (a[property] > b[property]) ? 1 : 0;
+ return result * sortOrder;
+ }
+}
+function dynamicSortMultiple() {
+ /*
+ * save the arguments object as it will be overwritten
+ * note that arguments object is an array-like object
+ * consisting of the names of the properties to sort by
+ */
+ var props = arguments;
+ return function (obj1, obj2) {
+ var i = 0, result = 0, numberOfProperties = props.length;
+ /* try getting a different result from 0 (equal)
+ * as long as we have extra properties to compare
+ */
+ while(result === 0 && i < numberOfProperties) {
+ result = dynamicSort(props[i])(obj1, obj2);
+ i++;
+ }
+ return result;
+ }
+}
+// -------------------------------------------------------
+// Convert log object into a string
+// -------------------------------------------------------
+function stringify(log_array) {
+ for (var i = 0; i < log_array.length; i++) {
+ log_array[i]['string'] = "";
+ log_array[i]['string'] += log_array[i].task_id + ", ";
+ log_array[i]['string'] += log_array[i].unit_id + ", ";
+ log_array[i]['string'] += log_array[i].assignment_id + ", ";
+ log_array[i]['string'] += log_array[i].session_id + ", ";
+ log_array[i]['string'] += log_array[i].dt_start + ", ";
+ log_array[i]['string'] += log_array[i].dt_end + ", ";
+ log_array[i]['string'] += log_array[i].status;
+ }
+ return log_array;
+}
+
+function endify(log_array) {
+ var Logs = log_array;
+ for (var i = 0; i < Logs.length; i++) {
+ if (Logs[i].status != 'closed') {
+ if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) {
+ Logs[i]['dt_end'] = Logs[i + 1]['dt_start'];
+ }
+ }
+ if (Logs[i]['dt_end'] == undefined) {
+ Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000;
+ }
+
+ }
+ return Logs;
+}
+// -------------------------------------------------------
+// Collect data for a given Job and make a plain Array of log objects
+// -------------------------------------------------------
+requestify.get(firebase_target_url, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var job_logs = response.getBody();
+ var units = job_logs.units
+ for (var unit_id in units) {
+ var assignments = units[unit_id]['assignments'];
+ for (var assignment_id in assignments) {
+ var sessions = assignments[assignment_id]['sessions'];
+ for (var session_id in sessions) {
+ var logs = sessions[session_id]['tab_visibilty'];
+ // console.log(logs);
+ for (var log_id in logs) {
+ var log_record = {
+ task_id: task_id,
+ unit_id: parseInt(unit_id),
+ assignment_id: assignment_id,
+ session_id: parseInt(session_id),
+ dt_start: parseInt((logs[log_id].status != "closed")?logs[log_id].dt:logs[log_id].dt+500),
+ status: logs[log_id].status
+ };
+ LOGS.push(log_record);
+ }
+ }
+ }
+ }
+ LOGS.sort(dynamicSortMultiple("session_id","dt_start"));
+ //LOGS.sort(dynamicSort("assignment_id"));
+ //LOGS.sort(dynamicSort("unit_id"));
+
+ LOGS = endify(LOGS);
+ LOGS = stringify(LOGS);
+ console.log(LOGS.length);
+ var all_text = "";
+ for (var i = 0; i < LOGS.length; i++) {
+ all_text+=LOGS[i]['string'] + '\n';
+ }
+ fs.appendFile(filename, all_text, function(err) {
+ if (err)
+ console.log(err);
+ });
+ console.log(firebase_target_url);
+ //console.log(firebase_target_url);
+
+ });
diff --git a/logger.js b/logger.js
index 063abc2..79650e1 100644
--- a/logger.js
+++ b/logger.js
@@ -1,8 +1,194 @@
-var firebase_reference = new Firebase("https://crowdworker-logger.firebaseio.com/trials");
-
-firebase_reference.push({
- pathname: document.location.pathname,
- search: document.location.search,
- hostname: document.location.hostname,
- hash: document.location.hash
-});
+var EDA_LOGGER = EDA_LOGGER || (function() {
+ var settings = {
+ session: Math.floor(Math.random() * 1000000) + 1,
+ activity_interval: 2000,
+ debug: true,
+ firebase: {
+ bucket: "crowdworker-logger"
+ }
+ }
+ // A log function which can be easily turned of using debug variable
+ var log = function(message) {
+ if (settings.debug) console.log("[EDA_LOGGER] LOG: " + message);
+ };
+ function fullPath(el){
+ var names = [];
+ while (el.parentNode){
+ if (el.id){
+ names.unshift('#'+el.id);
+ break;
+ }else{
+ if (el==el.ownerDocument.documentElement)
+ names.unshift(el.tagName);
+ else{
+ for (var c=1,e=el;e.previousElementSibling;e=e.previousElementSibling,c++);
+ names.unshift(el.tagName+":nth-child("+c+")");
+ }
+ el=el.parentNode;
+ }
+ }
+ return names.join(" > ");
+ }
+ function isPageHidden() {
+ return document.hidden || document.msHidden || document.webkitHidden || document.mozHidden;
+ }
+ function getSelectedText() {
+ var text = "";
+ if (typeof window.getSelection != "undefined") {
+ text = window.getSelection().toString();
+ } else if (typeof document.selection != "undefined" && document.selection.type == "Text") {
+ text = document.selection.createRange().text;
+ }
+ return text;
+ }
+ var _args = {
+ key_name: "test_name",
+ key_value: "test_value"
+ }; // private
+ return {
+ init: function(Args) {
+ _args = Args;
+ _args['task_id'] = parseInt(document.getElementById("assignment-job-id").innerHTML);
+ _args['worker_id'] = parseInt(document.getElementById("assignment-worker-id").innerHTML);
+ // Include Firebase library
+ var logger = this;
+ logger.init_firebase(function() {
+ logger.init_events_capturing();
+ logger.init_activity_capturing();
+ });
+ },
+ init_firebase: function(callback) {
+ var firebase_script = document.createElement('script');
+ firebase_script.src = "https://cdn.firebase.com/js/client/2.2.9/firebase.js";
+
+ log(firebase_script.src);
+
+ document.getElementsByTagName('head')[0].appendChild(firebase_script);
+ var logger = this;
+ firebase_script.onload = function() {
+ // get the assignment code from the url
+ var assignment_code = document.location.pathname.substring(document.location.pathname.lastIndexOf("/"), document.location.pathname.length);
+ log(assignment_code);
+ // get the platform code from the url
+ var platform_code = document.location.hostname.replace(/\./g, '');
+ // form the firebase endpoint url
+ var firebase_endpoint_url = "https://" + settings.firebase.bucket + ".firebaseio.com/" + platform_code + "/" + _args["task_id"] + "/units/" + _args['key_value'] + "/assignments" + assignment_code;
+ log(firebase_endpoint_url);
+ _args["firebase_assignment"] = new Firebase(firebase_endpoint_url);
+ /*_args["firebase_assignment"].update({
+ key_name: _args.key_name,
+ key_value: _args.key_value,
+ unit_id: _args.key_value
+ });*/
+ _args["firebase_logs"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/tab_visibilty");
+ _args["firebase_activity"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/page_activity");
+ _args["firebase_keys"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/key_pressed");
+ _args["firebase_clicks"] = _args["firebase_assignment"].child('sessions/' + settings.session + "/clicks");
+
+ callback();
+ };
+ },
+ init_activity_capturing: function() {
+ var logger = this;
+ var activity_statuses = {
+ "keyboard": 0,
+ "mouse": 0,
+ "scroll": 0,
+ "scroll_top":0,
+ "text_selected":0
+ };
+ setInterval(function() {
+ logger.log_event(_args["firebase_activity"], (function(a) {
+ activity_statuses = {
+ "keyboard": 0,
+ "mouse": 0,
+ "scroll": 0,
+ "scroll_top":0,
+ "text_selected":0
+ };
+ return a;
+ }(activity_statuses)));
+ }, settings.activity_interval);
+
+ document.onkeydown = function(evt) {
+ activity_statuses["keyboard"] = 1;
+ };
+ document.onmouseup = function(evt){
+ if (getSelectedText() != "")
+ activity_statuses["text_selected"] = 1;
+ }
+ document.onmousemove = function(evt) {
+ activity_statuses["mouse"] = 1;
+ };
+
+ window.onscroll = function(evt) {
+ activity_statuses["scroll"] = 1;
+ activity_statuses["scroll_top"] = document.body.scrollTop;
+ };
+
+ document.onkeydown = function(evt) {
+ var key = evt.keyCode || evt.charCode;
+ logger.log_event(_args["firebase_keys"],{"key":key});
+ };
+ document.onmousedown = function(evt) {
+ var element_path = fullPath(evt.path[0]);
+ logger.log_event(_args["firebase_clicks"],{"element":element_path});
+ };
+ },
+ init_events_capturing: function() {
+ var logger = this;
+ // Log the task was opened by the worker
+ logger.log_event(_args["firebase_logs"], {
+ status: "opened"
+ });
+ // Log the task page was closed by the worker
+ window.onbeforeunload = function() {
+ logger.log_event(_args["firebase_logs"], {
+ status: "closed"
+ });
+ };
+ logger.init_visibility_changes();
+ },
+ init_visibility_changes: function() {
+ var hidden, visibilityChange;
+ if (typeof document.hidden !== "undefined") { // Opera 12.10 and Firefox 18 and later support
+ hidden = "hidden";
+ visibilityChange = "visibilitychange";
+ } else if (typeof document.mozHidden !== "undefined") {
+ hidden = "mozHidden";
+ visibilityChange = "mozvisibilitychange";
+ } else if (typeof document.msHidden !== "undefined") {
+ hidden = "msHidden";
+ visibilityChange = "msvisibilitychange";
+ } else if (typeof document.webkitHidden !== "undefined") {
+ hidden = "webkitHidden";
+ visibilityChange = "webkitvisibilitychange";
+ }
+ var logger = this;
+
+ function handleVisibilityChange() {
+ if (document[hidden]) {
+ logger.log_event(_args["firebase_logs"], {
+ status: "hidden"
+ });
+ } else {
+ logger.log_event(_args["firebase_logs"], {
+ status: "active"
+ });
+ }
+ }
+ // Warn if the browser doesn't support addEventListener or the Page Visibility API
+ if (typeof document.addEventListener === "undefined" ||
+ typeof document[hidden] === "undefined") {
+ //alert("This demo requires a browser, such as Google Chrome or Firefox, that supports the Page Visibility API.");
+ } else {
+ // Handle page visibility change
+ document.addEventListener(visibilityChange, handleVisibilityChange, false);
+ }
+ },
+ log_event: function(firebase_reference, data) {
+ data['dt'] = Firebase.ServerValue.TIMESTAMP;
+ firebase_reference.push(data);
+ }
+ };
+}());
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..8579f97
--- /dev/null
+++ b/package.json
@@ -0,0 +1,20 @@
+{
+ "name": "Relauncher-Logger-ReferenceMaker",
+ "version": "0.0.1",
+ "description": "",
+ "main": "unit_references_maker.js",
+ "engines": {
+ "node": "0.10.29"
+ },
+ "scripts": {
+ "start": "node unit_references_maker.js"
+ },
+ "repository": {
+ "type": "git",
+ "url": "https://github.com/ReLauncher/worker-activity-logger.git"
+ },
+ "dependencies": {
+ "firebase":"2.3.1",
+ "requestify":"0.1.17"
+ }
+}
\ No newline at end of file
diff --git a/procfile b/procfile
new file mode 100644
index 0000000..104c7e4
--- /dev/null
+++ b/procfile
@@ -0,0 +1 @@
+web: node unit_references_maker.js
\ No newline at end of file
diff --git a/queue_management.js b/queue_management.js
new file mode 100644
index 0000000..d120d72
--- /dev/null
+++ b/queue_management.js
@@ -0,0 +1,147 @@
+var Firebase = require('firebase');
+var requestify = require('requestify');
+
+
+var JOB_TO_BE_PROCESSED = 850870;
+var DELAY_QUEUE_TIME = 30000;
+
+
+var Crowdflower = {
+ base: "http://api.crowdflower.com/v1/",
+ api_key: process.env.CROWDFLOWER_API_KEY,
+ getUnit: function(job_id, unit_id, callback) {
+ var cf = Crowdflower;
+ var crowdflower_unit_url = cf.base + "/jobs/" + job_id + "/units/" + unit_id + ".json?key=" + cf.api_key;
+ console.log(crowdflower_unit_url);
+ requestify.get(crowdflower_unit_url, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var crowdflower_unit = response.getBody();
+ callback(crowdflower_unit);
+ });
+ }
+}
+var Firebase_urls = {
+ base: 'https://crowdworker-logger.firebaseio.com/',
+ crowdflower: "taskscrowdflowercom/",
+ queue: function() {
+ var fb = Firebase_urls;
+ return fb.base + "queue/";
+ },
+ tasks: function() {
+ var fb = Firebase_urls;
+ var url = fb.base + fb.crowdflower;
+ return url;
+ },
+ assignments: function(job_id) {
+ var fb = Firebase_urls;
+ var url = fb.base + fb.crowdflower + job_id + "/";
+ return url;
+ }
+}
+
+function getLastLog(logs) {
+ if (logs) {
+ var last_log_id = Object.keys(logs)[Object.keys(logs).length - 1];
+ var last_log = logs[last_log_id];
+ return last_log;
+ }
+ return false;
+}
+
+
+function addToQueue(job_id, unit_id, assignment_id) {
+ console.log(DELAY_QUEUE_TIME / 1000 + " seconds passed");
+ var assignment_ref = new Firebase(Firebase_urls.queue());
+ assignment_ref.push({
+ suspect: "abandoned",
+ job_id: job_id,
+ unit_id: unit_id,
+ assignment_id: assignment_id,
+ dt: Firebase.ServerValue.TIMESTAMP
+ });
+ console.log("event is added to the queue");
+}
+
+function relaunchUnit(job_id, unit_id) {
+ requestify.post("http://localhost:3000/jobs/" + job_id + "/units/" + unit_id + "/relaunch", {
+ api_key: Crowdflower.api_key
+ }, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var resp = response.getBody();
+ console.log(resp);
+ });
+}
+
+console.log("====================\nStart Logs Monitoring\n====================");
+
+var logs_ref = new Firebase(Firebase_urls.assignments(JOB_TO_BE_PROCESSED));
+logs_ref.on('child_changed', function(assignment_snapshot, prevchildname) {
+ var job_id = JOB_TO_BE_PROCESSED;
+
+ var assignment_details = assignment_snapshot.val();
+ var assignment_id = assignment_snapshot.key();
+
+ var unit_id = assignment_details['unit_id'];
+ var logs = assignment_details['logs'];
+
+ if (unit_id) {
+ last_log = getLastLog(logs);
+ if (last_log && last_log.status == "closed") {
+ console.log(unit_id, assignment_id, assignment_details["status"]);
+ //add the unit to queue to be processed
+ setTimeout(function() {
+ addToQueue(job_id, unit_id, assignment_id);
+ }, DELAY_QUEUE_TIME);
+ }
+ }
+});
+
+
+
+
+
+console.log("====================\nStart Queue Monitoring\n====================");
+var ref_queue = new Firebase(Firebase_urls.queue());
+ref_queue.on('child_added', function(queue_element_snapshot, prevchildname) {
+ console.log("processing a queue element");
+
+ var element = queue_element_snapshot.val();
+ var unit_id = element['unit_id'];
+ var job_id = element['job_id'];
+ var assignment_id = element['assignment_id'];
+ var assignment_ref = new Firebase(Firebase_urls.assignments(job_id) + assignment_id + '/logs/');
+
+ assignment_ref.orderByChild("dt").limitToLast(1).once("value", function(log_snapshot) {
+ var logs = log_snapshot.val();
+ var last_log = getLastLog(logs);
+ console.log(last_log);
+ // var logs = assignment['logs'];
+ // getLastLog(logs, function(last_log) {
+ if (last_log.status == "closed") {
+ console.log("check unit state at CrowdFlower");
+ setTimeout(function() {
+ console.log(DELAY_QUEUE_TIME/1000+" seconds passed");
+ Crowdflower.getUnit(job_id, unit_id, function(cf_unit) {
+ console.log(cf_unit['state']);
+ if (cf_unit['state'] == "judging") {
+ console.log("to relaunch the unit");
+ relaunchUnit(job_id, unit_id);
+ }
+ });
+ }, DELAY_QUEUE_TIME);
+
+ }
+ });
+ // remove element from queue, because it is checked;
+ var queue_element_to_be_deleted = new Firebase(Firebase_urls.queue() + queue_element_snapshot.key());
+ queue_element_to_be_deleted.set(null);
+ console.log("element " + queue_element_snapshot.key() + " is removed");
+});
diff --git a/test.js b/test.js
new file mode 100644
index 0000000..f860259
--- /dev/null
+++ b/test.js
@@ -0,0 +1,3 @@
+process.argv.forEach(function (val, index, array) {
+ console.log(index + ': ' + val);
+});
\ No newline at end of file
diff --git a/unit_ids_maker.js b/unit_ids_maker.js
new file mode 100644
index 0000000..648eecd
--- /dev/null
+++ b/unit_ids_maker.js
@@ -0,0 +1,54 @@
+var Firebase = require('firebase');
+var requestify = require('requestify');
+//var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+//var Ref = new Firebase(firebase_base_url);
+var CROWDFLOWER_API_KEY = process.env.CROWDFLOWER_API_KEY;
+var JOB_TO_BE_PROCESSED = 855249;
+
+var Crowdflower = {
+ base: "http://api.crowdflower.com/v1/",
+ api_key: process.env.CROWDFLOWER_API_KEY,
+ getUnits: function(job_id, callback) {
+ var cf = Crowdflower;
+ var crowdflower_units_url = cf.base + "/jobs/" + job_id + "/units.json?key=" + cf.api_key;
+ console.log(crowdflower_units_url);
+ requestify.get(crowdflower_units_url, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var units = response.getBody();
+ if (callback)
+ callback(units);
+ });
+ },
+ updateUnit: function(job_id, unit_id, data, callback) {
+ var cf = Crowdflower;
+ var crowdflower_unit_url = cf.base + "/jobs/" + job_id + "/units/" + unit_id + ".json?key=" + cf.api_key;
+
+ var put_data = {
+ unit: {
+ data: data
+ }
+ };
+ console.log(put_data);
+ requestify.put(crowdflower_unit_url, put_data)
+ .then(function(crowdflower_resp) {
+ var unit_info = crowdflower_resp.getBody();
+ if (callback)
+ callback(unit_info);
+ });
+ }
+}
+
+Crowdflower.getUnits(JOB_TO_BE_PROCESSED, function(units) {
+ for (var unit_id in units) {
+ var unit_data = units[unit_id];
+ unit_data['re_unit_id'] = unit_id;
+ //console.log(unit_data);
+ Crowdflower.updateUnit(JOB_TO_BE_PROCESSED, unit_id, unit_data, function(updated_unit) {
+ console.log(updated_unit);
+ });
+ }
+})
diff --git a/unit_references_maker.js b/unit_references_maker.js
new file mode 100644
index 0000000..6802f95
--- /dev/null
+++ b/unit_references_maker.js
@@ -0,0 +1,45 @@
+var Firebase = require('firebase');
+var requestify = require('requestify');
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var Ref = new Firebase(firebase_base_url);
+var CROWDFLOWER_API_KEY = process.env.CROWDFLOWER_API_KEY;
+
+Ref.on('child_changed', function(task_snapshot, prevchildname) {
+ var task_id = task_snapshot.key();
+ console.log(task_id);
+ var task_reference = Ref.child(task_id);
+
+ task_reference.on('child_added', function(assignment_snapshot, prevchildname) {
+ var assignment_details = assignment_snapshot.val();
+ var unit_id = assignment_details['unit_id'];
+ if (unit_id == undefined) {
+ var key_name = assignment_details['key_name'];
+ var unit_key = {
+ 'name': assignment_details['key_name'],
+ 'value': assignment_details['key_value']
+ };
+ console.log(unit_key);
+ var units_url = "http://api.crowdflower.com/v1/jobs/" + task_id + "/units/?key=" + CROWDFLOWER_API_KEY;
+ console.log(units_url);
+
+ requestify.get(units_url, {
+ headers: {
+ "Accept": "application/json"
+ }
+ })
+ .then(function(response) {
+ var units = response.getBody();
+ //console.log(units);
+ for (var key in units) {
+ if (units[key][unit_key["name"]] == unit_key["value"]) {
+ task_reference.child(assignment_snapshot.key()).update({
+ unit_id: key
+ });
+ }
+ // do something with key
+ }
+ });
+
+ }
+ });
+});