ReLauncher · pavelk2 · Oct 5, 2015 · Oct 5, 2015 · Oct 5, 2015 · Oct 5, 2015
diff --git a/README.md b/README.md
@@ -1 +1,39 @@
-# worker-activity-logger
+# ReLauncher | Worker Activity Logger
+
+#### Setup worker browser logs collection for your tasks in 5 steps:
+
+1. fork this repository,
+2. create a bucket in [Firebase](https://www.firebase.com/) (it is free),
+3. insert your firebase bucket name in [logger.js](https://github.com/ReLauncher/worker-activity-logger/blob/gh-pages/logger.js#L17):
+
+  ```javascript
+  var settings = {
+            debug: true,
+            firebase: {
+                bucket: "YOUR_FIREBASE_BUCKET_NAME"
+            }
+        }
+  ```
+4. [create a job](https://success.crowdflower.com/hc/en-us/articles/204056975-Getting-Started-on-CrowdFlower-An-Overview-to-Building-a-Job) in CrowdFlower, upload your data. Make sure you have a column, which can be used as an identificator of a given data row (unit). If you collect tags for images, it can be *image_url*
+5. add the following in [CML](https://success.crowdflower.com/hc/en-us/articles/202817989-CML-CrowdFlower-Markup-Language-Overview) of your CrowdFlower job: 
+
+  ```html
+  <script src="https://YOUR_GITHUB_USERNAME.github.io/worker-activity-logger/logger.js"></script>
+  <script>
+  EDA_LOGGER.init({
+  key_name:"KEY_COLUMN_NAME",
+  key_value:"{{KEY_COLUMN_NAME}}",
+  task_id: YOUR_CROWDFLOWER_JOB_ID
+  });
+  </script>
+  ```
+
+When you launch the job you should see logs appearing in your firebase bucket. If you do not - check all the steps carefully. If it still does not work, create an *issue* in this repository.
+
+- [ ] run unit_references_maker.js (once)
+- [ ] run queue_management.js (keep it running during execution)
+- [ ] run ReLauncher server (npm start)
+- [ ] launch the job
+- [ ] start ReLauncher for this job
+
+[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy)
diff --git a/app.json b/app.json
@@ -0,0 +1,17 @@
+{
+    "name": "Relauncher ReferenceMaker",
+    "description": "Aapplication listening to a given Firebase bucket and fixing unit references.",
+    "repository": "https://github.com/ReLauncher/worker-activity-logger",
+    "logo": "https://avatars3.githubusercontent.com/u/14817691?v=3&s=200",
+    "keywords": ["relauncher", "crowdflower", "crowdsourcing", "microtasks"],
+    "env": {
+        "HEROKU_URL": {
+            "description": "The URL of this Heroku app.",
+            "value": "http://YOURAPPNAME.herokuapp.com"
+        },
+        "CROWDFLOWER_API_KEY":{
+            "description" : "your API key from https://make.crowdflower.com/account/user",
+            "value":""
+        }
+    }
+}
diff --git a/index.html → example.html b/index.html → example.html
@@ -1,8 +1,16 @@
 <html>
+
 <head>
-<script src="https://cdn.firebase.com/js/client/2.2.9/firebase.js"></script>
-<script src="logger.js"></script>
+    <script src="logger.js"></script>
+    <script>
+    EDA_LOGGER.init({
+        key_name: "image_url",
+        key_value: "http://kucherbaev.com/123123",
+        task_id:32452345
+    });
+    </script>
 </head>
+
 <body>
     <div>
         <div class="hero">
@@ -18,4 +26,5 @@ <h1>CrowdFlower Launcher</h1>
         </div>
     </div>
 </body>
+
 </html>
diff --git a/firebase_to_csv.js b/firebase_to_csv.js
@@ -0,0 +1,101 @@
+var requestify = require('requestify');
+var fs = require('fs');
+
+
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var task_id = 837900;
+var firebase_target_url = firebase_base_url + task_id + ".json";
+var folder = "logs/";
+var filename = folder + task_id + "_logs.csv";
+
+fs.createWriteStream(filename);
+fs.truncate(filename, 0, function() {
+    console.log('file ' + filename + ' was cleaned up.')
+});
+fs.appendFile(filename, 'unit_number, unit_id, assignment_id, dt_start, dt_end, status\n', function(err) {});
+
+var Logs = [];
+var Units = {};
+var MaxUnit = 0;
+// -------------------------------------------------------
+// Convert log object into a string
+// -------------------------------------------------------
+function stringify(log_array) {
+    for (var i = 0; i < log_array.length; i++) {
+        log_array[i]['string'] = "";
+        log_array[i]['string'] += log_array[i].unit_number + ", ";
+        log_array[i]['string'] += log_array[i].unit_id + ", ";
+        log_array[i]['string'] += log_array[i].assignment_id + ", ";
+        //log_array[i]['string'] += log_array[i].worker_id + ", ";
+        log_array[i]['string'] += log_array[i].dt_start + ", ";
+        log_array[i]['string'] += log_array[i].dt_end + ", ";
+        log_array[i]['string'] += log_array[i].status;
+    }
+    return log_array;
+}
+// -------------------------------------------------------
+// Collect data for a given Job and make a plain Array of log objects
+// -------------------------------------------------------
+requestify.get(firebase_target_url, {
+        headers: {
+            "Accept": "application/json"
+        }
+    })
+    .then(function(response) {
+        var assignments = response.getBody();
+        for (var a_id in assignments) {
+            var a = assignments[a_id];
+            if (a_id != "editor_preview" && a.unit_id) {
+                var unit_number;
+                if (Units[a.unit_id.toString()]) {
+                    unit_number = Units[a.unit_id.toString()];
+                } else {
+                    MaxUnit++;
+                    Units[a.unit_id.toString()] = MaxUnit;
+                    unit_number = Units[a.unit_id.toString()];
+                }
+
+
+
+                //console.log(a);
+                for (var l_id in a.logs) {
+                    var l = a.logs[l_id];
+                    var log_record = {
+                        'unit_number': unit_number,
+                        'unit_id': a.unit_id,
+                        'assignment_id': a_id,
+                        //'worker_id': a.worker_id || "0",
+                        'dt_start': l.dt,
+                        'status': l.status
+                    }
+                    Logs.push(log_record);
+                }
+            } else {
+                console.log("PROBLEMATIC ASSIGNMENT", a_id, a);
+            }
+        }
+        // -------------------------------------------------------
+        // Add end events to log objects
+        // -------------------------------------------------------
+        for (var i = 0; i < Logs.length; i++) {
+            if (Logs[i].status != 'closed') {
+                if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id) {
+                    Logs[i]['dt_end'] = Logs[i + 1]['dt_start'];
+                }
+            }
+            if (Logs[i]['dt_end'] == undefined) {
+                Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000;
+            }
+
+        }
+        // -------------------------------------------------------
+        // Convert each object in array into string and save it into the log file
+        // -------------------------------------------------------
+        Logs = stringify(Logs);
+        //console.log(Logs.length);
+        for (var i = 0; i < Logs.length; i++) {
+            console.log(Logs[i]['string']);
+            fs.appendFile(filename, Logs[i]['string'] + '\n', function(err) {});
+        }
+        console.log(firebase_target_url);
+    });
diff --git a/generators/clicks_to_csv.js b/generators/clicks_to_csv.js
@@ -0,0 +1,94 @@
+var requestify = require('requestify');
+var fs = require('fs');
+
+
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var task_id = process.argv[2];
+var firebase_target_url = firebase_base_url + task_id + ".json";
+var folder = "../logs/"+task_id+"/";
+var filename = folder + task_id + "_clicks.csv";
+
+fs.createWriteStream(filename);
+fs.truncate(filename, 0, function() {
+    console.log('file ' + filename + ' was cleaned up.')
+});
+fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start, element\n', function(err) {});
+
+var LOGS = [];
+
+var MaxUnit = 0;
+// -------------------------------------------------------
+// Convert log object into a string
+// -------------------------------------------------------
+function stringify(log_array) {
+    for (var i = 0; i < log_array.length; i++) {
+        log_array[i]['string'] = "";
+        log_array[i]['string'] += log_array[i].task_id + ", ";
+        log_array[i]['string'] += log_array[i].unit_id + ", ";
+        log_array[i]['string'] += log_array[i].assignment_id + ", ";
+        log_array[i]['string'] += log_array[i].session_id + ", ";
+        log_array[i]['string'] += log_array[i].dt_start + ", ";
+        log_array[i]['string'] += log_array[i].element;
+    }
+    return log_array;
+}
+
+function endify(log_array) {
+    var Logs = log_array;
+    for (var i = 0; i < Logs.length; i++) {
+        if (Logs[i].status != 'closed') {
+            if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) {
+                Logs[i]['dt_end'] = Logs[i + 1]['dt_start'];
+            }
+        }
+        if (Logs[i]['dt_end'] == undefined) {
+            Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000;
+        }
+
+    }
+    return Logs;
+}
+// -------------------------------------------------------
+// Collect data for a given Job and make a plain Array of log objects
+// -------------------------------------------------------
+requestify.get(firebase_target_url, {
+        headers: {
+            "Accept": "application/json"
+        }
+    })
+    .then(function(response) {
+        var job_logs = response.getBody();
+        var units = job_logs.units
+        for (var unit_id in units) {
+            var assignments = units[unit_id]['assignments'];
+            for (var assignment_id in assignments) {
+                var sessions = assignments[assignment_id]['sessions'];
+                for (var session_id in sessions) {
+                    var logs = sessions[session_id]['clicks'];
+                    //console.log(logs);
+                    for (var log_id in logs) {
+                        var log_record = {
+                            task_id: task_id,
+                            unit_id: unit_id,
+                            assignment_id: assignment_id,
+                            session_id: session_id,
+                            dt_start: logs[log_id].dt,
+                            element: logs[log_id].element
+                        };
+                        LOGS.push(log_record);
+                    }
+                }
+            }
+        }
+        //LOGS = endify(LOGS);
+        LOGS = stringify(LOGS);
+        console.log(LOGS.length);
+        var all_text = "";
+        for (var i = 0; i < LOGS.length; i++) {
+            all_text+=LOGS[i]['string'] + '\n';
+        }
+        fs.appendFile(filename, all_text, function(err) {
+            if (err)
+                console.log(err);
+        });
+    });
diff --git a/generators/generate_logs.sh b/generators/generate_logs.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+echo "===================================="
+echo "= FIREBASE CROWDFLOWER LOGS TO CSV ="
+echo "===================================="
+
+echo "\nNow we create a folder for $@ task"
+mkdir "../logs/$@"
+echo "\nNow we generate logs of key presses from firebase..."
+node key_presses_to_csv.js $@
+echo "\nNow we generate logs of clicks from firebase..."
+node clicks_to_csv.js $@
+echo "\nNow we generate logs of page activity from firebase..."
+node page_activity_to_csv.js $@
+echo "\nNow we generate logs of tab visibility from firebase..."
+node tab_visibility_to_csv.js $@
+
+echo "\nNow we copy the folder with logs to stats-and-graphs folder to be further processed"
+cp -r "../logs/$@" "../../stats-and-graphs/logs/$@"
+
+echo "\nDONE"
+echo "===================================="
diff --git a/generators/key_presses_to_csv.js b/generators/key_presses_to_csv.js
@@ -0,0 +1,94 @@
+var requestify = require('requestify');
+var fs = require('fs');
+
+
+var firebase_base_url = 'https://crowdworker-logger.firebaseio.com/' + "taskscrowdflowercom" + '/';
+var task_id = process.argv[2];
+var firebase_target_url = firebase_base_url + task_id + ".json";
+var folder = "../logs/"+task_id+"/";
+var filename = folder + task_id + "_keys.csv";
+
+fs.createWriteStream(filename);
+fs.truncate(filename, 0, function() {
+    console.log('file ' + filename + ' was cleaned up.')
+});
+fs.appendFile(filename, 'task_id, unit_id, assignment_id, session_id, dt_start, key\n', function(err) {});
+
+var LOGS = [];
+
+var MaxUnit = 0;
+// -------------------------------------------------------
+// Convert log object into a string
+// -------------------------------------------------------
+function stringify(log_array) {
+    for (var i = 0; i < log_array.length; i++) {
+        log_array[i]['string'] = "";
+        log_array[i]['string'] += log_array[i].task_id + ", ";
+        log_array[i]['string'] += log_array[i].unit_id + ", ";
+        log_array[i]['string'] += log_array[i].assignment_id + ", ";
+        log_array[i]['string'] += log_array[i].session_id + ", ";
+        log_array[i]['string'] += log_array[i].dt_start + ", ";
+        log_array[i]['string'] += log_array[i].key;
+    }
+    return log_array;
+}
+
+function endify(log_array) {
+    var Logs = log_array;
+    for (var i = 0; i < Logs.length; i++) {
+        if (Logs[i].status != 'closed') {
+            if (i < (Logs.length - 1) && Logs[i].assignment_id == Logs[i + 1].assignment_id && Logs[i].session_id == Logs[i + 1].session_id) {
+                Logs[i]['dt_end'] = Logs[i + 1]['dt_start'];
+            }
+        }
+        if (Logs[i]['dt_end'] == undefined) {
+            Logs[i]['dt_end'] = Logs[i]['dt_start'] + 1000;
+        }
+
+    }
+    return Logs;
+}
+// -------------------------------------------------------
+// Collect data for a given Job and make a plain Array of log objects
+// -------------------------------------------------------
+requestify.get(firebase_target_url, {
+        headers: {
+            "Accept": "application/json"
+        }
+    })
+    .then(function(response) {
+        var job_logs = response.getBody();
+        var units = job_logs.units
+        for (var unit_id in units) {
+            var assignments = units[unit_id]['assignments'];
+            for (var assignment_id in assignments) {
+                var sessions = assignments[assignment_id]['sessions'];
+                for (var session_id in sessions) {
+                    var logs = sessions[session_id]['key_pressed'];
+                    //console.log(logs);
+                    for (var log_id in logs) {
+                        var log_record = {
+                            task_id: task_id,
+                            unit_id: unit_id,
+                            assignment_id: assignment_id,
+                            session_id: session_id,
+                            dt_start: logs[log_id].dt,
+                            key: logs[log_id].key
+                        };
+                        LOGS.push(log_record);
+                    }
+                }
+            }
+        }
+        //LOGS = endify(LOGS);
+        LOGS = stringify(LOGS);
+        console.log(LOGS.length);
+        var all_text = "";
+        for (var i = 0; i < LOGS.length; i++) {
+            all_text+=LOGS[i]['string'] + '\n';
+        }
+        fs.appendFile(filename, all_text, function(err) {
+            if (err)
+                console.log(err);
+        });
+    });