6915 - Telemetry daily frequency

scripts/get_users_meta_docs.js

@@ -1,3 +1,5 @@
+#!/usr/bin/env node
+
 const inquirer = require('inquirer');
 const PouchDB = require('pouchdb-core');
 const fs = require('fs');
@@ -86,7 +88,7 @@ const actionQuestions = [{
           }
           docs.forEach(doc => console.log(JSON.stringify(doc, null, 2) + ','));
         } else if (i === 0) {
-          console.log('\x1b[31m%s\x1b[0m', `There are no documents of type ${type}`);
+          console.error('\x1b[31m%s\x1b[0m', `There are no documents of type ${type}`);
           break;
         } else {
           console.log('{}]');
@@ -100,7 +102,7 @@ const actionQuestions = [{
       let docIndex = 0;
 
       if (docs.length === 0) {
-        console.log('\x1b[31m%s\x1b[0m', `There are no documents of type ${type}`);
+        console.error('\x1b[31m%s\x1b[0m', `There are no documents of type ${type}`);
       } else {
         console.log(JSON.stringify(docs[docIndex], null, 2));
 
@@ -125,7 +127,7 @@ const actionQuestions = [{
 
             console.log(JSON.stringify(docs[docIndex], null, 2));
             if (printMessage) {
-              console.log('\x1b[31m%s\x1b[0m', `No next document. This is the last one.`);
+              console.error('\x1b[31m%s\x1b[0m', `No next document. This is the last one.`);
             }
           } else if (response.action === 'save_current') {
             const filePath = path.join(path.resolve(__dirname), docs[docIndex]._id + '.json');
@@ -154,6 +156,6 @@ const actionQuestions = [{
       }
     }
   } catch(err) {
-    console.log(err);
+    console.error(err);
   }
 })();

webapp/src/ts/services/telemetry.service.ts

@@ -9,13 +9,13 @@ import { SessionService } from '@mm-services/session.service';
   providedIn: 'root'
 })
 /**
- * TelemetryService: Records, aggregates, and submits telemetry data
+ * TelemetryService: Records, aggregates, and submits telemetry data.
  */
 export class TelemetryService {
   // Intentionally scoped to the whole browser (for this domain). We can then tell if multiple users use the same device
   private readonly DEVICE_ID_KEY = 'medic-telemetry-device-id';
   private DB_ID_KEY;
-  private LAST_AGGREGATED_DATE_KEY;
+  private FIRST_AGGREGATED_DATE_KEY;
 
   private queue = Promise.resolve();
 
@@ -27,7 +27,7 @@ export class TelemetryService {
     // Intentionally scoped to the specific user, as they may perform a
     // different role (online vs. offline being being the most obvious) with different performance implications
     this.DB_ID_KEY = ['medic', this.sessionService.userCtx().name, 'telemetry-db'].join('-');
-    this.LAST_AGGREGATED_DATE_KEY = ['medic', this.sessionService.userCtx().name, 'telemetry-date'].join('-');
+    this.FIRST_AGGREGATED_DATE_KEY = ['medic', this.sessionService.userCtx().name, 'telemetry-date'].join('-');
   }
 
   private getDb() {
@@ -54,15 +54,21 @@ export class TelemetryService {
     return uniqueDeviceId;
   }
 
-  private getLastAggregatedDate() {
-    let date = parseInt(window.localStorage.getItem(this.LAST_AGGREGATED_DATE_KEY));
+  /**
+   * Returns a Moment object when the first telemetry record was created.
+   *
+   * This date is computed and stored in milliseconds (since Unix epoch)
+   * when we call this method for the first time and after every aggregation.
+   */
+  private getFirstAggregatedDate() {
+    let date = parseInt(window.localStorage.getItem(this.FIRST_AGGREGATED_DATE_KEY));
 
     if (!date) {
       date = Date.now();
-      window.localStorage.setItem(this.LAST_AGGREGATED_DATE_KEY, date.toString());
+      window.localStorage.setItem(this.FIRST_AGGREGATED_DATE_KEY, date.toString());
     }
 
-    return date;
+    return moment(date);
   }
 
   private storeIt(db, key, value) {
@@ -73,11 +79,17 @@ export class TelemetryService {
     });
   }
 
+  // moment when the aggregation starts (the beginning of the current day)
+  private aggregateStartsAt() {
+    return moment().startOf('day');
+  }
+
+  // if there is telemetry data from previous days, aggregation is performed and the data destroyed
   private submitIfNeeded(db) {
-    const monthStart = moment().startOf('month');
-    const dbDate = moment(this.getLastAggregatedDate());
+    const startOf = this.aggregateStartsAt();
+    const dbDate = this.getFirstAggregatedDate();
 
-    if (dbDate.isBefore(monthStart)) {
+    if (dbDate.isBefore(startOf)) {
       return this
         .aggregate(db)
         .then(() => this.reset(db));
@@ -97,6 +109,7 @@ export class TelemetryService {
       'telemetry',
       metadata.year,
       metadata.month,
+      metadata.day,
       metadata.user,
       metadata.deviceId,
     ].join('-');
@@ -109,7 +122,7 @@ export class TelemetryService {
         this.dbService.get().query('medic-client/doc_by_type', { key: ['form'], include_docs: true })
       ])
       .then(([ddoc, formResults]) => {
-        const date = moment(this.getLastAggregatedDate());
+        const date = this.getFirstAggregatedDate();
         const version = (ddoc.deploy_info && ddoc.deploy_info.version) || 'unknown';
         const forms = formResults.rows.reduce((keyToVersion, row) => {
           keyToVersion[row.doc.internalId] = row.doc._rev;
@@ -120,6 +133,7 @@ export class TelemetryService {
         return {
           year: date.year(),
           month: date.month() + 1,
+          day: date.date(),
           user: this.sessionService.userCtx().name,
           deviceId: this.getUniqueDeviceId(),
           versions: {
@@ -204,7 +218,7 @@ export class TelemetryService {
 
   private reset(db) {
     window.localStorage.removeItem(this.DB_ID_KEY);
-    window.localStorage.removeItem(this.LAST_AGGREGATED_DATE_KEY);
+    window.localStorage.removeItem(this.FIRST_AGGREGATED_DATE_KEY);
 
     return db.destroy();
   }
@@ -225,7 +239,7 @@ export class TelemetryService {
    *   metric_b:  { sum: -16, min: -4, max: -4, count: 4, sumsqr: 64 }
    * }
    *
-   * See: https://wiki.apache.org/couchdb/Built-In_Reduce_Functions#A_stats
+   * See: https://docs.couchdb.org/en/stable/ddocs/ddocs.html#_stats
    *
    * This single month aggregate document is of type 'telemetry', and is
    * stored in the user's meta DB (which replicates up to the main server)
@@ -255,14 +269,14 @@ export class TelemetryService {
     let db;
     this.queue = this.queue
       .then(() => db = this.getDb())
-      .then(() => this.storeIt(db, key, value))
       .then(() => this.submitIfNeeded(db))
+      .then(() => db = this.getDb())  // db is fetched again in case submitIfNeeded dropped the old reference
+      .then(() => this.storeIt(db, key, value))
       .catch(err => console.error('Error in telemetry service', err))
       .finally(() => {
         if (!db || db._destroyed || db._closed) {
           return;
         }
-
         try {
           db.close();
         } catch (err) {