diff --git a/tests/e2e/default/telemetry/telemetry.wdio-spec.js b/tests/e2e/default/telemetry/telemetry.wdio-spec.js index 32cb9f63a22..277c5d0b7ab 100644 --- a/tests/e2e/default/telemetry/telemetry.wdio-spec.js +++ b/tests/e2e/default/telemetry/telemetry.wdio-spec.js @@ -32,15 +32,9 @@ const setupUser = () => { }; }; -const setOldTelemetryDate = async (user, date) => { - const telemetryDateStorageKey = `medic-${user.username}-telemetry-date`; - await browser.execute((telemetryDateStorageKey, yesterday) => { - // eslint-disable-next-line no-undef - window.localStorage.setItem(telemetryDateStorageKey, yesterday); - }, telemetryDateStorageKey, date.valueOf()); -}; - describe('Telemetry', () => { + const DATE_FORMAT = 'YYYY-MM-DD'; + const TELEMETRY_PREFIX = 'telemetry'; let user; let docs; @@ -49,16 +43,25 @@ describe('Telemetry', () => { await utils.saveDocs(docs); await utils.createUsers([user]); await loginPage.login(user); + await commonPage.waitForPageLoaded(); }); it('should record telemetry', async () => { const yesterday = moment().subtract(1, 'day'); + const yesterdayDBName = `${TELEMETRY_PREFIX}-${yesterday.format(DATE_FORMAT)}-${user.username}`; + const telemetryRecord = { + key: 'a-telemetry-record', + value: 3, + date_recorded: yesterday.toDate(), + }; + await browser.execute(async (dbName, record) => { + // eslint-disable-next-line no-undef + await window.PouchDB(dbName).post(record); + }, yesterdayDBName, telemetryRecord); + // Some user activities to generate telemetry records await commonPage.goToReports(); await commonPage.goToPeople(); - await setOldTelemetryDate(user, yesterday); - - // generate telemetry aggregate await commonPage.goToReports(); await commonPage.sync(); @@ -67,7 +70,7 @@ describe('Telemetry', () => { const options = { auth: { username: user.username, password: user.password }, userName: user.username }; const metaDocs = await utils.requestOnTestMetaDb({ ...options, path: '/_all_docs?include_docs=true' }); - const telemetryEntry = metaDocs.rows.find(row => row.id.startsWith('telemetry')); + const telemetryEntry = metaDocs.rows.find(row => row.id.startsWith(TELEMETRY_PREFIX)); expect(telemetryEntry.doc).to.deep.nested.include({ 'metadata.year': yesterday.year(), 'metadata.month': yesterday.month() + 1, diff --git a/webapp/src/ts/services/db-sync.service.ts b/webapp/src/ts/services/db-sync.service.ts index b80853bc663..a4a63eea5d9 100644 --- a/webapp/src/ts/services/db-sync.service.ts +++ b/webapp/src/ts/services/db-sync.service.ts @@ -250,13 +250,11 @@ export class DBSyncService { ])) .then(([ push, pull ]) => { telemetryEntry.recordSuccess({ push, pull }); + this.ngZone.runOutsideAngular(() => purger.writeMetaPurgeLog(local, { syncedSeq: currentSeq })); }) .catch(err => { telemetryEntry.recordFailure(err, this.knownOnlineState); - }) - .then(() => this.ngZone.runOutsideAngular(() => { - purger.writeMetaPurgeLog(local, { syncedSeq: currentSeq }); - })); + }); } private sendUpdate(syncState: SyncState) { diff --git a/webapp/src/ts/services/telemetry.service.ts b/webapp/src/ts/services/telemetry.service.ts index f6f44256307..6526a1f4320 100644 --- a/webapp/src/ts/services/telemetry.service.ts +++ b/webapp/src/ts/services/telemetry.service.ts @@ -1,4 +1,5 @@ -import { Injectable, NgZone } from '@angular/core'; +import { Inject, Injectable, NgZone } from '@angular/core'; +import { DOCUMENT } from '@angular/common'; import { v4 as uuidv4 } from 'uuid'; import * as moment from 'moment'; @@ -8,94 +9,37 @@ import { SessionService } from '@mm-services/session.service'; @Injectable({ providedIn: 'root' }) -/** - * TelemetryService: Records, aggregates, and submits telemetry data. - */ export class TelemetryService { + private readonly TELEMETRY_PREFIX = 'telemetry'; + private readonly POUCH_PREFIX = '_pouch_'; + private readonly NAME_DIVIDER = '-'; + private readonly DATE_FORMAT = 'YYYY-MM-DD'; // Intentionally scoped to the whole browser (for this domain). We can then tell if multiple users use the same device private readonly DEVICE_ID_KEY = 'medic-telemetry-device-id'; - private DB_ID_KEY; - private FIRST_AGGREGATED_DATE_KEY; - - private queue = Promise.resolve(); + private isAggregationRunning = false; + private hasTransitionFinished = false; + private windowRef; constructor( private dbService:DbService, private sessionService:SessionService, private ngZone:NgZone, + @Inject(DOCUMENT) private document:Document, ) { - // Intentionally scoped to the specific user, as they may perform a - // different role (online vs. offline being being the most obvious) with different performance implications - this.DB_ID_KEY = ['medic', this.sessionService.userCtx().name, 'telemetry-db'].join('-'); - this.FIRST_AGGREGATED_DATE_KEY = ['medic', this.sessionService.userCtx().name, 'telemetry-date'].join('-'); - } - - private getDb() { - let dbName = window.localStorage.getItem(this.DB_ID_KEY); - - if (!dbName) { - // We're adding a UUID onto the end of the DB name to make it unique. In - // the past we've had trouble with PouchDB being able to delete a DB and - // then instantly create a new DB with the same name. - dbName = 'medic-user-' + this.sessionService.userCtx().name + '-telemetry-' + uuidv4(); - window.localStorage.setItem(this.DB_ID_KEY, dbName); - } - return window.PouchDB(dbName); // avoid angular-pouch as digest isn't necessary here + this.windowRef = this.document.defaultView; } getUniqueDeviceId() { - let uniqueDeviceId = window.localStorage.getItem(this.DEVICE_ID_KEY); + let uniqueDeviceId = this.windowRef.localStorage.getItem(this.DEVICE_ID_KEY); if (!uniqueDeviceId) { uniqueDeviceId = uuidv4(); - window.localStorage.setItem(this.DEVICE_ID_KEY, uniqueDeviceId!); + this.windowRef.localStorage.setItem(this.DEVICE_ID_KEY, uniqueDeviceId!); } return uniqueDeviceId; } - /** - * Returns a Moment object when the first telemetry record was created. - * - * This date is computed and stored in milliseconds (since Unix epoch) - * when we call this method for the first time and after every aggregation. - */ - private getFirstAggregatedDate() { - let date = parseInt(window.localStorage.getItem(this.FIRST_AGGREGATED_DATE_KEY)!); - - if (!date) { - date = Date.now(); - window.localStorage.setItem(this.FIRST_AGGREGATED_DATE_KEY, date.toString()); - } - - return moment(date); - } - - private storeIt(db, key, value) { - return db.post({ - key: key, - value: value, - date_recorded: Date.now(), - }); - } - - // moment when the aggregation starts (the beginning of the current day) - private aggregateStartsAt() { - return moment().startOf('day'); - } - - // if there is telemetry data from previous days, aggregation is performed and the data destroyed - private submitIfNeeded(db) { - const startOf = this.aggregateStartsAt(); - const dbDate = this.getFirstAggregatedDate(); - - if (dbDate.isBefore(startOf)) { - return this - .aggregate(db) - .then(() => this.reset(db)); - } - } - private convertReduceToKeyValues(reduce) { const kv = {}; @@ -106,24 +50,33 @@ export class TelemetryService { private generateAggregateDocId(metadata) { return [ - 'telemetry', + this.TELEMETRY_PREFIX, metadata.year, metadata.month, metadata.day, metadata.user, metadata.deviceId, - ].join('-'); + ].join(this.NAME_DIVIDER); } - private generateMetadataSection() { + private generateTelemetryDBName(today: TodayMoment): string { + return [ + this.TELEMETRY_PREFIX, + today.formatted, + // Scoped by user as they may perform a different role (online vs offline) with different performance implications + this.sessionService.userCtx().name, + ].join(this.NAME_DIVIDER); + } + + private generateMetadataSection(dbName) { return Promise .all([ this.dbService.get().get('_design/medic-client'), this.dbService.get().query('medic-client/doc_by_type', { key: ['form'], include_docs: true }), this.dbService.get().allDocs({ key: 'settings' }) ]) - .then(([ddoc, formResults, settingsResults]) => { - const date = this.getFirstAggregatedDate(); + .then(([ ddoc, formResults, settingsResults ]) => { + const date = this.getDBDate(dbName); const version = ddoc?.build_info?.version || 'unknown'; const forms = formResults.rows.reduce((keyToVersion, row) => { keyToVersion[row.doc.internalId] = row.doc._rev; @@ -132,9 +85,9 @@ export class TelemetryService { }, {}); return { - year: date.year(), - month: date.month() + 1, - day: date.date(), + year: date.year, + month: date.month, + day: date.date, user: this.sessionService.userCtx().name, deviceId: this.getUniqueDeviceId(), versions: { @@ -164,65 +117,132 @@ export class TelemetryService { }; } - // This should never happen (famous last words..), because we should only - // generate a new document for every month, which is part of the _id. + private async getTelemetryDBs(databases): Promise { + return databases + ?.map(db => db.name?.replace(this.POUCH_PREFIX, '') || '') + .filter(dbName => dbName?.startsWith(this.TELEMETRY_PREFIX)); + } + + /** + * This should never happen (famous last words..), because we should only + * generate a new document for every month, which is part of the _id. + */ private storeConflictedAggregate(aggregateDoc) { aggregateDoc.metadata.conflicted = true; - aggregateDoc._id = [aggregateDoc._id, 'conflicted', Date.now()].join('-'); + aggregateDoc._id = [ aggregateDoc._id, 'conflicted', Date.now() ].join(this.NAME_DIVIDER); return this.dbService .get({meta: true}) .put(aggregateDoc); } - private aggregate(db) { - const reduceQuery = db.query( - { - map: (doc, emit) => emit(doc.key, doc.value), - reduce: '_stats', - }, - { - group: true, + private getDBDate(dbName) { + const parts = dbName.split(this.NAME_DIVIDER); + return { + year: Number(parts[1]), + month: Number(parts[2]), + date: Number(parts[3]), + }; + } + + private async aggregate(db, dbName) { + const [ metadata, dbInfo, reduceResult ] = await Promise.all([ + this.generateMetadataSection(dbName), + this.dbService + .get() + .info(), + db.query( + { reduce: '_stats', map: (doc, emit) => emit(doc.key, doc.value) }, + { group: true }, + ) + ]); + + const aggregateDoc = { + _id: this.generateAggregateDocId(metadata), + type: this.TELEMETRY_PREFIX, + metrics: this.convertReduceToKeyValues(reduceResult), + device: this.generateDeviceStats(), + metadata, + dbInfo, + }; + + try { + await this.dbService + .get({ meta: true }) + .put(aggregateDoc); + } catch (error) { + if (error.status === 409) { + return this.storeConflictedAggregate(aggregateDoc); } - ); + throw error; + } + } - return Promise - .all([ - reduceQuery, - this.dbService.get().info(), - this.generateMetadataSection() - ]) - .then(qAll => { - const reduceResult = qAll[0]; - const infoResult = qAll[1]; - const metadata = qAll[2]; - - const aggregateDoc: any = { type: 'telemetry' }; - - aggregateDoc.metrics = this.convertReduceToKeyValues(reduceResult); - aggregateDoc.metadata = metadata; - aggregateDoc._id = this.generateAggregateDocId(aggregateDoc.metadata); - aggregateDoc.device = this.generateDeviceStats(); - aggregateDoc.dbInfo = infoResult; - - return this.dbService - .get({ meta: true }) - .put(aggregateDoc) - .catch(err => { - if (err.status === 409) { - return this.storeConflictedAggregate(aggregateDoc); - } - - throw err; - }); - }); + /** + * Moment when the aggregation starts (i.e. the beginning of the current day) + */ + private getToday(): TodayMoment { + const today = moment().startOf('day'); + return { + today, + formatted: today.format(this.DATE_FORMAT), + }; + } + + private async getCurrentTelemetryDB(today: TodayMoment, telemetryDBs) { + let currentDB = telemetryDBs?.find(db => db.includes(today.formatted)); + + if (!currentDB) { + currentDB = this.generateTelemetryDBName(today); + } + + return this.windowRef.PouchDB(currentDB); // Avoid angular-pouch as digest isn't necessary here + } + + private storeIt(db, key, value) { + return db.post({ + key: key, + value: value, + date_recorded: Date.now(), + }); + } + + private async submitIfNeeded(today: TodayMoment, telemetryDBs: string[] = []) { + if (this.isAggregationRunning) { + // Avoid multiple calls of the telemetry record function to prevent from doing duplicate aggregation work. + // It can throw "Failed to execute transaction on IDBDatabase" exception. + return; + } + + for (const dbName of telemetryDBs) { + if (dbName.includes(today.formatted)) { + // Don't submit today's telemetry records + continue; + } + + try { + this.isAggregationRunning = true; + const db = this.windowRef.PouchDB(dbName); + await this.aggregate(db, dbName); + await db.destroy(); + } catch (error) { + console.error('Error when aggregating the telemetry records', error); + } finally { + this.isAggregationRunning = false; + } + } } - private reset(db) { - window.localStorage.removeItem(this.DB_ID_KEY); - window.localStorage.removeItem(this.FIRST_AGGREGATED_DATE_KEY); + private closeDataBase(db) { + if (!db || db?._destroyed || db?._closed) { + return; + } - return db.destroy(); + try { + db.close(); + } catch (error) { + console.error('Error closing telemetry DB', error); + } } /** @@ -232,7 +252,7 @@ export class TelemetryService { * are recording a timing (as opposed to an event) a value. * * The first time this API is called each month, the telemetry recording - * is followed by an aggregation of all of the previous months data. + * is followed by an aggregation of all the previous months' data. * Aggregation is done using the `_stats` reduce function, which * generates data like so: * @@ -263,29 +283,62 @@ export class TelemetryService { return this.ngZone.runOutsideAngular(() => this._record(key, value)); } - private _record(key, value?) { + private async _record(key, value?) { if (value === undefined) { value = 1; } - let db; - this.queue = this.queue - .then(() => db = this.getDb()) - .then(() => this.submitIfNeeded(db)) - .then(() => db = this.getDb()) // db is fetched again in case submitIfNeeded dropped the old reference - .then(() => this.storeIt(db, key, value)) - .catch(err => console.error('Error in telemetry service', err)) - .finally(() => { - if (!db || db._destroyed || db._closed) { - return; - } - try { - db.close(); - } catch (err) { - console.error('Error closing telemetry DB', err); + try { + const today = this.getToday(); + const databases = await this.windowRef?.indexedDB?.databases(); + await this.deleteDeprecatedTelemetryDB(databases); + const telemetryDBs = await this.getTelemetryDBs(databases); + await this.submitIfNeeded(today, telemetryDBs); + const currentDB = await this.getCurrentTelemetryDB(today, telemetryDBs); + await this + .storeIt(currentDB, key, value) + .finally(() => this.closeDataBase(currentDB)); + } catch (error) { + console.error('Error in telemetry service', error); + } + } + + /** + * ToDo: Remove this function in a future release: https://github.com/medic/cht-core/issues/8657 + * The way telemetry was stored in the client side changed (https://github.com/medic/cht-core/pull/8555), + * this function contains all the transition code where it deletes the old Telemetry DB from the DB. + * It was decided to not aggregate the DB content. + * @private + */ + private async deleteDeprecatedTelemetryDB(databases) { + if (this.hasTransitionFinished) { + return; + } + + databases?.forEach(db => { + const nameNoPrefix = db.name?.replace(this.POUCH_PREFIX, '') || ''; + + // Skips new Telemetry DB, then matches the old deprecated Telemetry DB. + if (!nameNoPrefix.startsWith(this.TELEMETRY_PREFIX) + && nameNoPrefix.includes(this.TELEMETRY_PREFIX) + && nameNoPrefix.includes(this.sessionService.userCtx().name)) { + this.windowRef?.indexedDB.deleteDatabase(db.name); + } + }); + + Object + .keys(this.windowRef?.localStorage) + .forEach(key => { + if (key.includes('telemetry-date') || key.includes('telemetry-db')) { + this.windowRef?.localStorage.removeItem(key); } }); - return this.queue; + this.hasTransitionFinished = true; } } + +type TodayMoment = { + today: Record; + formatted: string; +} diff --git a/webapp/tests/karma/ts/services/db-sync.service.spec.ts b/webapp/tests/karma/ts/services/db-sync.service.spec.ts index cbd42bc3545..06f95d2ed90 100644 --- a/webapp/tests/karma/ts/services/db-sync.service.spec.ts +++ b/webapp/tests/karma/ts/services/db-sync.service.spec.ts @@ -948,6 +948,16 @@ describe('DBSync service', () => { }); }); + it('should not update the current seq in the purge log when sync fails', () => { + localMetaDb.info.resolves({ update_seq: 100 }); + localMetaDb.replicate.to.rejects('some error'); + return service.sync().then(() => { + expect(localMetaDb.info.calledOnce).to.be.true; + expect(localMetaDb.get.notCalled).to.be.true; + expect(localMetaDb.put.notCalled).to.be.true; + }); + }); + it('should record telemetry when successful', async () => { let metaToResolve; let metaFromResolve; diff --git a/webapp/tests/karma/ts/services/telemetry.service.spec.ts b/webapp/tests/karma/ts/services/telemetry.service.spec.ts index cef8d4b8cd9..606ff16c486 100644 --- a/webapp/tests/karma/ts/services/telemetry.service.spec.ts +++ b/webapp/tests/karma/ts/services/telemetry.service.spec.ts @@ -1,7 +1,7 @@ import { TestBed } from '@angular/core/testing'; +import { DOCUMENT } from '@angular/common'; import sinon from 'sinon'; import { expect } from 'chai'; -import * as moment from 'moment'; import { TelemetryService } from '@mm-services/telemetry.service'; import { DbService } from '@mm-services/db.service'; @@ -16,11 +16,8 @@ describe('TelemetryService', () => { let sessionService; let clock; let telemetryDb; - let storageGetItemStub; - let storageSetItemStub; let consoleErrorSpy; - - const windowPouchOriginal = window.PouchDB; + let windowMock; const windowScreenOriginal = { availWidth: window.screen.availWidth, @@ -77,19 +74,6 @@ describe('TelemetryService', () => { ); }; - const subtractDays = (numDays) => { - return moment() - .subtract(numDays, 'days') - .valueOf() - .toString(); - }; - - const sameDay = () => { - return moment() - .valueOf() - .toString(); - }; - beforeEach(() => { defineWindow(); metaDb = { @@ -102,7 +86,7 @@ describe('TelemetryService', () => { allDocs: sinon.stub() }; const getStub = sinon.stub(); - getStub.withArgs({meta: true}).returns(metaDb); + getStub.withArgs({ meta: true }).returns(metaDb); getStub.returns(medicDb); dbService = { get: getStub }; consoleErrorSpy = sinon.spy(console, 'error'); @@ -116,63 +100,78 @@ describe('TelemetryService', () => { }) }; sessionService = { userCtx: sinon.stub().returns({ name: 'greg' }) }; - storageGetItemStub = sinon.stub(window.localStorage, 'getItem'); - storageSetItemStub = sinon.stub(window.localStorage, 'setItem'); + windowMock = { + PouchDB: sinon.stub().returns(telemetryDb), + indexedDB: { databases: sinon.stub() }, + localStorage: { getItem: sinon.stub(), setItem: sinon.stub() }, + }; + const documentMock = { + defaultView: windowMock, + querySelectorAll: sinon.stub().returns([]), + }; TestBed.configureTestingModule({ providers: [ { provide: DbService, useValue: dbService }, - { provide: SessionService, useValue: sessionService } + { provide: SessionService, useValue: sessionService }, + { provide: DOCUMENT, useValue: documentMock }, ] }); service = TestBed.inject(TelemetryService); clock = sinon.useFakeTimers(NOW); - window.PouchDB = () => telemetryDb; }); afterEach(() => { clock.restore(); sinon.restore(); - window.PouchDB = windowPouchOriginal; restoreWindow(); }); describe('record()', () => { it('should record a piece of telemetry', async () => { - storageGetItemStub.withArgs('medic-greg-telemetry-db').returns('dbname'); - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(Date.now().toString()); + medicDb.query.resolves({ rows: [] }); + telemetryDb.query.resolves({ rows: [] }); + windowMock.indexedDB.databases.resolves([ + { name: '_pouch_telemetry-2018-11-10-greg' }, + { name: '_pouch_some-other-db' }, + { name: '_pouch_telemetry-2018-11-09-greg' }, + ]); await service.record('test', 100); - expect(consoleErrorSpy.callCount).to.equal(0); - expect(telemetryDb.post.callCount).to.equal(1); + expect(consoleErrorSpy.notCalled).to.be.true; + expect(telemetryDb.post.calledOnce).to.be.true; expect(telemetryDb.post.args[0][0]).to.deep.include({ key: 'test', value: 100 }); expect(telemetryDb.post.args[0][0].date_recorded).to.be.above(0); - expect(storageGetItemStub.callCount).to.equal(3); - expect(storageGetItemStub.args[0]).to.deep.equal(['medic-greg-telemetry-db']); - expect(storageGetItemStub.args[1]).to.deep.equal(['medic-greg-telemetry-date']); - expect(storageGetItemStub.args[2]).to.deep.equal(['medic-greg-telemetry-db']); - expect(telemetryDb.close.callCount).to.equal(1); + expect(windowMock.indexedDB.databases.calledOnce).to.be.true; + expect(windowMock.PouchDB.calledTwice).to.be.true; + expect(windowMock.PouchDB.args[0]).to.deep.equal([ 'telemetry-2018-11-09-greg' ]); + expect(windowMock.PouchDB.args[1]).to.deep.equal([ 'telemetry-2018-11-10-greg' ]); + expect(telemetryDb.destroy.calledOnce).to.be.true; }); it('should default the value to 1 if not passed', async () => { - storageGetItemStub.withArgs('medic-greg-telemetry-db').returns('dbname'); - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(Date.now().toString()); + medicDb.query.resolves({ rows: [] }); + telemetryDb.query.resolves({ rows: [] }); + windowMock.indexedDB.databases.resolves([ + { name: 'telemetry-2018-11-10-greg' }, + { name: 'some-other-db' }, + { name: 'telemetry-2018-11-09-greg' }, + ]); await service.record('test'); - expect(consoleErrorSpy.callCount).to.equal(0); + expect(consoleErrorSpy.notCalled).to.be.true; expect(telemetryDb.post.args[0][0].value).to.equal(1); - expect(telemetryDb.close.callCount).to.equal(1); + expect(telemetryDb.destroy.calledOnce).to.be.true; }); const setupDbMocks = () => { - storageGetItemStub.returns('dbname'); telemetryDb.query.resolves({ rows: [ - { key: 'foo', value: {sum: 2876, min: 581, max: 2295, count: 2, sumsqr: 5604586} }, - { key: 'bar', value: {sum: 93, min: 43, max: 50, count: 2, sumsqr: 4349} }, + { key: 'foo', value: { sum: 2876, min: 581, max: 2295, count: 2, sumsqr: 5604586 } }, + { key: 'bar', value: { sum: 93, min: 43, max: 50, count: 2, sumsqr: 4349 } }, ], }); medicDb.info.resolves({ some: 'stats' }); @@ -205,179 +204,195 @@ describe('TelemetryService', () => { }); }; - it('should aggregate once a day and resets the db first', async () => { + it('should aggregate once a day and delete previous telemetry databases', async () => { + windowMock.indexedDB.databases.resolves([ + { name: 'telemetry-2018-11-10-greg' }, + { name: 'some-other-db' }, + { name: 'telemetry-2018-11-09-greg' }, + { name: 'telemetry-2018-10-02-greg' }, + ]); setupDbMocks(); - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(subtractDays(5)); await service.record('test', 1); - expect(telemetryDb.post.callCount).to.equal(1); + expect(telemetryDb.post.calledOnce).to.be.true; expect(telemetryDb.post.args[0][0]).to.deep.include({ key: 'test', value: 1 }); + expect(metaDb.put.calledTwice).to.be.true; - expect(metaDb.put.callCount).to.equal(1); - const aggregatedDoc = metaDb.put.args[0][0]; - expect(aggregatedDoc._id).to.match(/^telemetry-2018-11-5-greg-[\w-]+$/); - expect(aggregatedDoc.metrics).to.deep.equal({ - foo: {sum: 2876, min: 581, max: 2295, count: 2, sumsqr: 5604586}, - bar: {sum: 93, min: 43, max: 50, count: 2, sumsqr: 4349}, + const aggregatedDocNov = metaDb.put.args[0][0]; + expect(aggregatedDocNov._id).to.match(/^telemetry-2018-11-9-greg-[\w-]+$/); + expect(aggregatedDocNov.metrics).to.deep.equal({ + foo: { sum: 2876, min: 581, max: 2295, count: 2, sumsqr: 5604586 }, + bar: { sum: 93, min: 43, max: 50, count: 2, sumsqr: 4349 }, + }); + expect(aggregatedDocNov.type).to.equal('telemetry'); + expect(aggregatedDocNov.metadata.year).to.equal(2018); + expect(aggregatedDocNov.metadata.month).to.equal(11); + expect(aggregatedDocNov.metadata.day).to.equal(9); + expect(aggregatedDocNov.metadata.user).to.equal('greg'); + expect(aggregatedDocNov.metadata.versions).to.deep.equal({ + app: '3.0.0', + forms: { anc_followup: '1-abc' }, + settings: 'somerandomrevision', + }); + expect(aggregatedDocNov.dbInfo).to.deep.equal({ some: 'stats' }); + expect(aggregatedDocNov.device).to.deep.equal({ + userAgent: 'Agent Smith', + hardwareConcurrency: 4, + screen: { width: 768, height: 1024 }, + deviceInfo: {} + }); + + const aggregatedDocOct = metaDb.put.args[1][0]; + expect(aggregatedDocOct._id.startsWith('telemetry-2018-10-2-greg')).to.be.true; + expect(aggregatedDocOct.metrics).to.deep.equal({ + foo: { sum: 2876, min: 581, max: 2295, count: 2, sumsqr: 5604586 }, + bar: { sum: 93, min: 43, max: 50, count: 2, sumsqr: 4349 }, }); - expect(aggregatedDoc.type).to.equal('telemetry'); - expect(aggregatedDoc.metadata.year).to.equal(2018); - expect(aggregatedDoc.metadata.month).to.equal(11); - expect(aggregatedDoc.metadata.day).to.equal(5); - expect(aggregatedDoc.metadata.user).to.equal('greg'); - expect(aggregatedDoc.metadata.versions).to.deep.equal({ + expect(aggregatedDocOct.type).to.equal('telemetry'); + expect(aggregatedDocOct.metadata.year).to.equal(2018); + expect(aggregatedDocOct.metadata.month).to.equal(10); + expect(aggregatedDocOct.metadata.day).to.equal(2); + expect(aggregatedDocOct.metadata.user).to.equal('greg'); + expect(aggregatedDocOct.metadata.versions).to.deep.equal({ app: '3.0.0', - forms: { - anc_followup: '1-abc' - }, - settings: 'somerandomrevision' + forms: { anc_followup: '1-abc' }, + settings: 'somerandomrevision', }); - expect(aggregatedDoc.dbInfo).to.deep.equal({ some: 'stats' }); - expect(aggregatedDoc.device).to.deep.equal({ + expect(aggregatedDocOct.dbInfo).to.deep.equal({ some: 'stats' }); + expect(aggregatedDocOct.device).to.deep.equal({ userAgent: 'Agent Smith', hardwareConcurrency: 4, - screen: { - width: 768, - height: 1024, - }, + screen: { width: 768, height: 1024 }, deviceInfo: {} }); - expect(medicDb.query.callCount).to.equal(1); + expect(medicDb.query.calledTwice).to.be.true; expect(medicDb.query.args[0][0]).to.equal('medic-client/doc_by_type'); - expect(medicDb.query.args[0][1]).to.deep.equal({ key: ['form'], include_docs: true }); - expect(telemetryDb.destroy.callCount).to.equal(1); - expect(telemetryDb.close.callCount).to.equal(0); + expect(medicDb.query.args[0][1]).to.deep.equal({ key: [ 'form' ], include_docs: true }); + expect(telemetryDb.destroy.calledTwice).to.be.true; + expect(telemetryDb.close.notCalled).to.be.true; - expect(consoleErrorSpy.callCount).to.equal(0); // no errors + expect(consoleErrorSpy.notCalled).to.be.true; }); it('should not aggregate when recording the day the db was created and next day it should aggregate', async () => { + windowMock.indexedDB.databases.resolves([ + { name: 'telemetry-2018-11-10-greg' }, + { name: 'some-other-db' }, + ]); setupDbMocks(); - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(sameDay()); await service.record('test', 10); - expect(telemetryDb.post.callCount).to.equal(1); // Telemetry entry has been recorded + expect(telemetryDb.post.calledOnce).to.be.true; // Telemetry entry has been recorded expect(telemetryDb.post.args[0][0]).to.deep.include({ key: 'test', value: 10 }); - expect(telemetryDb.query.called).to.be.false; // NO telemetry aggregation has - expect(metaDb.put.callCount).to.equal(0); // been recorded yet + expect(telemetryDb.query.notCalled).to.be.true; // NO telemetry aggregation has + expect(metaDb.put.notCalled).to.be.true; // been recorded yet + expect(windowMock.PouchDB.calledOnce).to.be.true; + expect(windowMock.PouchDB.args[0]).to.deep.equal([ 'telemetry-2018-11-10-greg' ]); clock.tick('01:00'); // 1 min later ... await service.record('test', 5); - expect(telemetryDb.post.callCount).to.equal(2); // second call + expect(telemetryDb.post.calledTwice).to.be.true; // second call expect(telemetryDb.post.args[1][0]).to.deep.include({ key: 'test', value: 5 }); - expect(telemetryDb.query.called).to.be.false; // still NO aggregation has - expect(metaDb.put.callCount).to.equal(0); // been recorded (same day) + expect(telemetryDb.query.notCalled).to.be.true; // still NO aggregation has + expect(metaDb.put.notCalled).to.be.true; // been recorded (same day) + expect(windowMock.PouchDB.calledTwice).to.be.true; + expect(windowMock.PouchDB.args[0]).to.deep.equal([ 'telemetry-2018-11-10-greg' ]); let postCalledAfterQuery = false; telemetryDb.post.callsFake(async () => postCalledAfterQuery = telemetryDb.query.called); clock.tick('24:00:00'); // 1 day later ... await service.record('test', 2); - expect(telemetryDb.post.callCount).to.equal(3); // third call + expect(telemetryDb.post.calledThrice).to.be.true; // third call expect(telemetryDb.post.args[2][0]).to.deep.include({ key: 'test', value: 2 }); - expect(telemetryDb.query.callCount).to.equal(1); // Now aggregation HAS been performed - expect(metaDb.put.callCount).to.equal(1); // and the stats recorded + expect(telemetryDb.query.calledOnce).to.be.true; // Now aggregation HAS been performed + expect(metaDb.put.calledOnce).to.be.true; // and the stats recorded + expect(windowMock.PouchDB.callCount).to.equal(4); + expect(windowMock.PouchDB.args[2]).to.deep.equal([ 'telemetry-2018-11-10-greg' ]); + expect(windowMock.PouchDB.args[3]).to.deep.equal([ 'telemetry-2018-11-11-greg' ]); // The telemetry record has been recorded after aggregation to not being included in the stats, // because the record belong to the current date, not the day aggregated (yesterday) expect(postCalledAfterQuery).to.be.true; const aggregatedDoc = metaDb.put.args[0][0]; - expect(aggregatedDoc._id).to.match(/^telemetry-2018-11-10-greg-[\w-]+$/); // Now is 2018-11-11 but aggregation - expect(telemetryDb.destroy.callCount).to.equal(1); // is from the previous day + // Now is 2018-11-11 and aggregated telemetry for 2018-11-10 + expect(aggregatedDoc._id).to.match(/^telemetry-2018-11-10-greg-[\w-]+$/); + expect(telemetryDb.destroy.calledOnce).to.be.true; // is from the previous day - expect(consoleErrorSpy.callCount).to.equal(0); // no errors + expect(consoleErrorSpy.notCalled).to.be.true; }); it('should aggregate from days with records skipping days without records', async () => { + windowMock.indexedDB.databases.resolves([]); setupDbMocks(); - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(sameDay()); await service.record('datapoint', 12); - expect(telemetryDb.post.callCount).to.equal(1); - expect(metaDb.put.callCount).to.equal(0); // NO telemetry has been recorded yet + expect(telemetryDb.post.calledOnce).to.be.true; + expect(windowMock.PouchDB.calledOnce).to.be.true; + expect(windowMock.PouchDB.args[0]).to.deep.equal([ 'telemetry-2018-11-10-greg' ]); + expect(metaDb.put.notCalled).to.be.true; // NO telemetry has been recorded yet clock.tick('01:00'); // 1 min later ... await service.record('another.datapoint'); - expect(telemetryDb.post.callCount).to.equal(2); // second call - expect(metaDb.put.callCount).to.equal(0); // still NO telemetry has been recorded (same day) + expect(telemetryDb.post.calledTwice).to.be.true; // second call + expect(windowMock.PouchDB.calledTwice).to.be.true; + expect(windowMock.PouchDB.args[0]).to.deep.equal([ 'telemetry-2018-11-10-greg' ]); + expect(metaDb.put.notCalled).to.be.true; // still NO telemetry has been recorded (same day) - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(sameDay()); clock.tick('48:00:00'); // 2 days later ... + windowMock.indexedDB.databases.resolves([ { name: 'telemetry-2018-11-10-greg' } ]); await service.record('test', 2); - expect(telemetryDb.post.callCount).to.equal(3); // third call - expect(metaDb.put.callCount).to.equal(1); // Now telemetry IS recorded + expect(telemetryDb.post.calledThrice).to.be.true; // third call + expect(windowMock.PouchDB.callCount).to.equal(4); + expect(windowMock.PouchDB.args[2]).to.deep.equal([ 'telemetry-2018-11-10-greg' ]); + expect(windowMock.PouchDB.args[3]).to.deep.equal([ 'telemetry-2018-11-12-greg' ]); + expect(metaDb.put.calledOnce).to.be.true; // Now telemetry IS recorded let aggregatedDoc = metaDb.put.args[0][0]; expect(aggregatedDoc._id).to.match(/^telemetry-2018-11-10-greg-[\w-]+$/); // Today 2018-11-12 but aggregation is - expect(telemetryDb.destroy.callCount).to.equal(1); // from from 2 days ago (not Yesterday) - - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(sameDay()); // same day now is 2 days ahead + expect(telemetryDb.destroy.calledOnce).to.be.true; // from 2 days ago (not Yesterday) clock.tick(5 * 24 * 60 * 60 * 1000); // 5 more days later ... + windowMock.indexedDB.databases.resolves([ { name: 'telemetry-2018-11-12-greg' } ]); await service.record('point.a', 1); expect(telemetryDb.post.callCount).to.equal(4); // 4th call - expect(metaDb.put.callCount).to.equal(2); // Telemetry IS recorded again + expect(windowMock.PouchDB.callCount).to.equal(6); + expect(windowMock.PouchDB.args[4]).to.deep.equal([ 'telemetry-2018-11-12-greg' ]); + expect(windowMock.PouchDB.args[5]).to.deep.equal([ 'telemetry-2018-11-17-greg' ]); + expect(metaDb.put.calledTwice).to.be.true; // Telemetry IS recorded again aggregatedDoc = metaDb.put.args[1][0]; - expect(aggregatedDoc._id).to.match(/^telemetry-2018-11-12-greg-[\w-]+$/); // Now is Nov 19 but agg. is from Nov 12 - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(sameDay()); // same day now is 7 days ahead + expect(aggregatedDoc._id).to.match(/^telemetry-2018-11-12-greg-[\w-]+$/); // Now is Nov 17 but agg. is from Nov 12 // A new record is added ... clock.tick('02:00:00'); // 2 hours later ... + windowMock.indexedDB.databases.resolves([]); await service.record('point.b', 0); // 1 record added // ...the aggregation count is the same because // the aggregation was already performed 2 hours ago within the same day expect(telemetryDb.post.callCount).to.equal(5); // 5th call - expect(metaDb.put.callCount).to.equal(2); // Telemetry count is the same - - expect(consoleErrorSpy.callCount).to.equal(0); // no errors - }); - }); - - describe('getDb()', () => { - it('should set localStorage values', async () => { - storageGetItemStub - .withArgs('medic-greg-telemetry-db') - .returns(undefined); - storageGetItemStub - .withArgs('medic-greg-telemetry-date') - .returns(undefined); - - await service.record('test', 1); + expect(metaDb.put.calledTwice).to.be.true; // Telemetry count is the same - expect(consoleErrorSpy.callCount).to.equal(0); - expect(storageSetItemStub.callCount).to.equal(3); - expect(storageSetItemStub.args[0][0]).to.equal('medic-greg-telemetry-db'); - expect(storageSetItemStub.args[0][1]).to.match(/^medic-user-greg-telemetry-[\w-]+$/); - expect(storageSetItemStub.args[1][0]).to.equal('medic-greg-telemetry-date'); - expect(storageSetItemStub.args[1][1]).to.equal(NOW.toString()); + expect(consoleErrorSpy.notCalled).to.be.true; }); }); describe('storeConflictedAggregate()', () => { - it('should deal with conflicts by making the ID unique and noting the conflict in the new document', async () => { - storageGetItemStub.withArgs('medic-greg-telemetry-db').returns('dbname'); - storageGetItemStub.withArgs('medic-greg-telemetry-date').returns(subtractDays(5)); + windowMock.indexedDB.databases.resolves([ { name: '_pouch_telemetry-2018-11-05-greg' } ]); telemetryDb.query = sinon.stub().resolves({ rows: [ - { - key: 'foo', - value: 'stats', - }, - { - key: 'bar', - value: 'more stats', - }, + { key: 'foo', value: 'stats' }, + { key: 'bar', value: 'more stats' }, ], }); medicDb.info.resolves({ some: 'stats' }); @@ -400,12 +415,12 @@ describe('TelemetryService', () => { await service.record('test', 1); - expect(consoleErrorSpy.callCount).to.equal(0); - expect(metaDb.put.callCount).to.equal(2); + expect(consoleErrorSpy.notCalled).to.be.true; + expect(metaDb.put.calledTwice).to.be.true; expect(metaDb.put.args[1][0]._id).to.match(/^telemetry-2018-11-5-greg-[\w-]+-conflicted-[\w-]+$/); expect(metaDb.put.args[1][0].metadata.conflicted).to.equal(true); - expect(telemetryDb.destroy.callCount).to.equal(1); - expect(telemetryDb.close.callCount).to.equal(0); + expect(telemetryDb.destroy.calledOnce).to.be.true; + expect(telemetryDb.close.notCalled).to.be.true; }); }); });