From d624a8904cb814c31ef8b6d4eddcbb28cec1f8a2 Mon Sep 17 00:00:00 2001 From: Jess Robinson Date: Wed, 28 Apr 2021 12:06:54 +0000 Subject: [PATCH] refactor(app): Update cache to handle bulk message downloading --- e2e/cypress/integration/message-caching.ts | 6 +- src/app/rmmapi/messagecache.ts | 28 ++++++- src/app/rmmapi/rbwebmail.ts | 96 ++++++++++++---------- 3 files changed, 82 insertions(+), 48 deletions(-) diff --git a/e2e/cypress/integration/message-caching.ts b/e2e/cypress/integration/message-caching.ts index 996295e6b..6bb0c3536 100644 --- a/e2e/cypress/integration/message-caching.ts +++ b/e2e/cypress/integration/message-caching.ts @@ -9,8 +9,8 @@ describe('Message caching', () => { }); - it('should cache all messages on first time page load', () => { - cy.intercept('/rest/v1/email/12').as('message12requested'); + it('should cache all messages on first time page load', () => { + cy.intercept('/rest/v1/email/download/*').as('message12requested'); cy.visit('/'); cy.wait('@message12requested', {'timeout':10000}); @@ -27,7 +27,7 @@ describe('Message caching', () => { // Now don't fetch it again: cy.visit('/#Inbox:12'); let called = false; - cy.intercept('/rest/v1/email/12', (_req) => { + cy.intercept('/rest/v1/email/download/*', (_req) => { called = true; }); diff --git a/src/app/rmmapi/messagecache.ts b/src/app/rmmapi/messagecache.ts index abddd995c..6073219eb 100644 --- a/src/app/rmmapi/messagecache.ts +++ b/src/app/rmmapi/messagecache.ts @@ -30,8 +30,10 @@ export class MessageCache { try { this.db = new Dexie(`messageCache-${userId}`); - this.db.version(2).stores({ - messages: '', // use out-of-line keys + this.db.version(3).stores({ + // use out-of-line keys, but index "id" + // yes, empty first arg is deliberate + messages: ',&id', }); } catch (err) { console.log(`Error initializing messagecache: ${err}`); @@ -45,6 +47,28 @@ export class MessageCache { ); } + // verify which ids we already have + async checkIds(ids: number[]): Promise { + return this.db?.table('messages') + .where('id') + .anyOf(ids) + .primaryKeys() + .then( + (keys) => keys.map((key) => key as number) + ); + } + + async getMany(ids: number[]): Promise { + return this.db?.table('messages') + .where('id') + .anyOf(ids) + .toArray() + .then( + result => result, + _error => null, + ); + } + set(id: number, contents: MessageContents): void { contents.version = this.message_version; this.db?.table('messages').put(contents, id).catch( diff --git a/src/app/rmmapi/rbwebmail.ts b/src/app/rmmapi/rbwebmail.ts index f836112fe..39883f57d 100644 --- a/src/app/rmmapi/rbwebmail.ts +++ b/src/app/rmmapi/rbwebmail.ts @@ -19,7 +19,7 @@ import { Injectable, NgZone } from '@angular/core'; import { Observable, from, Subject, AsyncSubject, firstValueFrom } from 'rxjs'; -import { share } from 'rxjs/operators'; +import { catchError, concatMap, share, map, mergeMap, tap } from 'rxjs/operators'; import { MessageInfo } from '../common/messageinfo'; import { MailAddressInfo } from '../common/mailaddressinfo'; import { FolderListEntry } from '../common/folderlistentry'; @@ -174,6 +174,10 @@ export class RunboxWebmailAPI { private messageCache: AsyncSubject = new AsyncSubject(); private messageContentsRequestCache = new LRUMessageCache>(); + // Track which messageids we're already fetching + // Else we attempt to refetch the same ones a fair bit on start-up + private downloadingMessages: number[] = []; + constructor( private http: HttpClient, private ngZone: NgZone, @@ -266,51 +270,57 @@ export class RunboxWebmailAPI { } public downloadMessages(messageIds: number[]): Promise { - const missingMessages = []; - for (const msgid of messageIds) { - if (!this.messageContentsCache[msgid]) { - this.messageContentsCache[msgid] = new AsyncSubject(); - missingMessages.push(msgid); - } - } - - const messagePromises = messageIds.map(id => this.messageContentsCache[id].toPromise()); - - if (missingMessages.length > 0) { - this.http.get(`/rest/v1/email/download/${missingMessages.join(',')}`).pipe( - catchError((err: HttpErrorResponse) => throwError(err.message)), - concatMap((res: any) => { - if (res.status === 'success') { - return of(res.result); - } else { - return throwError(res.errors[0]); - } - }), - ).subscribe( - (result: any) => { - for (const resultKey of Object.keys(result)) { - const msgid = parseInt(resultKey, 10); - const contents = result[msgid]?.json; - if (contents) { - this.messageContentsCache[msgid].next(contents); - this.messageContentsCache[msgid].complete(); + const cached = this.messageCache.checkIds([...messageIds]); + return cached.then((inCache) => { + const missingMessages = messageIds.filter( + (msgId) => !inCache.includes(msgId) + && !this.downloadingMessages.includes(msgId) + ); + const messagePromises = missingMessages.map(id => this.messageCache.get(id)); + + if (missingMessages.length > 0) { + this.downloadingMessages = this.downloadingMessages.concat(missingMessages); + this.http.get(`/rest/v1/email/download/${missingMessages.join(',')}`).pipe( + catchError((err: HttpErrorResponse) => throwError(err.message)), + concatMap((res: any) => { + if (res.status === 'success') { + return of(res.result); } else { - this.messageContentsCache[msgid].error(result[msgid]?.error); - delete this.messageContentsCache[msgid]; + return throwError(res.errors[0]); + } + }), + ).subscribe( + (result: any) => { + for (const resultKey of Object.keys(result)) { + const msgid = parseInt(resultKey, 10); + const contents = result[msgid]?.json; + if (contents) { + this.messageCache.set(msgid, contents); + } else { + this.deleteCachedMessageContents(msgid); + } + const msgIndex = this.downloadingMessages + .findIndex((id) => msgid === id); + if (msgIndex > -1) { + this.downloadingMessages.splice(msgIndex, 1); + } + } + }, + (err: Error) => { + for (const msgid of missingMessages) { + this.deleteCachedMessageContents(msgid); + const msgIndex = this.downloadingMessages + .findIndex((id) => msgid === id); + if (msgIndex > -1) { + this.downloadingMessages.splice(msgIndex, 1); + } } } - }, - (err: Error) => { - for (const msgid of missingMessages) { - this.messageContentsCache[msgid].error(err.toString()); - delete this.messageContentsCache[msgid]; - } - } - ); - } - - // return Promise.allSettled(messagePromises); - return Promise.all(messagePromises); + ); + } + // return Promise.allSettled(messagePromises); + return Promise.all(messagePromises); + }); } public updateLastOn(): Observable {