perf(app): Load missing msg bodies in bulk, instead of one at a time

runbox · Dec 17, 2024 · ce7a552 · ce7a552
1 parent d624a89
commit ce7a552
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 43 deletions.
diff --git a/src/app/rmmapi/rbwebmail.spec.ts b/src/app/rmmapi/rbwebmail.spec.ts
@@ -35,6 +35,13 @@ describe('RBWebMail', () => {
             ],
             providers: [
                 RunboxWebmailAPI,
+                { provide: MessageCache, useValue: {
+                    get: (_) => Promise.resolve(null),
+                    set: (_, __) => {},
+                    delete: (_) => {},
+                    checkIds: (_) => Promise.resolve([]),
+                    getMany: (_) => Promise.resolve([]),
+                } },
             ]
         });
     });

diff --git a/src/app/rmmapi/rbwebmail.ts b/src/app/rmmapi/rbwebmail.ts
@@ -272,53 +272,62 @@ export class RunboxWebmailAPI {
     public downloadMessages(messageIds: number[]): Promise<MessageContents[]> {
         const cached = this.messageCache.checkIds([...messageIds]);
         return cached.then((inCache) => {
+            // Filter out items we already have
+            // or are busy fetching
             const missingMessages = messageIds.filter(
                 (msgId) => !inCache.includes(msgId)
                     && !this.downloadingMessages.includes(msgId)
             );
-            const messagePromises = missingMessages.map(id => this.messageCache.get(id));
 
+            const messagePromises = [];
             if (missingMessages.length > 0) {
                 this.downloadingMessages = this.downloadingMessages.concat(missingMessages);
-                this.http.get(`/rest/v1/email/download/${missingMessages.join(',')}`).pipe(
-                    catchError((err: HttpErrorResponse) => throwError(err.message)),
-                    concatMap((res: any) => {
-                        if (res.status === 'success') {
-                            return of(res.result);
-                        } else {
-                            return throwError(res.errors[0]);
-                        }
-                    }),
-                ).subscribe(
-                    (result: any) => {
-                        for (const resultKey of Object.keys(result)) {
-                            const msgid = parseInt(resultKey, 10);
-                            const contents = result[msgid]?.json;
-                            if (contents) {
-                                this.messageCache.set(msgid, contents);
-                            } else {
-                                this.deleteCachedMessageContents(msgid);
-                            }
-                            const msgIndex = this.downloadingMessages
-                                .findIndex((id) => msgid === id);
-                            if (msgIndex > -1) {
-                                this.downloadingMessages.splice(msgIndex, 1);
-                            }
-                        }
-                    },
-                    (err: Error) => {
-                        for (const msgid of missingMessages) {
-                            this.deleteCachedMessageContents(msgid);
-                            const msgIndex = this.downloadingMessages
-                                .findIndex((id) => msgid === id);
-                            if (msgIndex > -1) {
-                                this.downloadingMessages.splice(msgIndex, 1);
+                messagePromises.push(
+                    new Promise((resolve, reject) => {
+                        this.http.get(`/rest/v1/email/download/${missingMessages.join(',')}`).pipe(
+                            catchError((err: HttpErrorResponse) => throwError(err.message)),
+                            concatMap((res: any) => {
+                                if (res.status === 'success') {
+                                    return of(res.result);
+                                } else {
+                                    return throwError(res.errors[0]);
+                                }
+                            }),
+                        ).subscribe(
+                            (result: any) => {
+                                for (const resultKey of Object.keys(result)) {
+                                    const msgid = parseInt(resultKey, 10);
+                                    console.log('RMMAPI Got some result:', result);
+                                    const contents = result[msgid]?.json;
+                                    if (contents) {
+                                        console.log(`RMMAPI setting cache for msgid = ${msgid}`);
+                                        this.messageCache.set(msgid, contents);
+                                    } else {
+                                        this.deleteCachedMessageContents(msgid);
+                                    }
+                                    const msgIndex = this.downloadingMessages
+                                        .findIndex((id) => msgid === id);
+                                    if (msgIndex > -1) {
+                                        this.downloadingMessages.splice(msgIndex, 1);
+                                    }
+                                }
+                                resolve();
+                            },
+                            (err: Error) => {
+                                for (const msgid of missingMessages) {
+                                    this.deleteCachedMessageContents(msgid);
+                                    const msgIndex = this.downloadingMessages
+                                        .findIndex((id) => msgid === id);
+                                    if (msgIndex > -1) {
+                                        this.downloadingMessages.splice(msgIndex, 1);
+                                    }
+                                }
+                                reject(err);
                             }
-                        }
-                    }
+                        );
+                    })
                 );
             }
-            // return Promise.allSettled(messagePromises);
             return Promise.all(messagePromises);
         });
     }

diff --git a/src/app/xapian/searchservice.spec.ts b/src/app/xapian/searchservice.spec.ts
@@ -106,6 +106,7 @@ describe('SearchService', () => {
           ],
             providers: [
                 SearchService,
+                MessageCache,
                 MessageListService,
                 RunboxWebmailAPI
                 // { provide: Worker, useValue: {
@@ -245,6 +246,7 @@ describe('SearchService', () => {
 
 
         expect(await searchService.initSubject.toPromise()).toBeTruthy();
+        console.log('search service initialised');
         expect(searchService.localSearchActivated).toBeTruthy();
         expect(localdir).toEqual(searchService.localdir);
 
@@ -286,12 +288,16 @@ describe('SearchService', () => {
 
         await new Promise(resolve => setTimeout(resolve, 1000));
 
-        req = httpMock.expectOne('/rest/v1/email/' + testMessageId);
+        req = httpMock.expectOne('/rest/v1/email/download/' + testMessageId);
         req.flush({
             status: 'success',
             result: {
-                text: {
-                    text: 'message body test text SecretSauceFormula'
+                [testMessageId]: {
+                    json: {
+                        text: {
+                            text: 'message body test text SecretSauceFormula'
+                        }
+                    }
                 }
             }
         });
@@ -314,7 +320,5 @@ describe('SearchService', () => {
 
         FS.chdir('/');
         FS.unmount('/' + localdir);
-
-        console.log(searchService.api.getXapianDocCount(), 'docs in xapian db');
     });
 });
diff --git a/src/app/xapian/searchservice.ts b/src/app/xapian/searchservice.ts
@@ -671,7 +671,7 @@ export class SearchService {
     ).map((pair: number[]) => pair[0]);
   }
 
-    checkIfDownloadableIndexExists(): Observable<boolean> {
+  checkIfDownloadableIndexExists(): Observable<boolean> {
       return this.httpclient.get('/mail/download_xapian_index?exists=check').pipe(
             map((stat: any) => {
               this.serverIndexSize = stat.size;