fix: Revert "feat: Do not use flexsearch store"

This reverts commit 104ea85. As we dynamically compute the path, that is not stored on PouchDB, it is required to store it in the cozy-client store and thus update all the files accordingly with the path. However, this is not usual to do so and might lead to unexpected side-effects. So for safety, we revert this commit for now and might rethink this later.
cozy · Nov 26, 2024 · 43b7b4b · 43b7b4b
1 parent bbdc587
commit 43b7b4b
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 134 deletions.
diff --git a/packages/cozy-dataproxy-lib/src/search/SearchEngine.ts b/packages/cozy-dataproxy-lib/src/search/SearchEngine.ts
@@ -22,7 +22,7 @@ import {
   shouldKeepFile
 } from './helpers/normalizeFile'
 import { normalizeSearchResult } from './helpers/normalizeSearchResult'
-import { queryAllDocs, queryFilesForSearch, queryDocsByIds } from './queries'
+import { queryAllDocs, queryFilesForSearch } from './queries'
 import {
   CozyDoc,
   RawSearchResult,
@@ -32,14 +32,13 @@ import {
   SearchIndex,
   SearchIndexes,
   SearchResult,
-  isSearchedDoctype,
-  EnrichedSearchResult
+  isSearchedDoctype
 } from './types'
 
 const log = Minilog('🗂️ [Indexing]')
 
 interface FlexSearchResultWithDoctype
-  extends FlexSearch.SimpleDocumentSearchResultSetUnit {
+  extends FlexSearch.EnrichedDocumentSearchResultSetUnit<CozyDoc> {
   doctype: SearchedDoctype
 }
 
@@ -186,7 +185,8 @@ export class SearchEngine {
       minlength: 2,
       document: {
         id: '_id',
-        index: fieldsToIndex
+        index: fieldsToIndex,
+        store: true
       }
     })
 
@@ -316,7 +316,7 @@ export class SearchEngine {
     return this.incrementalIndexation(doctype, searchIndex)
   }
 
-  async search(query: string): Promise<SearchResult[]> {
+  search(query: string): SearchResult[] {
     if (!this.searchIndexes) {
       // TODO: What if the indexing is running but not finished yet?
       log.warn('[SEARCH] No search index available')
@@ -325,8 +325,7 @@ export class SearchEngine {
 
     const allResults = this.searchOnIndexes(query)
     const dedupResults = this.deduplicateAndFlatten(allResults)
-    const enrichedResults = await this.enrichResults(dedupResults)
-    const sortedResults = this.sortSearchResults(enrichedResults)
+    const sortedResults = this.sortSearchResults(dedupResults)
     const results = this.limitSearchResults(sortedResults)
 
     const normResults: SearchResult[] = []
@@ -360,25 +359,8 @@ export class SearchEngine {
       const FLEXSEARCH_LIMIT = 10000
       const indexResults = index.index.search(query, FLEXSEARCH_LIMIT, {
         limit: FLEXSEARCH_LIMIT,
-        enrich: false
+        enrich: true
       })
-      /*
-        Search result example:
-        [
-          {
-              "field": "displayName",
-              "result": [
-                  "604627c6bafee013ec5f27f7f72029f6"
-              ]
-          },
-          {
-              "field": "fullname",
-              "result": [
-                  "604627c6bafee013ec5f27f7f72029f6", "604627c6bafee013ec5f27f3f714568"
-              ]
-          }
-        ]
-      */
 
       const newResults = indexResults.map(res => ({
         ...res,
@@ -394,82 +376,30 @@ export class SearchEngine {
     searchResults: FlexSearchResultWithDoctype[]
   ): RawSearchResult[] {
     const combinedResults = searchResults.flatMap(item =>
-      item.result.map(id => ({
-        id: id.toString(), // Because of flexsearch Id typing
-        doctype: item.doctype,
-        field: item.field
-      }))
+      item.result.map(r => ({ ...r, field: item.field, doctype: item.doctype }))
     )
 
-    const resultMap = new Map<string, RawSearchResult>()
+    type MapItem = Omit<(typeof combinedResults)[number], 'field'> & {
+      fields: string[]
+    }
+    const resultMap = new Map<FlexSearch.Id[], MapItem>()
 
-    combinedResults.forEach(({ id, field, doctype }) => {
+    combinedResults.forEach(({ id, field, ...rest }) => {
       if (resultMap.has(id)) {
         resultMap.get(id)?.fields.push(field)
       } else {
-        resultMap.set(id, { id, fields: [field], doctype })
+        resultMap.set(id, { id, fields: [field], ...rest })
       }
     })
-    return [...resultMap.values()]
-  }
 
-  async enrichResults(
-    results: RawSearchResult[]
-  ): Promise<EnrichedSearchResult[]> {
-    const enrichedResults = [...results] as EnrichedSearchResult[]
-
-    // Group by doctype
-    const resultsByDoctype = results.reduce<Record<string, string[]>>(
-      (acc, { id, doctype }) => {
-        if (!acc[doctype]) {
-          acc[doctype] = []
-        }
-        acc[doctype].push(id)
-        return acc
-      },
-      {}
-    )
-    let docs = [] as CozyDoc[]
-    for (const doctype of Object.keys(resultsByDoctype)) {
-      const ids = resultsByDoctype[doctype]
-
-      const startQuery = performance.now()
-      let queryDocs
-      // Query docs directly from store, for better performances
-      queryDocs = await queryDocsByIds(this.client, doctype, ids, {
-        fromStore: true
-      })
-      if (queryDocs.length < 1) {
-        log.warn('Ids not found on store: query PouchDB')
-        // This should not happen, but let's add a fallback to query Pouch in case the store
-        // returned nothing. This is not done by default, as querying PouchDB is much slower.
-        queryDocs = await queryDocsByIds(this.client, doctype, ids, {
-          fromStore: false
-        })
-      }
-      const endQuery = performance.now()
-      docs = docs.concat(queryDocs)
-      log.debug(`Query took ${(endQuery - startQuery).toFixed(2)} ms`)
-    }
-    for (const res of enrichedResults) {
-      const id = res.id?.toString() // Because of flexsearch Id typing
-      const doc = docs?.find(doc => doc._id === id)
-      if (!doc) {
-        log.error(`${id} is found in search but not in local data`)
-      } else {
-        res.doc = doc
-      }
-    }
-    return enrichedResults
+    return [...resultMap.values()]
   }
 
   compareStrings(str1: string, str2: string): number {
     return str1.localeCompare(str2, undefined, { numeric: true })
   }
 
-  sortSearchResults(
-    searchResults: EnrichedSearchResult[]
-  ): EnrichedSearchResult[] {
+  sortSearchResults(searchResults: RawSearchResult[]): RawSearchResult[] {
     return searchResults.sort((a, b) => {
       const doctypeComparison =
         DOCTYPE_ORDER[a.doctype] - DOCTYPE_ORDER[b.doctype]
@@ -498,7 +428,7 @@ export class SearchEngine {
     })
   }
 
-  sortFiles(aRes: EnrichedSearchResult, bRes: EnrichedSearchResult): number {
+  sortFiles(aRes: RawSearchResult, bRes: RawSearchResult): number {
     if (!isIOCozyFile(aRes.doc) || !isIOCozyFile(bRes.doc)) {
       return 0
     }
@@ -514,9 +444,7 @@ export class SearchEngine {
     return this.compareStrings(aRes.doc.name, bRes.doc.name)
   }
 
-  limitSearchResults(
-    searchResults: EnrichedSearchResult[]
-  ): EnrichedSearchResult[] {
+  limitSearchResults(searchResults: RawSearchResult[]): RawSearchResult[] {
     return searchResults.slice(0, LIMIT_DOCTYPE_SEARCH)
   }
 }
diff --git a/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.spec.ts b/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.spec.ts
@@ -3,7 +3,7 @@ import { IOCozyContact, IOCozyFile } from 'cozy-client/types/types'
 
 import { cleanFilePath, normalizeSearchResult } from './normalizeSearchResult'
 import { FILES_DOCTYPE } from '../consts'
-import { EnrichedSearchResult } from '../types'
+import { RawSearchResult } from '../types'
 
 const fakeFlatDomainClient = {
   getStackClient: () => ({
@@ -27,7 +27,7 @@ describe('Should normalize files results', () => {
     const searchResult = {
       doctype: 'io.cozy.files',
       doc: doc
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -61,7 +61,7 @@ describe('Should normalize files results', () => {
     const searchResult = {
       doctype: 'io.cozy.files',
       doc: doc
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -91,7 +91,7 @@ describe('Should normalize files results', () => {
     const searchResult = {
       doctype: 'io.cozy.files',
       doc: doc
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -122,7 +122,7 @@ describe('Should normalize contacts results', () => {
       doctype: 'io.cozy.files',
       doc: doc,
       fields: ['displayName', 'jobTitle']
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -151,7 +151,7 @@ describe('Should normalize contacts results', () => {
       doctype: 'io.cozy.files',
       doc: doc,
       fields: ['displayName', 'jobTitle']
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -179,7 +179,7 @@ describe('Should normalize contacts results', () => {
       doctype: 'io.cozy.files',
       doc: doc,
       fields: ['displayName', 'jobTitle']
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -208,7 +208,7 @@ describe('Should normalize contacts results', () => {
       doctype: 'io.cozy.files',
       doc: doc,
       fields: []
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -241,7 +241,7 @@ describe('Should normalize contacts results', () => {
       doctype: 'io.cozy.files',
       doc: doc,
       fields: ['displayName', 'email[]:address']
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -277,7 +277,7 @@ describe('Should normalize apps results', () => {
       doctype: 'io.cozy.files',
       doc: doc,
       fields: ['displayName', 'email[]:address']
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -306,7 +306,7 @@ describe('Should normalize apps results', () => {
       doctype: 'io.cozy.files',
       doc: doc,
       fields: ['displayName', 'email[]:address']
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,
@@ -337,7 +337,7 @@ describe('Should normalize unknown doctypes', () => {
       doctype: 'io.cozy.files',
       doc: doc,
       fields: ['displayName', 'email[]:address']
-    } as unknown as EnrichedSearchResult
+    } as unknown as RawSearchResult
 
     const result = normalizeSearchResult(
       fakeFlatDomainClient,

diff --git a/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.ts b/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.ts
@@ -4,7 +4,7 @@ import { IOCozyContact } from 'cozy-client/types/types'
 import { APPS_DOCTYPE, TYPE_DIRECTORY } from '../consts'
 import {
   CozyDoc,
-  EnrichedSearchResult,
+  RawSearchResult,
   isIOCozyApp,
   isIOCozyContact,
   isIOCozyFile,
@@ -13,7 +13,7 @@ import {
 
 export const normalizeSearchResult = (
   client: CozyClient,
-  searchResults: EnrichedSearchResult,
+  searchResults: RawSearchResult,
   query: string
 ): SearchResult => {
   const doc = cleanFilePath(searchResults.doc)

diff --git a/packages/cozy-dataproxy-lib/src/search/queries/index.ts b/packages/cozy-dataproxy-lib/src/search/queries/index.ts
@@ -21,10 +21,6 @@ interface QueryResponseSingleDoc {
   data: CozyDoc
 }
 
-interface QueryResponseMultipleDoc {
-  data: CozyDoc[]
-}
-
 export const queryFilesForSearch = async (
   client: CozyClient
 ): Promise<CozyDoc[]> => {
@@ -62,22 +58,3 @@ export const queryDocById = async (
   })) as QueryResponseSingleDoc
   return resp.data
 }
-
-export const queryDocsByIds = async (
-  client: CozyClient,
-  doctype: string,
-  ids: string[],
-  { fromStore = true } = {}
-): Promise<CozyDoc[]> => {
-  if (fromStore) {
-    // This is much more efficient to query from store than PouchDB
-    const allDocs = client.getCollectionFromState(doctype)
-    const docs = allDocs.filter(doc => doc._id && ids.includes(doc._id))
-    return docs as CozyDoc[]
-  }
-
-  const resp = (await client.query(
-    Q(doctype).getByIds(ids)
-  )) as QueryResponseMultipleDoc
-  return resp.data
-}
diff --git a/packages/cozy-dataproxy-lib/src/search/types.ts b/packages/cozy-dataproxy-lib/src/search/types.ts
@@ -35,14 +35,10 @@ export const isSearchedDoctype = (
   return searchedDoctypes.includes(doctype)
 }
 
-export interface RawSearchResult {
+export interface RawSearchResult
+  extends FlexSearch.EnrichedDocumentSearchResultSetUnitResultUnit<CozyDoc> {
   fields: string[]
   doctype: SearchedDoctype
-  id: string
-}
-
-export interface EnrichedSearchResult extends RawSearchResult {
-  doc: CozyDoc
 }
 
 export interface SearchResult {