diff --git a/packages/cozy-dataproxy-lib/src/search/SearchEngine.ts b/packages/cozy-dataproxy-lib/src/search/SearchEngine.ts index 8e5b12c0e5..ff9b48d58d 100644 --- a/packages/cozy-dataproxy-lib/src/search/SearchEngine.ts +++ b/packages/cozy-dataproxy-lib/src/search/SearchEngine.ts @@ -22,7 +22,7 @@ import { shouldKeepFile } from './helpers/normalizeFile' import { normalizeSearchResult } from './helpers/normalizeSearchResult' -import { queryAllDocs, queryFilesForSearch } from './queries' +import { queryAllDocs, queryFilesForSearch, queryDocsByIds } from './queries' import { CozyDoc, RawSearchResult, @@ -32,13 +32,14 @@ import { SearchIndex, SearchIndexes, SearchResult, - isSearchedDoctype + isSearchedDoctype, + EnrichedSearchResult } from './types' const log = Minilog('🗂️ [Indexing]') interface FlexSearchResultWithDoctype - extends FlexSearch.EnrichedDocumentSearchResultSetUnit { + extends FlexSearch.SimpleDocumentSearchResultSetUnit { doctype: SearchedDoctype } @@ -179,14 +180,13 @@ export class SearchEngine { const fieldsToIndex = SEARCH_SCHEMA[doctype] const flexsearchIndex = new FlexSearch.Document({ - tokenize: 'forward', + tokenize: 'full', encode: getSearchEncoder(), // @ts-expect-error minlength is not described by Flexsearch types but exists minlength: 2, document: { id: '_id', - index: fieldsToIndex, - store: true + index: fieldsToIndex } }) @@ -316,7 +316,7 @@ export class SearchEngine { return this.incrementalIndexation(doctype, searchIndex) } - search(query: string): SearchResult[] { + async search(query: string): Promise { if (!this.searchIndexes) { // TODO: What if the indexing is running but not finished yet? log.warn('[SEARCH] No search index available') @@ -325,7 +325,8 @@ export class SearchEngine { const allResults = this.searchOnIndexes(query) const dedupResults = this.deduplicateAndFlatten(allResults) - const sortedResults = this.sortSearchResults(dedupResults) + const enrichedResults = await this.enrichResults(dedupResults) + const sortedResults = this.sortSearchResults(enrichedResults) const results = this.limitSearchResults(sortedResults) const normResults: SearchResult[] = [] @@ -359,8 +360,25 @@ export class SearchEngine { const FLEXSEARCH_LIMIT = 10000 const indexResults = index.index.search(query, FLEXSEARCH_LIMIT, { limit: FLEXSEARCH_LIMIT, - enrich: true + enrich: false }) + /* + Search result example: + [ + { + "field": "displayName", + "result": [ + "604627c6bafee013ec5f27f7f72029f6" + ] + }, + { + "field": "fullname", + "result": [ + "604627c6bafee013ec5f27f7f72029f6", "604627c6bafee013ec5f27f3f714568" + ] + } + ] + */ const newResults = indexResults.map(res => ({ ...res, @@ -376,30 +394,82 @@ export class SearchEngine { searchResults: FlexSearchResultWithDoctype[] ): RawSearchResult[] { const combinedResults = searchResults.flatMap(item => - item.result.map(r => ({ ...r, field: item.field, doctype: item.doctype })) + item.result.map(id => ({ + id: id.toString(), // Because of flexsearch Id typing + doctype: item.doctype, + field: item.field + })) ) - type MapItem = Omit<(typeof combinedResults)[number], 'field'> & { - fields: string[] - } - const resultMap = new Map() + const resultMap = new Map() - combinedResults.forEach(({ id, field, ...rest }) => { + combinedResults.forEach(({ id, field, doctype }) => { if (resultMap.has(id)) { resultMap.get(id)?.fields.push(field) } else { - resultMap.set(id, { id, fields: [field], ...rest }) + resultMap.set(id, { id, fields: [field], doctype }) } }) - return [...resultMap.values()] } + async enrichResults( + results: RawSearchResult[] + ): Promise { + const enrichedResults = [...results] as EnrichedSearchResult[] + + // Group by doctype + const resultsByDoctype = results.reduce>( + (acc, { id, doctype }) => { + if (!acc[doctype]) { + acc[doctype] = [] + } + acc[doctype].push(id) + return acc + }, + {} + ) + let docs = [] as CozyDoc[] + for (const doctype of Object.keys(resultsByDoctype)) { + const ids = resultsByDoctype[doctype] + + const startQuery = performance.now() + let queryDocs + // Query docs directly from store, for better performances + queryDocs = await queryDocsByIds(this.client, doctype, ids, { + fromStore: true + }) + if (queryDocs.length < 1) { + log.warn('Ids not found on store: query PouchDB') + // This should not happen, but let's add a fallback to query Pouch in case the store + // returned nothing. This is not done by default, as querying PouchDB is much slower. + queryDocs = await queryDocsByIds(this.client, doctype, ids, { + fromStore: false + }) + } + const endQuery = performance.now() + docs = docs.concat(queryDocs) + log.debug(`Query took ${(endQuery - startQuery).toFixed(2)} ms`) + } + for (const res of enrichedResults) { + const id = res.id?.toString() // Because of flexsearch Id typing + const doc = docs?.find(doc => doc._id === id) + if (!doc) { + log.error(`${id} is found in search but not in local data`) + } else { + res.doc = doc + } + } + return enrichedResults + } + compareStrings(str1: string, str2: string): number { return str1.localeCompare(str2, undefined, { numeric: true }) } - sortSearchResults(searchResults: RawSearchResult[]): RawSearchResult[] { + sortSearchResults( + searchResults: EnrichedSearchResult[] + ): EnrichedSearchResult[] { return searchResults.sort((a, b) => { const doctypeComparison = DOCTYPE_ORDER[a.doctype] - DOCTYPE_ORDER[b.doctype] @@ -428,7 +498,7 @@ export class SearchEngine { }) } - sortFiles(aRes: RawSearchResult, bRes: RawSearchResult): number { + sortFiles(aRes: EnrichedSearchResult, bRes: EnrichedSearchResult): number { if (!isIOCozyFile(aRes.doc) || !isIOCozyFile(bRes.doc)) { return 0 } @@ -444,7 +514,9 @@ export class SearchEngine { return this.compareStrings(aRes.doc.name, bRes.doc.name) } - limitSearchResults(searchResults: RawSearchResult[]): RawSearchResult[] { + limitSearchResults( + searchResults: EnrichedSearchResult[] + ): EnrichedSearchResult[] { return searchResults.slice(0, LIMIT_DOCTYPE_SEARCH) } } diff --git a/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.spec.ts b/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.spec.ts index 9ff25d5bc5..0d49377673 100644 --- a/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.spec.ts +++ b/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.spec.ts @@ -3,7 +3,7 @@ import { IOCozyContact, IOCozyFile } from 'cozy-client/types/types' import { cleanFilePath, normalizeSearchResult } from './normalizeSearchResult' import { FILES_DOCTYPE } from '../consts' -import { RawSearchResult } from '../types' +import { EnrichedSearchResult } from '../types' const fakeFlatDomainClient = { getStackClient: () => ({ @@ -27,7 +27,7 @@ describe('Should normalize files results', () => { const searchResult = { doctype: 'io.cozy.files', doc: doc - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -61,7 +61,7 @@ describe('Should normalize files results', () => { const searchResult = { doctype: 'io.cozy.files', doc: doc - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -91,7 +91,7 @@ describe('Should normalize files results', () => { const searchResult = { doctype: 'io.cozy.files', doc: doc - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -122,7 +122,7 @@ describe('Should normalize contacts results', () => { doctype: 'io.cozy.files', doc: doc, fields: ['displayName', 'jobTitle'] - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -151,7 +151,7 @@ describe('Should normalize contacts results', () => { doctype: 'io.cozy.files', doc: doc, fields: ['displayName', 'jobTitle'] - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -179,7 +179,7 @@ describe('Should normalize contacts results', () => { doctype: 'io.cozy.files', doc: doc, fields: ['displayName', 'jobTitle'] - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -208,7 +208,7 @@ describe('Should normalize contacts results', () => { doctype: 'io.cozy.files', doc: doc, fields: [] - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -241,7 +241,7 @@ describe('Should normalize contacts results', () => { doctype: 'io.cozy.files', doc: doc, fields: ['displayName', 'email[]:address'] - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -277,7 +277,7 @@ describe('Should normalize apps results', () => { doctype: 'io.cozy.files', doc: doc, fields: ['displayName', 'email[]:address'] - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -306,7 +306,7 @@ describe('Should normalize apps results', () => { doctype: 'io.cozy.files', doc: doc, fields: ['displayName', 'email[]:address'] - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, @@ -337,7 +337,7 @@ describe('Should normalize unknown doctypes', () => { doctype: 'io.cozy.files', doc: doc, fields: ['displayName', 'email[]:address'] - } as unknown as RawSearchResult + } as unknown as EnrichedSearchResult const result = normalizeSearchResult( fakeFlatDomainClient, diff --git a/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.ts b/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.ts index 636a46b8fe..2f96b67c26 100644 --- a/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.ts +++ b/packages/cozy-dataproxy-lib/src/search/helpers/normalizeSearchResult.ts @@ -4,7 +4,7 @@ import { IOCozyContact } from 'cozy-client/types/types' import { APPS_DOCTYPE, TYPE_DIRECTORY } from '../consts' import { CozyDoc, - RawSearchResult, + EnrichedSearchResult, isIOCozyApp, isIOCozyContact, isIOCozyFile, @@ -13,7 +13,7 @@ import { export const normalizeSearchResult = ( client: CozyClient, - searchResults: RawSearchResult, + searchResults: EnrichedSearchResult, query: string ): SearchResult => { const doc = cleanFilePath(searchResults.doc) diff --git a/packages/cozy-dataproxy-lib/src/search/queries/index.ts b/packages/cozy-dataproxy-lib/src/search/queries/index.ts index f30d46827e..44a1cc8d1e 100644 --- a/packages/cozy-dataproxy-lib/src/search/queries/index.ts +++ b/packages/cozy-dataproxy-lib/src/search/queries/index.ts @@ -21,6 +21,10 @@ interface QueryResponseSingleDoc { data: CozyDoc } +interface QueryResponseMultipleDoc { + data: CozyDoc[] +} + export const queryFilesForSearch = async ( client: CozyClient ): Promise => { @@ -58,3 +62,22 @@ export const queryDocById = async ( })) as QueryResponseSingleDoc return resp.data } + +export const queryDocsByIds = async ( + client: CozyClient, + doctype: string, + ids: string[], + { fromStore = true } = {} +): Promise => { + if (fromStore) { + // This is much more efficient to query from store than PouchDB + const allDocs = client.getCollectionFromState(doctype) + const docs = allDocs.filter(doc => doc._id && ids.includes(doc._id)) + return docs as CozyDoc[] + } + + const resp = (await client.query( + Q(doctype).getByIds(ids) + )) as QueryResponseMultipleDoc + return resp.data +} diff --git a/packages/cozy-dataproxy-lib/src/search/types.ts b/packages/cozy-dataproxy-lib/src/search/types.ts index b7e2928428..eb21709b6c 100644 --- a/packages/cozy-dataproxy-lib/src/search/types.ts +++ b/packages/cozy-dataproxy-lib/src/search/types.ts @@ -35,10 +35,14 @@ export const isSearchedDoctype = ( return searchedDoctypes.includes(doctype) } -export interface RawSearchResult - extends FlexSearch.EnrichedDocumentSearchResultSetUnitResultUnit { +export interface RawSearchResult { fields: string[] doctype: SearchedDoctype + id: string +} + +export interface EnrichedSearchResult extends RawSearchResult { + doc: CozyDoc } export interface SearchResult {