From a07c2bf92b12caed5df626185cf539ea07f19b8d Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Tue, 15 Oct 2024 11:56:42 +0200 Subject: [PATCH 1/5] feat: Rely on PouchDB to build indexes We set up a PouchDB database to replicate doctypes on build the flexsearch indexes based on it. --- package.json | 2 + src/dataproxy/common/DataProxyInterface.ts | 2 +- src/dataproxy/worker/platformWorker.ts | 114 ++++++++++ src/dataproxy/worker/shared-worker.ts | 67 ++++-- src/search/SearchEngine.ts | 253 +++++++++++++++++++++ src/search/helpers/client.ts | 9 + src/search/initIndexes.ts | 68 ------ src/search/search.ts | 162 ------------- src/search/types.ts | 6 +- yarn.lock | 203 ++++++++++++++++- 10 files changed, 623 insertions(+), 263 deletions(-) create mode 100644 src/dataproxy/worker/platformWorker.ts create mode 100644 src/search/SearchEngine.ts create mode 100644 src/search/helpers/client.ts delete mode 100644 src/search/initIndexes.ts delete mode 100644 src/search/search.ts diff --git a/package.json b/package.json index 5d49adf..b0a6aa4 100644 --- a/package.json +++ b/package.json @@ -19,9 +19,11 @@ "cozy-flags": "^4.0.0", "cozy-logger": "^1.10.4", "cozy-minilog": "^3.3.1", + "cozy-pouch-link": "^49.5.0", "cozy-tsconfig": "^1.2.0", "flexsearch": "^0.7.43", "lodash": "^4.17.21", + "pouchdb-browser": "^9.0.0", "react": "^18.3.1", "react-dom": "^18.3.1" }, diff --git a/src/dataproxy/common/DataProxyInterface.ts b/src/dataproxy/common/DataProxyInterface.ts index 24360c9..b674a25 100644 --- a/src/dataproxy/common/DataProxyInterface.ts +++ b/src/dataproxy/common/DataProxyInterface.ts @@ -1,7 +1,7 @@ import type { InstanceOptions } from 'cozy-client' import type { ClientCapabilities } from 'cozy-client/types/types' -export type { SearchIndex } from '@/search/types' +export type { SearchIndexes } from '@/search/types' export interface DataProxyWorker { search: (query: string) => Promise diff --git a/src/dataproxy/worker/platformWorker.ts b/src/dataproxy/worker/platformWorker.ts new file mode 100644 index 0000000..b39a83e --- /dev/null +++ b/src/dataproxy/worker/platformWorker.ts @@ -0,0 +1,114 @@ +import PouchDB from 'pouchdb-browser' + +const dbName = 'sharedWorkerStorage' +let db: IDBDatabase | null = null + +const openDB = (): Promise => { + return new Promise((resolve, reject) => { + if (db) { + return resolve(db) + } + + const request = indexedDB.open(dbName, 1) + + request.onupgradeneeded = (event: IDBVersionChangeEvent): void => { + const database = (event.target as IDBOpenDBRequest).result + db = database + if (!db.objectStoreNames.contains('store')) { + db.createObjectStore('store', { keyPath: 'key' }) + } + } + + request.onsuccess = (event: Event): void => { + db = (event.target as IDBOpenDBRequest).result + resolve(db) + } + + request.onerror = (event: Event): void => { + reject((event.target as IDBOpenDBRequest).error) + } + }) +} + +// Define the storage object with TypeScript types +const storage = { + getItem: async (key: string): Promise => { + const db = await openDB() + return new Promise((resolve, reject) => { + const transaction = db.transaction('store', 'readonly') + const store = transaction.objectStore('store') + const request = store.get(key) + + request.onsuccess = (): void => { + resolve(request.result ? request.result.value : null) + } + + request.onerror = (): void => { + reject(request.error) + } + }) + }, + + setItem: async (key: string, value: any): Promise => { + const db = await openDB() + return new Promise((resolve, reject) => { + const transaction = db.transaction('store', 'readwrite') + const store = transaction.objectStore('store') + const request = store.put({ key, value }) + + request.onsuccess = (): void => { + resolve() + } + + request.onerror = (): void => { + reject(request.error) + } + }) + }, + + removeItem: async (key: string): Promise => { + const db = await openDB() + return new Promise((resolve, reject) => { + const transaction = db.transaction('store', 'readwrite') + const store = transaction.objectStore('store') + const request = store.delete(key) + + request.onsuccess = (): void => { + resolve() + } + + request.onerror = (): void => { + reject(request.error) + } + }) + } +} + +// Define the event handling object with proper types +const events = { + addEventListener: ( + eventName: string, + handler: EventListenerOrEventListenerObject + ): void => { + self.addEventListener(eventName, handler) + }, + removeEventListener: ( + eventName: string, + handler: EventListenerOrEventListenerObject + ): void => { + self.removeEventListener(eventName, handler) + } +} + +// Define the isOnline function with the proper type +const isOnline = async (): Promise => { + return self.navigator.onLine +} + +// Export the platformWorker object with TypeScript types +export const platformWorker = { + storage, + events, + pouchAdapter: PouchDB, + isOnline +} diff --git a/src/dataproxy/worker/shared-worker.ts b/src/dataproxy/worker/shared-worker.ts index b8d7fb6..4723506 100644 --- a/src/dataproxy/worker/shared-worker.ts +++ b/src/dataproxy/worker/shared-worker.ts @@ -2,22 +2,23 @@ import * as Comlink from 'comlink' import CozyClient from 'cozy-client' import Minilog from 'cozy-minilog' +import PouchLink from 'cozy-pouch-link' import { ClientData, DataProxyWorker, - DataProxyWorkerPartialState, - SearchIndex + DataProxyWorkerPartialState } from '@/dataproxy/common/DataProxyInterface' +import { platformWorker } from '@/dataproxy/worker/platformWorker' import schema from '@/doctypes' -import { initIndexes } from '@/search/initIndexes' -import { search } from '@/search/search' +import SearchEngine from '@/search/SearchEngine' +import { FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE } from '@/search/consts' const log = Minilog('👷‍♂️ [shared-worker]') Minilog.enable() let client: CozyClient | undefined = undefined -let searchIndexes: SearchIndex[] | undefined = undefined +let searchEngine: SearchEngine = null const broadcastChannel = new BroadcastChannel('DATA_PROXY_BROADCAST_CHANANEL') @@ -26,6 +27,24 @@ const dataProxy: DataProxyWorker = { log.debug('Received data for setting client') if (client) return updateState() + + const pouchLinkOptions = { + doctypes: [FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE], + initialSync: true, + platform: { ...platformWorker }, + doctypesReplicationOptions: { + [FILES_DOCTYPE]: { + strategy: 'fromRemote' + }, + [CONTACTS_DOCTYPE]: { + strategy: 'fromRemote' + }, + [APPS_DOCTYPE]: { + strategy: 'fromRemote' + } + } + } + client = new CozyClient({ uri: clientData.uri, token: clientData.token, @@ -34,43 +53,45 @@ const dataProxy: DataProxyWorker = { version: '1' }, schema, - store: true + store: true, + links: [new PouchLink(pouchLinkOptions)] }) client.instanceOptions = clientData.instanceOptions client.capabilities = clientData.capabilities - if (!searchIndexes) { - const indexes = await initIndexes(client) - searchIndexes = indexes - updateState() - } + + searchEngine = new SearchEngine(client) + updateState() }, + search: async (query: string) => { log.debug('Received data for search') if (!client) { throw new Error( - 'Client is required to execute a seach, please initialize CozyClient' + 'Client is required to execute a search, please initialize CozyClient' ) } - - if (!searchIndexes) { - return [] + if (!searchEngine) { + throw new Error('SearchEngine is not initialized') } - return search(query, searchIndexes, client) + return searchEngine.search(query) } } const updateState = (): void => { const state = {} as DataProxyWorkerPartialState - if (client && searchIndexes) { + if (client && searchEngine && searchEngine.searchIndexes) { state.status = 'Ready' - state.indexLength = searchIndexes.map(searchIndex => ({ - doctype: searchIndex.doctype, - // @ts-expect-error index.store is not TS typed - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - count: Object.keys(searchIndex.index.store).length - })) + state.indexLength = Object.keys(searchEngine.searchIndexes).map( + (indexKey: string) => ({ + doctype: indexKey, + // @ts-expect-error index.store is not TS typed + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + count: Object.keys(searchEngine.searchIndexes[indexKey].index.store) + .length + }) + ) broadcastChannel.postMessage(state) return } diff --git a/src/search/SearchEngine.ts b/src/search/SearchEngine.ts new file mode 100644 index 0000000..c7b5363 --- /dev/null +++ b/src/search/SearchEngine.ts @@ -0,0 +1,253 @@ +import FlexSearch from 'flexsearch' +// @ts-ignore +import { encode as encode_balance } from 'flexsearch/dist/module/lang/latin/balance' + +import CozyClient, { Q } from 'cozy-client' +import Minilog from 'cozy-minilog' + +import { + SEARCH_SCHEMA, + APPS_DOCTYPE, + FILES_DOCTYPE, + CONTACTS_DOCTYPE, + DOCTYPE_ORDER, + LIMIT_DOCTYPE_SEARCH +} from '@/search/consts' +import { getPouchLink } from '@/search/helpers/client' +import { normalizeSearchResult } from '@/search/helpers/normalizeSearchResult' +import { + queryFilesForSearch, + queryAllContacts, + queryAllApps +} from '@/search/queries' +import { + CozyDoc, + RawSearchResult, + isIOCozyApp, + isIOCozyContact, + isIOCozyFile, + SearchedDoctype, + SearchIndex, + SearchIndexes, + SearchResult +} from '@/search/types' + +const log = Minilog('🗂️ [Indexing]') + +interface FlexSearchResultWithDoctype + extends FlexSearch.EnrichedDocumentSearchResultSetUnit { + doctype: SearchedDoctype +} + +class SearchEngine { + client: CozyClient + searchIndexes: SearchIndexes + + constructor(client: CozyClient) { + this.client = client + this.searchIndexes = {} + } + + buildSearchIndex( + doctype: keyof typeof SEARCH_SCHEMA, + docs: CozyDoc[] + ): FlexSearch.Document { + const fieldsToIndex = SEARCH_SCHEMA[doctype] + + const flexsearchIndex = new FlexSearch.Document({ + tokenize: 'forward', + encode: encode_balance as FlexSearch.Encoders, + minlength: 2, + document: { + id: '_id', + index: fieldsToIndex, + store: true + } + }) + + for (const doc of docs) { + flexsearchIndex.add(doc) + } + + return flexsearchIndex + } + + async indexDocsForSearch(doctype: string): Promise { + const searchIndex = this.searchIndexes[doctype] + const pouchLink = getPouchLink(this.client) + + if (!pouchLink) return null + + if (!searchIndex) { + const docs = await this.client.queryAll(Q(doctype).limitBy(null)) + const index = this.buildSearchIndex(doctype, docs) + const info = await pouchLink.getDbInfo(doctype) + + this.searchIndexes[doctype] = { + index, + lastSeq: info?.update_seq + } + return this.searchIndexes[doctype] + } + + const lastSeq = searchIndex.lastSeq || 0 + const changes = await pouchLink.getChanges(doctype, { + include_docs: true, + since: lastSeq + }) + + for (const change of changes.results) { + if (change.deleted) { + searchIndex.index.remove(change.id) + } else { + const normalizedDoc = { ...change.doc, _type: doctype } + searchIndex.index.add(normalizedDoc) + } + } + + searchIndex.lastSeq = changes.last_seq + return searchIndex + } + + initIndexesFromStack = async (): Promise => { + log.debug('Initializing indexes') + + const files = await queryFilesForSearch(this.client) + const filesIndex = this.buildSearchIndex('io.cozy.files', files) + + const contacts = await queryAllContacts(this.client) + const contactsIndex = this.buildSearchIndex('io.cozy.contacts', contacts) + + const apps = await queryAllApps(this.client) + const appsIndex = this.buildSearchIndex('io.cozy.apps', apps) + + log.debug('Finished initializing indexes') + this.searchIndexes = { + [FILES_DOCTYPE]: { index: filesIndex, lastSeq: null }, + [CONTACTS_DOCTYPE]: { index: contactsIndex, lastSeq: null }, + [APPS_DOCTYPE]: { index: appsIndex, lastSeq: null } + } + return this.searchIndexes + } + + search(query: string): SearchResult[] { + log.debug('[SEARCH] indexes : ', this.searchIndexes) + + if (!this.searchIndexes) { + // TODO: What if the indexing is running but not finished yet? + log.warn('[SEARCH] No search index available') + return [] + } + + const allResults = this.searchOnIndexes(query) + const results = this.deduplicateAndFlatten(allResults) + const sortedResults = this.sortAndLimitSearchResults(results) + + return sortedResults + .map(res => normalizeSearchResult(this.client, res, query)) + .filter(res => res.title) + } + + searchOnIndexes(query: string): FlexSearchResultWithDoctype[] { + let searchResults: FlexSearchResultWithDoctype[] = [] + for (const doctype in this.searchIndexes) { + const index = this.searchIndexes[doctype] + if (!index) { + log.warn('[SEARCH] No search index available for ', doctype) + continue + } + const indexResults = index.index.search(query, LIMIT_DOCTYPE_SEARCH, { + enrich: true + }) + const newResults = indexResults.map(res => ({ + ...res, + doctype + })) + searchResults = searchResults.concat(newResults) + } + return searchResults + } + + deduplicateAndFlatten( + searchResults: FlexSearchResultWithDoctype[] + ): RawSearchResult[] { + const combinedResults = searchResults.flatMap(item => + item.result.map(r => ({ ...r, field: item.field, doctype: item.doctype })) + ) + + const resultMap = new Map() + + combinedResults.forEach(({ id, field, ...rest }) => { + if (resultMap.has(id)) { + resultMap.get(id).fields.push(field) + } else { + resultMap.set(id, { id, fields: [field], ...rest }) + } + }) + + return [...resultMap.values()] + } + + sortAndLimitSearchResults( + searchResults: RawSearchResult[] + ): RawSearchResult[] { + const sortedResults = this.sortSearchResults(searchResults) + return this.limitSearchResults(sortedResults) + } + + sortSearchResults(searchResults: RawSearchResult[]): RawSearchResult[] { + return searchResults.sort((a, b) => { + const doctypeComparison = + DOCTYPE_ORDER[a.doctype] - DOCTYPE_ORDER[b.doctype] + if (doctypeComparison !== 0) return doctypeComparison + + if ( + a.doctype === APPS_DOCTYPE && + isIOCozyApp(a.doc) && + isIOCozyApp(b.doc) + ) { + return a.doc.slug.localeCompare(b.doc.slug) + } else if ( + a.doctype === CONTACTS_DOCTYPE && + isIOCozyContact(a.doc) && + isIOCozyContact(b.doc) + ) { + return a.doc.displayName.localeCompare(b.doc.displayName) + } else if ( + a.doctype === FILES_DOCTYPE && + isIOCozyFile(a.doc) && + isIOCozyFile(b.doc) + ) { + if (a.doc.type !== b.doc.type) { + return a.doc.type === 'directory' ? -1 : 1 + } + return a.doc.name.localeCompare(b.doc.name) + } + + return 0 + }) + } + + limitSearchResults(searchResults: RawSearchResult[]): RawSearchResult[] { + const limitedResults = { + [APPS_DOCTYPE]: [], + [CONTACTS_DOCTYPE]: [], + [FILES_DOCTYPE]: [] + } + + searchResults.forEach(item => { + const type = item.doctype as SearchedDoctype + if (limitedResults[type].length < LIMIT_DOCTYPE_SEARCH) { + limitedResults[type].push(item) + } + }) + + return [ + ...limitedResults[APPS_DOCTYPE], + ...limitedResults[CONTACTS_DOCTYPE], + ...limitedResults[FILES_DOCTYPE] + ] + } +} + +export default SearchEngine diff --git a/src/search/helpers/client.ts b/src/search/helpers/client.ts new file mode 100644 index 0000000..198b0a6 --- /dev/null +++ b/src/search/helpers/client.ts @@ -0,0 +1,9 @@ +import CozyClient from 'cozy-client' +import PouchLink from 'cozy-pouch-link' + +export const getPouchLink = (client: CozyClient): PouchLink | null => { + if (!client) { + return null + } + return client.links.find(link => link instanceof PouchLink) || null +} diff --git a/src/search/initIndexes.ts b/src/search/initIndexes.ts deleted file mode 100644 index 2c5dd42..0000000 --- a/src/search/initIndexes.ts +++ /dev/null @@ -1,68 +0,0 @@ -import FlexSearch from 'flexsearch' -// @ts-ignore -import { encode as encode_balance } from 'flexsearch/dist/module/lang/latin/balance' - -import CozyClient from 'cozy-client' -import Minilog from 'cozy-minilog' - -import { - SEARCH_SCHEMA, - APPS_DOCTYPE, - FILES_DOCTYPE, - CONTACTS_DOCTYPE -} from '@/search/consts' -import { - queryFilesForSearch, - queryAllContacts, - queryAllApps -} from '@/search/queries' -import { CozyDoc, SearchIndex } from '@/search/types' - -const log = Minilog('🗂️ [Indexing]') - -export const initIndexes = async ( - client: CozyClient -): Promise => { - log.debug('Initializing indexes') - - const files = await queryFilesForSearch(client) - const filesIndex = indexDocs('io.cozy.files', files) - - const contacts = await queryAllContacts(client) - const contactsIndex = indexDocs('io.cozy.contacts', contacts) - - const apps = await queryAllApps(client) - const appsIndex = indexDocs('io.cozy.apps', apps) - - log.debug('Finished initializing indexes') - return [ - { index: filesIndex, doctype: FILES_DOCTYPE }, - { index: contactsIndex, doctype: CONTACTS_DOCTYPE }, - { index: appsIndex, doctype: APPS_DOCTYPE } - ] -} - -const indexDocs = ( - doctype: keyof typeof SEARCH_SCHEMA, - docs: CozyDoc[] -): FlexSearch.Document => { - const fieldsToIndex = SEARCH_SCHEMA[doctype] - - const flexsearchIndex = new FlexSearch.Document({ - tokenize: 'forward', - encode: encode_balance as FlexSearch.Encoders, - // @ts-expect-error IndexOptions.minlength is not TS typed - minlength: 2, - document: { - id: '_id', - index: fieldsToIndex, - store: true - } - }) - - for (const doc of docs) { - flexsearchIndex.add(doc) - } - - return flexsearchIndex -} diff --git a/src/search/search.ts b/src/search/search.ts deleted file mode 100644 index b5e324b..0000000 --- a/src/search/search.ts +++ /dev/null @@ -1,162 +0,0 @@ -import FlexSearch from 'flexsearch' - -import CozyClient from 'cozy-client' -import Minilog from 'cozy-minilog' - -import { - APPS_DOCTYPE, - FILES_DOCTYPE, - CONTACTS_DOCTYPE, - DOCTYPE_ORDER, - LIMIT_DOCTYPE_SEARCH -} from '@/search/consts' -import { normalizeSearchResult } from '@/search/helpers/normalizeSearchResult' -import { - CozyDoc, - RawSearchResult, - isIOCozyApp, - isIOCozyContact, - isIOCozyFile, - SearchedDoctype, - SearchIndex, - SearchResult -} from '@/search/types' - -const log = Minilog('🗂️ [Indexing]') - -export const search = ( - query: string, - indexes: SearchIndex[], - client: CozyClient -): SearchResult[] => { - log.debug('[SEARCH] indexes : ', indexes) - - const allResults = searchOnIndexes(query, indexes) - log.debug('[SEARCH] results : ', allResults) - const results = deduplicateAndFlatten(allResults) - log.debug('[SEARCH] dedup : ', results) - const sortedResults = sortAndLimitSearchResults(results) - log.debug('[SEARCH] sort : ', sortedResults) - - return sortedResults.map(res => normalizeSearchResult(client, res, query)) -} - -interface FlexSearchResultWithDoctype - extends FlexSearch.EnrichedDocumentSearchResultSetUnit { - doctype: SearchedDoctype -} - -const searchOnIndexes = ( - query: string, - indexes: SearchIndex[] -): FlexSearchResultWithDoctype[] => { - log.debug('Searching on indexes') - let searchResults: FlexSearchResultWithDoctype[] = [] - for (const index of indexes) { - // FIXME: The given limit seems ignored? - const indexResults = index.index.search(query, LIMIT_DOCTYPE_SEARCH, { - enrich: true - }) - const newResults = indexResults.map(res => ({ - ...res, - doctype: index.doctype - })) - searchResults = searchResults.concat(newResults) - } - log.debug('Finished seaching on indexes') - return searchResults -} - -const deduplicateAndFlatten = ( - searchResults: FlexSearchResultWithDoctype[] -): RawSearchResult[] => { - const combinedResults = searchResults.flatMap(item => - item.result.map(r => ({ ...r, field: item.field, doctype: item.doctype })) - ) - - type MapItem = Omit<(typeof combinedResults)[number], 'field'> & { - fields: string[] - } - const resultMap = new Map() - - combinedResults.forEach(({ id, field, ...rest }) => { - if (resultMap.has(id)) { - // @ts-expect-error TODO - resultMap.get(id).fields.push(field) - } else { - resultMap.set(id, { id, fields: [field], ...rest }) - } - }) - - return [...resultMap.values()] -} - -const sortAndLimitSearchResults = ( - searchResults: RawSearchResult[] -): RawSearchResult[] => { - const sortedResults = sortSearchResults(searchResults) - return limitSearchResults(sortedResults) -} - -const sortSearchResults = ( - searchResults: RawSearchResult[] -): RawSearchResult[] => { - return searchResults.sort((a, b) => { - // First, sort by doctype order - const doctypeComparison = - DOCTYPE_ORDER[a.doctype] - DOCTYPE_ORDER[b.doctype] - if (doctypeComparison !== 0) return doctypeComparison - - // Then, sort within each doctype by the specified field - if ( - a.doctype === APPS_DOCTYPE && - isIOCozyApp(a.doc) && - isIOCozyApp(b.doc) - ) { - return a.doc.slug.localeCompare(b.doc.slug) - } else if ( - a.doctype === CONTACTS_DOCTYPE && - isIOCozyContact(a.doc) && - isIOCozyContact(b.doc) - ) { - return a.doc.displayName.localeCompare(b.doc.displayName) - } else if ( - a.doctype === FILES_DOCTYPE && - isIOCozyFile(a.doc) && - isIOCozyFile(b.doc) - ) { - if (a.doc.type !== b.doc.type) { - return a.doc.type === 'directory' ? -1 : 1 - } - return a.doc.name.localeCompare(b.doc.name) - } - - return 0 - }) -} - -const limitSearchResults = ( - searchResults: RawSearchResult[] -): RawSearchResult[] => { - const limitedResults: { - [id in SearchedDoctype]: RawSearchResult[] - } = { - [APPS_DOCTYPE]: [], - [CONTACTS_DOCTYPE]: [], - [FILES_DOCTYPE]: [] - } - // Limit the results, grouped by doctype - searchResults.forEach(item => { - const type = item.doctype as SearchedDoctype - - if (limitedResults[type].length < LIMIT_DOCTYPE_SEARCH) { - limitedResults[type].push(item) - } - }) - - return [ - ...limitedResults[APPS_DOCTYPE], - ...limitedResults[CONTACTS_DOCTYPE], - ...limitedResults[FILES_DOCTYPE] - ] -} diff --git a/src/search/types.ts b/src/search/types.ts index c693183..4bcf3cd 100644 --- a/src/search/types.ts +++ b/src/search/types.ts @@ -47,5 +47,9 @@ export interface SearchResult { export interface SearchIndex { index: FlexSearch.Document - doctype: SearchedDoctype + lastSeq: number +} + +export type SearchIndexes = { + [key: string]: SearchIndex } diff --git a/yarn.lock b/yarn.lock index 88cb30c..b0c4e53 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1420,7 +1420,7 @@ abab@^2.0.6: resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291" integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA== -abort-controller@^3.0.0: +abort-controller@3.0.0, abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== @@ -1525,6 +1525,11 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== +argsarray@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/argsarray/-/argsarray-0.0.1.tgz#6e7207b4ecdb39b0af88303fa5ae22bda8df61cb" + integrity sha512-u96dg2GcAKtpTrBdDoFIM7PjcBA+6rSP0OR94MOReNRyUECL6MtQt5XXmRr4qrftYaef9+l5hcpO5te7sML1Cg== + aria-query@5.1.3: version "5.1.3" resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.1.3.tgz#19db27cd101152773631396f7a95a3b58c22c35e" @@ -1927,7 +1932,7 @@ btoa@^1.2.1: resolved "https://registry.yarnpkg.com/btoa/-/btoa-1.2.1.tgz#01a9909f8b2c93f6bf680ba26131eb30f7fa3d73" integrity sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g== -buffer-from@^1.0.0: +buffer-from@1.1.2, buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== @@ -2061,6 +2066,11 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" +clone-buffer@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/clone-buffer/-/clone-buffer-1.0.0.tgz#e3e25b207ac4e701af721e2cb5a16792cac3dc58" + integrity sha512-KLLTJWrvwIP+OPfMn0x2PheDEP20RPUcGXj/ERegTgdmPEZylALQldygiqrPPu8P45uNuPs7ckmReLY6v/iA5g== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -2188,6 +2198,31 @@ cozy-client@^49.0.0: sift "^6.0.0" url-search-params-polyfill "^8.0.0" +cozy-client@^49.4.0: + version "49.4.0" + resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-49.4.0.tgz#248788e5e7a3595dd9137f5d0563ac6a5f5c0231" + integrity sha512-jIapZfMzJzCblI4pKnir3nXgOvrXavsRXkm7QB2qbDl9WSCJcpKvMw8Z1yDsma7nB/16E0rKzeEBh+BxXbStTw== + dependencies: + "@cozy/minilog" "1.0.0" + "@types/jest" "^26.0.20" + "@types/lodash" "^4.14.170" + btoa "^1.2.1" + cozy-stack-client "^49.4.0" + date-fns "2.29.3" + json-stable-stringify "^1.0.1" + lodash "^4.17.13" + microee "^0.0.6" + node-fetch "^2.6.1" + node-polyglot "2.4.2" + open "7.4.2" + prop-types "^15.6.2" + react-redux "^7.2.0" + redux "3 || 4" + redux-thunk "^2.3.0" + server-destroy "^1.0.1" + sift "^6.0.0" + url-search-params-polyfill "^8.0.0" + cozy-device-helper@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/cozy-device-helper/-/cozy-device-helper-3.1.0.tgz#140079bf4a6844b21d6ac0a3a2d75741dba15faa" @@ -2217,10 +2252,19 @@ cozy-minilog@^3.3.1: dependencies: microee "0.0.6" -cozy-stack-client@^49.0.0: - version "49.0.0" - resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-49.0.0.tgz#1bda328d0d62b00bb8895be5b991d59ad6b41cfc" - integrity sha512-mlh/hR9KsIve+et17P6WXlO33FjXftzXK8ovWAKr8zk+5FcD/wy/yxV/9Mr3Q+SSabdUbFbBIqu8kZncafUzdg== +cozy-pouch-link@^49.5.0: + version "49.5.0" + resolved "https://registry.yarnpkg.com/cozy-pouch-link/-/cozy-pouch-link-49.5.0.tgz#ee2b43725ccd63f793800d62562706eaa2ff9888" + integrity sha512-T4kcKrwajE0N3URu2lA/YaoKiPRrSVRSFxh9QmAdOhfUHS5qZFrd0ox5gvyWTtNRceeJh1H9jC7LNDV7Qy0q9A== + dependencies: + cozy-client "^49.4.0" + pouchdb-browser "^7.2.2" + pouchdb-find "^7.2.2" + +cozy-stack-client@^49.0.0, cozy-stack-client@^49.4.0: + version "49.4.0" + resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-49.4.0.tgz#775d2d48e74182049977e2e423f7a42d39a4ac61" + integrity sha512-iy2vTpj25vHqErfPeclN3flI99il+WBm+Kt0FWI5YzM4H76+fE07/6Up4zOqtyOuaF7S22OHSim4Zl2hFB/SpA== dependencies: detect-node "^2.0.4" mime "^2.4.0" @@ -3094,6 +3138,13 @@ fb-watchman@^2.0.0: dependencies: bser "2.1.1" +fetch-cookie@0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-0.11.0.tgz#e046d2abadd0ded5804ce7e2cae06d4331c15407" + integrity sha512-BQm7iZLFhMWFy5CZ/162sAGjBfdNWb7a8LEqqnzsHFhxT/X/SVj/z2t2nu3aJvjlbQkrAlTUApplPRjWyH4mhA== + dependencies: + tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0" + file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -3531,6 +3582,11 @@ ignore@^5.2.0, ignore@^5.3.1: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.2.tgz#3cd40e729f3643fd87cb04e50bf0eb722bc596f5" integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== +immediate@3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.3.0.tgz#1aef225517836bcdf7f2a2de2600c79ff0269266" + integrity sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q== + import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" @@ -3565,7 +3621,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3, inherits@~2.0.4: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3, inherits@~2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -5006,6 +5062,127 @@ possible-typed-array-names@^1.0.0: resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f" integrity sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q== +pouchdb-abstract-mapreduce@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-abstract-mapreduce/-/pouchdb-abstract-mapreduce-7.3.1.tgz#96ff4a0f41cbe273f3f52fde003b719005a2093c" + integrity sha512-0zKXVFBvrfc1KnN0ggrB762JDmZnUpePHywo9Bq3Jy+L1FnoG7fXM5luFfvv5/T0gEw+ZTIwoocZECMnESBI9w== + dependencies: + pouchdb-binary-utils "7.3.1" + pouchdb-collate "7.3.1" + pouchdb-collections "7.3.1" + pouchdb-errors "7.3.1" + pouchdb-fetch "7.3.1" + pouchdb-mapreduce-utils "7.3.1" + pouchdb-md5 "7.3.1" + pouchdb-utils "7.3.1" + +pouchdb-binary-utils@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-binary-utils/-/pouchdb-binary-utils-7.3.1.tgz#eea22d9a5f880fcd95062476f4f5484cdf61496f" + integrity sha512-crZJNfAEOnUoRk977Qtmk4cxEv6sNKllQ6vDDKgQrQLFjMUXma35EHzNyIJr1s76J77Q4sqKQAmxz9Y40yHGtw== + dependencies: + buffer-from "1.1.2" + +pouchdb-browser@^7.2.2: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-browser/-/pouchdb-browser-7.3.1.tgz#6b2f9f35f42d2c83fc205de5e0403c0aae7046aa" + integrity sha512-qZ8awkXl/woBHvEVqNHjDtwPDA7A9v4ItHtX1y1eVpKel4mlYqnIJ8K6pRcFUZmVaHinJW8K3uS32eHC1q0yOA== + dependencies: + argsarray "0.0.1" + immediate "3.3.0" + inherits "2.0.4" + spark-md5 "3.0.2" + uuid "8.3.2" + vuvuzela "1.0.3" + +pouchdb-browser@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/pouchdb-browser/-/pouchdb-browser-9.0.0.tgz#296902da1587919f027987f0999cb70c2c9c05ac" + integrity sha512-0uKFWhsTtiVOF0+aGo7mvtCTP40f6dlsLNmJUvc/lwjsX1C3v+eBfVbvykyxpFl7UTAoJkXl+g/GOzNvyMtV1g== + dependencies: + spark-md5 "3.0.2" + uuid "8.3.2" + vuvuzela "1.0.3" + +pouchdb-collate@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-collate/-/pouchdb-collate-7.3.1.tgz#19d7b87dd173d1c765da8cc9987c5aa9eb24f11f" + integrity sha512-o4gyGqDMLMSNzf6EDTr3eHaH/JRMoqRhdc+eV+oA8u00nTBtr9wD+jypVe2LbgKLJ4NWqx2qVkXiTiQdUFtsLQ== + +pouchdb-collections@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-collections/-/pouchdb-collections-7.3.1.tgz#4f1819cf4dd6936a422c29f7fa26a9b5dca428f5" + integrity sha512-yUyDqR+OJmtwgExOSJegpBJXDLAEC84TWnbAYycyh+DZoA51Yw0+XVQF5Vh8Ii90/Ut2xo88fmrmp0t6kqom8w== + +pouchdb-errors@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-errors/-/pouchdb-errors-7.3.1.tgz#78be36721e2edc446fac158a236a9218c7bcdb14" + integrity sha512-Zktz4gnXEUcZcty8FmyvtYUYsHskoST05m6H5/E2gg/0mCfEXq/XeyyLkZHaZmqD0ZPS9yNmASB1VaFWEKEaDw== + dependencies: + inherits "2.0.4" + +pouchdb-fetch@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-fetch/-/pouchdb-fetch-7.3.1.tgz#d54b1807be0f0a5d4b6d06e416c7d54952bbc348" + integrity sha512-205xAtvdHRPQ4fp1h9+RmT9oQabo9gafuPmWsS9aEl3ER54WbY8Vaj1JHZGbU4KtMTYvW7H5088zLS7Nrusuag== + dependencies: + abort-controller "3.0.0" + fetch-cookie "0.11.0" + node-fetch "2.6.7" + +pouchdb-find@^7.2.2: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-find/-/pouchdb-find-7.3.1.tgz#07a633d5ee2bd731dae9f991281cd25212088d29" + integrity sha512-AeqUfAVY1c7IFaY36BRT0vIz9r4VTKq/YOWTmiqndOZUQ/pDGxyO2fNFal6NN3PyYww0JijlD377cPvhnrhJVA== + dependencies: + pouchdb-abstract-mapreduce "7.3.1" + pouchdb-collate "7.3.1" + pouchdb-errors "7.3.1" + pouchdb-fetch "7.3.1" + pouchdb-md5 "7.3.1" + pouchdb-selector-core "7.3.1" + pouchdb-utils "7.3.1" + +pouchdb-mapreduce-utils@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-mapreduce-utils/-/pouchdb-mapreduce-utils-7.3.1.tgz#f0ac2c8400fbedb705e9226082453ac7d3f2a066" + integrity sha512-oUMcq82+4pTGQ6dtrhgORHOVHZSr6w/5tFIUGlv7RABIDvJarL4snMawADjlpiEwPdiQ/ESG8Fqt8cxqvqsIgg== + dependencies: + argsarray "0.0.1" + inherits "2.0.4" + pouchdb-collections "7.3.1" + pouchdb-utils "7.3.1" + +pouchdb-md5@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-md5/-/pouchdb-md5-7.3.1.tgz#70fae44f9d27eb4c6a8e7106156b4593d31c1762" + integrity sha512-aDV8ui/mprnL3xmt0gT/81DFtTtJiKyn+OxIAbwKPMfz/rDFdPYvF0BmDC9QxMMzGfkV+JJUjU6at0PPs2mRLg== + dependencies: + pouchdb-binary-utils "7.3.1" + spark-md5 "3.0.2" + +pouchdb-selector-core@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-selector-core/-/pouchdb-selector-core-7.3.1.tgz#08245662de3d61f16ab8dae2b56ef622935b3fb3" + integrity sha512-HBX+nNGXcaL9z0uNpwSMRq2GNZd3EZXW+fe9rJHS0hvJohjZL7aRJLoaXfEdHPRTNW+CpjM3Rny60eGekQdI/w== + dependencies: + pouchdb-collate "7.3.1" + pouchdb-utils "7.3.1" + +pouchdb-utils@7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/pouchdb-utils/-/pouchdb-utils-7.3.1.tgz#d25f0a034427f388ba5ae37d9ae3fbed210e8720" + integrity sha512-R3hHBo1zTdTu/NFs3iqkcaQAPwhIH0gMIdfVKd5lbDYlmP26rCG5pdS+v7NuoSSFLJ4xxnaGV+Gjf4duYsJ8wQ== + dependencies: + argsarray "0.0.1" + clone-buffer "1.0.0" + immediate "3.3.0" + inherits "2.0.4" + pouchdb-collections "7.3.1" + pouchdb-errors "7.3.1" + pouchdb-md5 "7.3.1" + uuid "8.3.2" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -5584,6 +5761,11 @@ source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== +spark-md5@3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.2.tgz#7952c4a30784347abcee73268e473b9c0167e3fc" + integrity sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -5846,7 +6028,7 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -tough-cookie@^4.1.2: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.1.2: version "4.1.4" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.4.tgz#945f1461b45b5a8c76821c33ea49c3ac192c1b36" integrity sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag== @@ -6131,6 +6313,11 @@ vm-browserify@^1.1.2: resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0" integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ== +vuvuzela@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/vuvuzela/-/vuvuzela-1.0.3.tgz#3be145e58271c73ca55279dd851f12a682114b0b" + integrity sha512-Tm7jR1xTzBbPW+6y1tknKiEhz04Wf/1iZkcTJjSFcpNko43+dFW6+OOeQe9taJIug3NdfUAjFKgUSyQrIKaDvQ== + w3c-xmlserializer@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz#aebdc84920d806222936e3cdce408e32488a3073" From 3d3c3e177472c8c9dc5485d61da24ef1176a66e7 Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Tue, 15 Oct 2024 15:48:05 +0200 Subject: [PATCH 2/5] feat: Periodically update search indexes Based on the `pouchlink:doctypesync:end` emitted by cozy-pouch-link after each replication, we are able to update the search indexes through the changes API. --- src/dataproxy/worker/platformWorker.ts | 4 ---- src/dataproxy/worker/shared-worker.ts | 3 ++- src/search/SearchEngine.ts | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/dataproxy/worker/platformWorker.ts b/src/dataproxy/worker/platformWorker.ts index b39a83e..3982b38 100644 --- a/src/dataproxy/worker/platformWorker.ts +++ b/src/dataproxy/worker/platformWorker.ts @@ -30,7 +30,6 @@ const openDB = (): Promise => { }) } -// Define the storage object with TypeScript types const storage = { getItem: async (key: string): Promise => { const db = await openDB() @@ -84,7 +83,6 @@ const storage = { } } -// Define the event handling object with proper types const events = { addEventListener: ( eventName: string, @@ -100,12 +98,10 @@ const events = { } } -// Define the isOnline function with the proper type const isOnline = async (): Promise => { return self.navigator.onLine } -// Export the platformWorker object with TypeScript types export const platformWorker = { storage, events, diff --git a/src/dataproxy/worker/shared-worker.ts b/src/dataproxy/worker/shared-worker.ts index 4723506..6588431 100644 --- a/src/dataproxy/worker/shared-worker.ts +++ b/src/dataproxy/worker/shared-worker.ts @@ -23,6 +23,7 @@ let searchEngine: SearchEngine = null const broadcastChannel = new BroadcastChannel('DATA_PROXY_BROADCAST_CHANANEL') const dataProxy: DataProxyWorker = { + // FIXME: change setClient name setClient: async (clientData: ClientData) => { log.debug('Received data for setting client') if (client) return @@ -60,11 +61,11 @@ const dataProxy: DataProxyWorker = { client.capabilities = clientData.capabilities searchEngine = new SearchEngine(client) + updateState() }, search: async (query: string) => { - log.debug('Received data for search') if (!client) { throw new Error( 'Client is required to execute a search, please initialize CozyClient' diff --git a/src/search/SearchEngine.ts b/src/search/SearchEngine.ts index c7b5363..4b6ec9e 100644 --- a/src/search/SearchEngine.ts +++ b/src/search/SearchEngine.ts @@ -46,6 +46,22 @@ class SearchEngine { constructor(client: CozyClient) { this.client = client this.searchIndexes = {} + + this.indexOnReplicationChanges() + } + + indexOnReplicationChanges(): void { + if (!this.client) { + return + } + this.client.on('pouchlink:doctypesync:end', async (doctype: string) => { + // TODO: lock to avoid conflict with concurrent index events? + const newIndex = await this.indexDocsForSearch(doctype) + if (newIndex) { + log('debug', `Index updated for doctype ${doctype}`) + this.searchIndexes[doctype] = newIndex + } + }) } buildSearchIndex( From 4e975bed240d8da251e07cbcd44d0989c5dcc41e Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Tue, 15 Oct 2024 15:53:15 +0200 Subject: [PATCH 3/5] feat: Add realtime to update search index For any change on listened doctypes, the realtime allows us to dynamically update the related search index. --- package.json | 3 +- src/dataproxy/worker/shared-worker.ts | 9 ++- src/search/SearchEngine.ts | 89 ++++++++++++++++++++++----- src/search/consts.ts | 2 + src/search/helpers/replication.ts | 29 +++++++++ yarn.lock | 53 ++++++++++++---- 6 files changed, 156 insertions(+), 29 deletions(-) create mode 100644 src/search/helpers/replication.ts diff --git a/package.json b/package.json index b0a6aa4..302762f 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "cozy-flags": "^4.0.0", "cozy-logger": "^1.10.4", "cozy-minilog": "^3.3.1", - "cozy-pouch-link": "^49.5.0", + "cozy-pouch-link": "^49.8.0", + "cozy-realtime": "^5.0.2", "cozy-tsconfig": "^1.2.0", "flexsearch": "^0.7.43", "lodash": "^4.17.21", diff --git a/src/dataproxy/worker/shared-worker.ts b/src/dataproxy/worker/shared-worker.ts index 6588431..95f7f78 100644 --- a/src/dataproxy/worker/shared-worker.ts +++ b/src/dataproxy/worker/shared-worker.ts @@ -12,13 +12,17 @@ import { import { platformWorker } from '@/dataproxy/worker/platformWorker' import schema from '@/doctypes' import SearchEngine from '@/search/SearchEngine' -import { FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE } from '@/search/consts' +import { + FILES_DOCTYPE, + CONTACTS_DOCTYPE, + APPS_DOCTYPE +} from '@/search/consts' const log = Minilog('👷‍♂️ [shared-worker]') Minilog.enable() let client: CozyClient | undefined = undefined -let searchEngine: SearchEngine = null +let searchEngine: SearchEngine const broadcastChannel = new BroadcastChannel('DATA_PROXY_BROADCAST_CHANANEL') @@ -32,6 +36,7 @@ const dataProxy: DataProxyWorker = { const pouchLinkOptions = { doctypes: [FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE], initialSync: true, + periodicSync: false, platform: { ...platformWorker }, doctypesReplicationOptions: { [FILES_DOCTYPE]: { diff --git a/src/search/SearchEngine.ts b/src/search/SearchEngine.ts index 4b6ec9e..de5810f 100644 --- a/src/search/SearchEngine.ts +++ b/src/search/SearchEngine.ts @@ -4,6 +4,7 @@ import { encode as encode_balance } from 'flexsearch/dist/module/lang/latin/bala import CozyClient, { Q } from 'cozy-client' import Minilog from 'cozy-minilog' +import { RealtimePlugin } from 'cozy-realtime' import { SEARCH_SCHEMA, @@ -11,10 +12,12 @@ import { FILES_DOCTYPE, CONTACTS_DOCTYPE, DOCTYPE_ORDER, - LIMIT_DOCTYPE_SEARCH + LIMIT_DOCTYPE_SEARCH, + REPLICATION_DEBOUNCE } from '@/search/consts' import { getPouchLink } from '@/search/helpers/client' import { normalizeSearchResult } from '@/search/helpers/normalizeSearchResult' +import { startReplicationWithDebounce } from '@/search/helpers/replication' import { queryFilesForSearch, queryAllContacts, @@ -42,28 +45,77 @@ interface FlexSearchResultWithDoctype class SearchEngine { client: CozyClient searchIndexes: SearchIndexes + debouncedReplication: () => void constructor(client: CozyClient) { this.client = client this.searchIndexes = {} - this.indexOnReplicationChanges() + this.indexOnChanges() + this.debouncedReplication = startReplicationWithDebounce( + client, + REPLICATION_DEBOUNCE + ) } - indexOnReplicationChanges(): void { + indexOnChanges(): void { if (!this.client) { return } this.client.on('pouchlink:doctypesync:end', async (doctype: string) => { - // TODO: lock to avoid conflict with concurrent index events? - const newIndex = await this.indexDocsForSearch(doctype) - if (newIndex) { - log('debug', `Index updated for doctype ${doctype}`) - this.searchIndexes[doctype] = newIndex - } + await this.indexDocsForSearch(doctype) + }) + this.client.on('login', () => { + // Ensure login is done before plugin register + this.client.registerPlugin(RealtimePlugin, {}) + this.handleUpdatedOrCreatedDoc = this.handleUpdatedOrCreatedDoc.bind(this) + this.handleDeletedDoc = this.handleDeletedDoc.bind(this) + + this.subscribeDoctype(this.client, FILES_DOCTYPE) + this.subscribeDoctype(this.client, CONTACTS_DOCTYPE) + this.subscribeDoctype(this.client, APPS_DOCTYPE) }) } + subscribeDoctype(client: CozyClient, doctype: string): void { + const realtime = this.client.plugins.realtime + realtime.subscribe('created', doctype, this.handleUpdatedOrCreatedDoc) + realtime.subscribe('updated', doctype, this.handleUpdatedOrCreatedDoc) + realtime.subscribe('deleted', doctype, this.handleDeletedDoc) + } + + handleUpdatedOrCreatedDoc(doc: CozyDoc): void { + const doctype: string | undefined = doc._type + if (!doctype) { + return + } + const searchIndex = this.searchIndexes?.[doctype] + if (!searchIndex) { + // No index yet: it will be done by querying the local db after first replication + return + } + log.debug('[REALTIME] index doc after update : ', doc) + searchIndex.index.add(doc) + + this.debouncedReplication() + } + + handleDeletedDoc(doc: CozyDoc): void { + const doctype: string | undefined = doc._type + if (!doctype) { + return + } + const searchIndex = this.searchIndexes?.[doctype] + if (!searchIndex) { + // No index yet: it will be done by querying the local db after first replication + return + } + log.debug('[REALTIME] remove doc from index after update : ', doc) + this.searchIndexes[doctype].index.remove(doc._id) + + this.debouncedReplication() + } + buildSearchIndex( doctype: keyof typeof SEARCH_SCHEMA, docs: CozyDoc[] @@ -88,13 +140,16 @@ class SearchEngine { return flexsearchIndex } - async indexDocsForSearch(doctype: string): Promise { + async indexDocsForSearch(doctype: string): Promise { const searchIndex = this.searchIndexes[doctype] const pouchLink = getPouchLink(this.client) - if (!pouchLink) return null + if (!pouchLink) { + return null + } if (!searchIndex) { + // First creation of search index const docs = await this.client.queryAll(Q(doctype).limitBy(null)) const index = this.buildSearchIndex(doctype, docs) const info = await pouchLink.getDbInfo(doctype) @@ -106,6 +161,10 @@ class SearchEngine { return this.searchIndexes[doctype] } + // Incremental index update + // At this point, the search index are supposed to be already up-to-date, + // thanks to the realtime. + // However, we check it is actually the case for safety, and update the lastSeq const lastSeq = searchIndex.lastSeq || 0 const changes = await pouchLink.getChanges(doctype, { include_docs: true, @@ -139,9 +198,9 @@ class SearchEngine { log.debug('Finished initializing indexes') this.searchIndexes = { - [FILES_DOCTYPE]: { index: filesIndex, lastSeq: null }, - [CONTACTS_DOCTYPE]: { index: contactsIndex, lastSeq: null }, - [APPS_DOCTYPE]: { index: appsIndex, lastSeq: null } + [FILES_DOCTYPE]: { index: filesIndex, lastSeq: 0 }, + [CONTACTS_DOCTYPE]: { index: contactsIndex, lastSeq: 0 }, + [APPS_DOCTYPE]: { index: appsIndex, lastSeq: 0 } } return this.searchIndexes } @@ -172,6 +231,8 @@ class SearchEngine { log.warn('[SEARCH] No search index available for ', doctype) continue } + // TODO: do not use flexsearch store and rely on pouch storage? + // It's better for memory, but might slow down search queries const indexResults = index.index.search(query, LIMIT_DOCTYPE_SEARCH, { enrich: true }) diff --git a/src/search/consts.ts b/src/search/consts.ts index d527ae8..ae2e463 100644 --- a/src/search/consts.ts +++ b/src/search/consts.ts @@ -15,6 +15,8 @@ export const SEARCH_SCHEMA = { 'io.cozy.apps': ['slug', 'name'] } +export const REPLICATION_DEBOUNCE = 30 * 1000 // 30s + export const FILES_DOCTYPE = 'io.cozy.files' export const CONTACTS_DOCTYPE = 'io.cozy.contacts' export const APPS_DOCTYPE = 'io.cozy.apps' diff --git a/src/search/helpers/replication.ts b/src/search/helpers/replication.ts new file mode 100644 index 0000000..8ca86c8 --- /dev/null +++ b/src/search/helpers/replication.ts @@ -0,0 +1,29 @@ +import CozyClient from 'cozy-client' +import Minilog from 'cozy-minilog' + +import { getPouchLink } from '@/search/helpers/client' + +const log = Minilog('🗂️ [Replication]') + +export const startReplicationWithDebounce = ( + client: CozyClient, + interval: number +) => { + let timeoutId: NodeJS.Timeout | null = null + + return (): void => { + if (timeoutId) { + log.debug('Reset replication debounce') + clearTimeout(timeoutId) + } + + timeoutId = setTimeout(() => { + const pouchLink = getPouchLink(client) + if (!pouchLink) { + return + } + log.debug('Start replication after debounce of ', interval) + pouchLink.startReplication() + }, interval) + } +} diff --git a/yarn.lock b/yarn.lock index b0c4e53..0d35211 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,7 +308,7 @@ resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== -"@cozy/minilog@1.0.0": +"@cozy/minilog@1.0.0", "@cozy/minilog@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@cozy/minilog/-/minilog-1.0.0.tgz#1acc1aad849261e931e255a5f181b638315f7b84" integrity sha512-IkDHF9CLh0kQeSEVsou59ar/VehvenpbEUjLfwhckJyOUqZnKAWmXy8qrBgMT5Loxr8Xjs2wmMnj0D67wP00eQ== @@ -2198,16 +2198,16 @@ cozy-client@^49.0.0: sift "^6.0.0" url-search-params-polyfill "^8.0.0" -cozy-client@^49.4.0: - version "49.4.0" - resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-49.4.0.tgz#248788e5e7a3595dd9137f5d0563ac6a5f5c0231" - integrity sha512-jIapZfMzJzCblI4pKnir3nXgOvrXavsRXkm7QB2qbDl9WSCJcpKvMw8Z1yDsma7nB/16E0rKzeEBh+BxXbStTw== +cozy-client@^49.8.0: + version "49.8.0" + resolved "https://registry.yarnpkg.com/cozy-client/-/cozy-client-49.8.0.tgz#d4f6b3a2fb26dc6bfa561046b075eb6e221af73e" + integrity sha512-peb2n+buOihzqqNN3/zULmytQI8wOtQZsIqLWEaw2WfNXha1T23wNBrMuE7SaMOof361f011MrM5+WVYhKj+VA== dependencies: "@cozy/minilog" "1.0.0" "@types/jest" "^26.0.20" "@types/lodash" "^4.14.170" btoa "^1.2.1" - cozy-stack-client "^49.4.0" + cozy-stack-client "^49.8.0" date-fns "2.29.3" json-stable-stringify "^1.0.1" lodash "^4.17.13" @@ -2252,16 +2252,24 @@ cozy-minilog@^3.3.1: dependencies: microee "0.0.6" -cozy-pouch-link@^49.5.0: - version "49.5.0" - resolved "https://registry.yarnpkg.com/cozy-pouch-link/-/cozy-pouch-link-49.5.0.tgz#ee2b43725ccd63f793800d62562706eaa2ff9888" - integrity sha512-T4kcKrwajE0N3URu2lA/YaoKiPRrSVRSFxh9QmAdOhfUHS5qZFrd0ox5gvyWTtNRceeJh1H9jC7LNDV7Qy0q9A== +cozy-pouch-link@^49.8.0: + version "49.8.0" + resolved "https://registry.yarnpkg.com/cozy-pouch-link/-/cozy-pouch-link-49.8.0.tgz#a5361bbc0ed23b9bc4d804d31869c615fdf1d13e" + integrity sha512-c5+dWx5BNi37JuLVpil9SB91vy3EtB7B8Qa6Krx+znzPYcRoK3Bqjuns5c7hXG5rD8MV2qls1yvxQWUrfy82kA== dependencies: - cozy-client "^49.4.0" + cozy-client "^49.8.0" pouchdb-browser "^7.2.2" pouchdb-find "^7.2.2" -cozy-stack-client@^49.0.0, cozy-stack-client@^49.4.0: +cozy-realtime@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/cozy-realtime/-/cozy-realtime-5.0.2.tgz#d515e6625e4386c812e8a0ce505c2e3abfea2245" + integrity sha512-ncFgsb2BeaYyM+Uax/qbcnDC1n7hpLNAiaqnVD02ApDpN1Uy/2GUzbcAr5xpc1gK4jkp6+fb8FrqyL+vcxXRsg== + dependencies: + "@cozy/minilog" "^1.0.0" + cozy-device-helper "^3.1.0" + +cozy-stack-client@^49.0.0: version "49.4.0" resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-49.4.0.tgz#775d2d48e74182049977e2e423f7a42d39a4ac61" integrity sha512-iy2vTpj25vHqErfPeclN3flI99il+WBm+Kt0FWI5YzM4H76+fE07/6Up4zOqtyOuaF7S22OHSim4Zl2hFB/SpA== @@ -2270,6 +2278,15 @@ cozy-stack-client@^49.0.0, cozy-stack-client@^49.4.0: mime "^2.4.0" qs "^6.7.0" +cozy-stack-client@^49.8.0: + version "49.8.0" + resolved "https://registry.yarnpkg.com/cozy-stack-client/-/cozy-stack-client-49.8.0.tgz#c57dfefe50e47f228fee7e1921c438d35f4e0877" + integrity sha512-sYJL2o+DsNs7V5eQghXpWKcMzxc39QAKtM8zAdmWl2MMCyiqO3lBehRomhstcJHtuZrMLXXPQPr1A0ONBlMmZg== + dependencies: + detect-node "^2.0.4" + mime "^2.4.0" + qs "^6.7.0" + cozy-tsconfig@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/cozy-tsconfig/-/cozy-tsconfig-1.2.0.tgz#17e61f960f139fae4d26cbac2254b9ab632b269e" @@ -4742,6 +4759,13 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +node-fetch@2.6.7: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" + node-fetch@^2.6.1: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" @@ -6271,6 +6295,11 @@ util@^0.12.5: is-typed-array "^1.1.3" which-typed-array "^1.1.2" +uuid@8.3.2: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" From acb2a61bd86e89c358d875a0ca7ff61c270021af Mon Sep 17 00:00:00 2001 From: Ldoppea Date: Tue, 15 Oct 2024 18:54:46 +0200 Subject: [PATCH 4/5] feat: Add test with mocks --- src/@types/cozy-client.d.ts | 1 + src/dataproxy/worker/shared-worker.ts | 6 +-- src/search/SearchEngine.ts | 7 ++-- src/search/helpers/getSearchEncoder.ts | 7 ++++ src/search/helpers/replication.spec.ts | 53 ++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 src/search/helpers/getSearchEncoder.ts create mode 100644 src/search/helpers/replication.spec.ts diff --git a/src/@types/cozy-client.d.ts b/src/@types/cozy-client.d.ts index 8c1e7d1..15f6036 100644 --- a/src/@types/cozy-client.d.ts +++ b/src/@types/cozy-client.d.ts @@ -170,6 +170,7 @@ declare module 'cozy-client' { } export default class CozyClient { + plugins: any constructor(rawOptions?: ClientOptions) getStackClient(): StackClient getInstanceOptions(): InstanceOptions diff --git a/src/dataproxy/worker/shared-worker.ts b/src/dataproxy/worker/shared-worker.ts index 95f7f78..961de13 100644 --- a/src/dataproxy/worker/shared-worker.ts +++ b/src/dataproxy/worker/shared-worker.ts @@ -12,11 +12,7 @@ import { import { platformWorker } from '@/dataproxy/worker/platformWorker' import schema from '@/doctypes' import SearchEngine from '@/search/SearchEngine' -import { - FILES_DOCTYPE, - CONTACTS_DOCTYPE, - APPS_DOCTYPE -} from '@/search/consts' +import { FILES_DOCTYPE, CONTACTS_DOCTYPE, APPS_DOCTYPE } from '@/search/consts' const log = Minilog('👷‍♂️ [shared-worker]') Minilog.enable() diff --git a/src/search/SearchEngine.ts b/src/search/SearchEngine.ts index de5810f..7ac3184 100644 --- a/src/search/SearchEngine.ts +++ b/src/search/SearchEngine.ts @@ -1,6 +1,4 @@ import FlexSearch from 'flexsearch' -// @ts-ignore -import { encode as encode_balance } from 'flexsearch/dist/module/lang/latin/balance' import CozyClient, { Q } from 'cozy-client' import Minilog from 'cozy-minilog' @@ -16,6 +14,7 @@ import { REPLICATION_DEBOUNCE } from '@/search/consts' import { getPouchLink } from '@/search/helpers/client' +import { getSearchEncoder } from '@/search/helpers/getSearchEncoder' import { normalizeSearchResult } from '@/search/helpers/normalizeSearchResult' import { startReplicationWithDebounce } from '@/search/helpers/replication' import { @@ -124,7 +123,7 @@ class SearchEngine { const flexsearchIndex = new FlexSearch.Document({ tokenize: 'forward', - encode: encode_balance as FlexSearch.Encoders, + encode: getSearchEncoder(), minlength: 2, document: { id: '_id', @@ -162,7 +161,7 @@ class SearchEngine { } // Incremental index update - // At this point, the search index are supposed to be already up-to-date, + // At this point, the search index are supposed to be already up-to-date, // thanks to the realtime. // However, we check it is actually the case for safety, and update the lastSeq const lastSeq = searchIndex.lastSeq || 0 diff --git a/src/search/helpers/getSearchEncoder.ts b/src/search/helpers/getSearchEncoder.ts new file mode 100644 index 0000000..cfdc22a --- /dev/null +++ b/src/search/helpers/getSearchEncoder.ts @@ -0,0 +1,7 @@ +import FlexSearch from 'flexsearch' +// @ts-ignore +import { encode as encode_balance } from 'flexsearch/dist/module/lang/latin/balance' + +export const getSearchEncoder = (): FlexSearch.Encoders => { + return encode_balance as FlexSearch.Encoders +} diff --git a/src/search/helpers/replication.spec.ts b/src/search/helpers/replication.spec.ts new file mode 100644 index 0000000..570f783 --- /dev/null +++ b/src/search/helpers/replication.spec.ts @@ -0,0 +1,53 @@ +import CozyClient from 'cozy-client' + +import { getPouchLink } from '@/search/helpers/client' + +import { startReplicationWithDebounce } from './replication' + +jest.mock('cozy-client') +jest.mock('@/search/helpers/client', () => ({ + getPouchLink: jest.fn() +})) + +describe('startReplicationWithDebounce', () => { + let client: CozyClient + let pouchLink: any + + beforeEach(() => { + client = new CozyClient() + pouchLink = { + startReplication: jest.fn() + } + ;(getPouchLink as jest.Mock).mockReturnValue(pouchLink) + jest.useFakeTimers() + }) + + afterEach(() => { + jest.clearAllMocks() + jest.clearAllTimers() + }) + + it('should start replication after the specified interval', () => { + const interval = 1000 + const replicate = startReplicationWithDebounce(client, interval) + + replicate() + expect(pouchLink.startReplication).not.toHaveBeenCalled() + jest.advanceTimersByTime(interval) + expect(pouchLink.startReplication).toHaveBeenCalledTimes(1) + }) + + it('should debounce replication calls within the interval', () => { + const interval = 1000 + const replicate = startReplicationWithDebounce(client, interval) + + replicate() + jest.advanceTimersByTime(interval / 2) + expect(pouchLink.startReplication).not.toHaveBeenCalled() + replicate() + replicate() + + jest.advanceTimersByTime(interval) + expect(pouchLink.startReplication).toHaveBeenCalledTimes(1) + }) +}) From aef7bc0e77cb7b12c3f4345e683179bd37ea5b7a Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Fri, 18 Oct 2024 13:54:52 +0200 Subject: [PATCH 5/5] chore: Make eslint less unpleasant --- eslint.config.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eslint.config.js b/eslint.config.js index ccf6236..e785e03 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -101,13 +101,13 @@ const config = [ '@typescript-eslint': tseslint.plugin }, rules: { - '@typescript-eslint/no-unsafe-argument': 'error', - '@typescript-eslint/no-unsafe-assignment': 'error', - '@typescript-eslint/no-unsafe-call': 'error', - '@typescript-eslint/no-unsafe-member-access': 'error', - '@typescript-eslint/no-unsafe-return': 'error', + '@typescript-eslint/no-unsafe-argument': 'warn', + '@typescript-eslint/no-unsafe-assignment': 'warn', + '@typescript-eslint/no-unsafe-call': 'warn', + '@typescript-eslint/no-unsafe-member-access': 'warn', + '@typescript-eslint/no-unsafe-return': 'warn', '@typescript-eslint/explicit-function-return-type': 'error', - '@typescript-eslint/no-explicit-any': 'error', + '@typescript-eslint/no-explicit-any': 'warn', '@typescript-eslint/no-unused-vars': [ 'error', { ignoreRestSiblings: true }