SQLite store implementation effort #4570
Labels
T-Other
Questions, user support, anything else
Comments
|
@theobouwman it's worth knowing that there are two stores in matrix-js-sdk: the |
|
@richvdh I know. I am first planning to implement the store without the end-to-end encryption. But, by implementing the IStore (https://github.com/matrix-org/matrix-js-sdk/blob/develop/src/store/index.ts) should be a good start right? And will the TimelineWindow class use the configured store as well? |
|
Fair enough. I don't think |
|
Okay. I made a start. import { MatrixEvent, Room, User, Filter, IStateEventWithRoomId, IStartClientOpts, IEvent, ISyncResponse, MemoryStore, IStoredClientOpts, KnownMembership, RoomState, SyncAccumulator } from 'matrix-js-sdk';
import { RoomSummary } from 'matrix-js-sdk/lib/models/room-summary';
import { ToDeviceBatchWithTxnId, IndexedToDeviceBatch } from 'matrix-js-sdk/lib/models/ToDeviceMessage';
import { ISavedSync, IStore, UserCreator } from 'matrix-js-sdk/lib/store';
import { deepCopy, MapWithDefault } from 'matrix-js-sdk/lib/utils';
import { MMKVInstance, MMKVLoader } from 'react-native-mmkv-storage';
const WRITE_DELAY_MS = 1000 * 60 * 1; // once every 1 minutes
const isValidFilterId = (filterId?: string | number | null): boolean => {
const isValidStr =
typeof filterId === "string" &&
!!filterId &&
filterId !== "undefined" && // exclude these as we've serialized undefined in localStorage before
filterId !== "null";
return isValidStr || typeof filterId === "number";
}
class MatrixMMKVStore implements IStore {
private storage: MMKVInstance;
private userCreator: UserCreator | null = null;
private readonly syncAccumulator: SyncAccumulator;
accountData = new Map<string, MatrixEvent>();
private syncTs = 0;
constructor() {
this.storage = new MMKVLoader().initialize();
this.syncAccumulator = new SyncAccumulator();
}
private prefixKey(key: string): string {
return `matrix:${key}`;
}
async isNewlyCreated(): Promise<boolean> {
const flag = await this.storage.getBoolAsync(this.prefixKey('isNewlyCreated'));
if (flag === null) {
await this.storage.setBoolAsync(this.prefixKey('isNewlyCreated'), true);
return Promise.resolve(true);
}
return Promise.resolve(false);
}
getSyncToken(): string | null {
return this.storage.getString(this.prefixKey('syncToken')) as string | null;
}
setSyncToken(token: string = ''): void {
this.storage.setString(this.prefixKey('syncToken'), token);
}
storeRoom(room: Room): void {
this.storage.setMap(this.prefixKey(`room:${room.roomId}`), room);
console.log('store room', room.roomId, room)
const storedRoomIds = this.storage.getArray<string>(this.prefixKey('roomIds')) ?? []
let newRoomIds = [...storedRoomIds, room.roomId]
this.storage.setArray(this.prefixKey('roomIds'), newRoomIds)
}
setUserCreator(creator: UserCreator): void {
this.userCreator = creator;
}
getRoom(roomId: string): Room | null {
const room = this.storage.getMap<Room>(this.prefixKey(`room:${roomId}`));
console.log('get room', roomId, room)
return room
}
getRooms(): Room[] {
const roomIds = this.storage.getArray<string>(this.prefixKey('roomIds')) ?? []
console.log('mmkv roomids', roomIds)
if (roomIds.length > 0) {
const roomKeys = roomIds.map(r => `room:${r}`)
const roomData = this.storage.getMultipleItems<Room>(roomKeys, 'map');
const rooms = roomData
.map((item) => item[1]!)
.filter((room) => room);
console.log('get rooms', rooms)
return rooms
}
return []
}
removeRoom(roomId: string): void {
console.log('remove room', roomId)
this.storage.removeItem(this.prefixKey(`room:${roomId}`));
const storedRoomIds = this.storage.getArray<string>(this.prefixKey('roomIds')) ?? []
this.storage.setArray(this.prefixKey('roomIds'), storedRoomIds.filter(r => r !== roomId))
}
getRoomSummaries(): RoomSummary[] {
console.log('get room summaries')
const rooms = this.getRooms();
return rooms.filter(room => room.summary !== null).map((room) => room.summary!);
}
storeUser(user: User): void {
console.log('store user', user)
this.storage.setMap(this.prefixKey(`user:${user.userId}`), user);
}
getUser(userId: string): User | null {
const user = this.storage.getMap<User>(this.prefixKey(`user:${userId}`))
console.log('get user', userId, user)
return user
}
getUsers(): User[] {
console.log('get users')
const keys = this.storage.getAllMMKVInstanceIDs();
const userKeys = keys.filter((key) => key.startsWith('matrix:user:'));
const userData = this.storage.getMultipleItems<User>(userKeys, 'map');
console.log('users:', userData)
return userData
.map((item) => item[1]!)
.filter((user) => user);
}
scrollback(room: Room, limit: number): MatrixEvent[] {
// Placeholder: implement logic for retrieving room scrollback
console.log('scrollback', room, limit)
return [];
}
storeEvents(room: Room, events: MatrixEvent[], token: string | null, toStart: boolean): void {
// Placeholder: implement logic to store room events
console.log('store events', room)
}
storeFilter(filter: Filter): void {
if (!filter?.userId || !filter?.filterId) return;
console.log('store filter', filter)
const filterIdUserIdKey = `filter:${filter.userId}:${filter.filterId}`
// let storedFilter = this.storage.getMap<Filter>(this.prefixKey(filterIdUserIdKey))
// console.log('storedFilter', storedFilter)
this.storage.setMap(this.prefixKey(filterIdUserIdKey), filter)
// if (!storedFilter) {
// console.log('do store filter', filter)
// this.storage.setMap(this.prefixKey(filterIdUserIdKey), filter)
// }
let storedFilters = this.storage.getArray<Filter>(this.prefixKey('filters'))
console.log('storedFilters array:', storedFilters)
if (!storedFilters || storedFilters.length === 0) {
storedFilters = []
console.log('new stored filter')
}
storedFilters = storedFilters.filter(f => `filter:${f.userId}:${f.filterId}` !== filterIdUserIdKey)
storedFilters.push(filter)
console.log('storedFilters set', filterIdUserIdKey ,storedFilters)
this.storage.setArray(this.prefixKey('filters'), storedFilters)
}
getFilter(userId: string, filterId: string): Filter | null {
const filter = this.storage.getMap<object>(this.prefixKey(`filter:${userId}:${filterId}`));
if (!filter) {
console.log('get filter not found', userId, filterId)
return null
}
const f = Filter.fromJson(userId, filterId, filter)
console.log('get filter', userId, filterId, 'filterOBJ', f)
return f
}
getFilterIdByName(filterName: string): string | null {
try {
const filterId = this.storage.getString(this.prefixKey(filterName.replace('FILTER_SYNC_', '')))
if (isValidFilterId(filterId)) {
console.log('get filter id by name', filterName, filterId)
return filterId as string | null;
}
} catch {}
console.log('get filter id by name', filterName, 'null-empty')
return null;
}
setFilterIdByName(filterName: string, filterId?: string): void {
console.log('set filter id by name with filterId', filterName, filterId)
if (isValidFilterId(filterId)) {
this.storage.setString(this.prefixKey(filterName.replace('FILTER_SYNC_', '')), filterId!);
} else {
this.storage.removeItem(this.prefixKey(filterName.replace('FILTER_SYNC_', '')));
}
}
storeAccountDataEvents(events: MatrixEvent[]): void {
console.log('store account data events', events.length)
events.forEach((event) => {
this.accountData.set(event.getType(), event);
});
}
getAccountData(eventType: string): MatrixEvent | undefined {
console.log('get account', eventType)
return this.accountData.get(eventType);
}
async setSyncData(syncData: ISyncResponse): Promise<void> {
this.storage.setMap(this.prefixKey('syncData'), syncData);
console.log('set sync data', syncData)
return Promise.resolve().then(() => {
this.syncAccumulator.accumulate(syncData)
})
}
wantsSave(): boolean {
const now = Date.now();
const want = now - this.syncTs > WRITE_DELAY_MS;
console.log('wants save', want)
return want
}
save(force?: boolean): Promise<void> {
if (force || this.wantsSave()) {
return this.reallySave();
}
return Promise.resolve();
}
reallySave(): Promise<void> {
this.syncTs = Date.now(); // set now to guard against multi-writes
console.info('store:reallySave')
// work out changed users (this doesn't handle deletions but you
// can't 'delete' users as they are just presence events).
// const userTuples: [userId: string, presenceEvent: Partial<IEvent>][] = [];
// for (const u of this.getUsers()) {
// if (this.userModifiedMap[u.userId] === u.getLastModifiedTime()) continue;
// if (!u.events.presence) continue;
// userTuples.push([u.userId, u.events.presence.event]);
// // note that we've saved this version of the user
// this.userModifiedMap[u.userId] = u.getLastModifiedTime();
// }
return Promise.resolve()
// return this.backend.syncToDatabase(userTuples);
}
async startup(): Promise<void> {
// No-op for MMKV initialization
console.log('-------- store startup')
const savedSync = await this.getSavedSync()
console.log('savwd sunc startup', savedSync)
if (savedSync) {
console.log('-------- store startup sync accumulator')
return this.syncAccumulator.accumulate({
next_batch: savedSync.nextBatch,
rooms: savedSync.roomsData,
account_data: {
events: savedSync.accountData
}
}, true)
}
return Promise.resolve()
}
async getSavedSync(copy = true): Promise<ISavedSync | null> {
const res = this.storage.getMap<ISavedSync|null>(this.prefixKey('syncData'))
console.log('get saved sync', res)
return Promise.resolve(res)
// const data = this.syncAccumulator.getJSON();
// if (!data.nextBatch) return Promise.resolve(null);
// if (copy) {
// // We must deep copy the stored data so that the /sync processing code doesn't
// // corrupt the internal state of the sync accumulator (it adds non-clonable keys)
// return Promise.resolve(deepCopy(data));
// } else {
// return Promise.resolve(data);
// }
}
async getSavedSyncToken(): Promise<string | null> {
const syncData = await this.getSavedSync();
return syncData?.nextBatch || null;
}
async deleteAllData(): Promise<void> {
this.storage.clearStore();
}
async getOutOfBandMembers(roomId: string): Promise<IStateEventWithRoomId[] | null> {
console.log('get out of band members', roomId)
const res = this.storage.getArray<IStateEventWithRoomId>(this.prefixKey(`oobMembers:${roomId}`));
return Promise.resolve(res)
}
async setOutOfBandMembers(roomId: string, membershipEvents: IStateEventWithRoomId[]): Promise<void> {
console.log('set out of band members', roomId)
this.storage.setArray(`oobMembers:${roomId}`, membershipEvents);
return Promise.resolve()
}
async clearOutOfBandMembers(roomId: string): Promise<void> {
console.log('clear out of band members', roomId)
this.storage.removeItem(this.prefixKey(`oobMembers:${roomId}`));
}
async getClientOptions(): Promise<IStartClientOpts | undefined> {
return await this.storage.getMapAsync<IStartClientOpts>(this.prefixKey('clientOptions')) as IStartClientOpts | undefined;
}
async storeClientOptions(options: IStartClientOpts): Promise<void> {
await this.storage.setMap(this.prefixKey('clientOptions'), options);
}
async getPendingEvents(roomId: string): Promise<Partial<IEvent>[]> {
return await this.storage.getMapAsync<Partial<IEvent>[]>(this.prefixKey(`pendingEvents:${roomId}`)) || [];
}
async setPendingEvents(roomId: string, events: Partial<IEvent>[]): Promise<void> {
await this.storage.setArray(this.prefixKey(`pendingEvents:${roomId}`), events);
}
async saveToDeviceBatches(batch: ToDeviceBatchWithTxnId[]): Promise<void> {
await this.storage.setArrayAsync(this.prefixKey('toDeviceBatches'), batch);
}
async getOldestToDeviceBatch(): Promise<IndexedToDeviceBatch | null> {
const batches = await this.storage.getArrayAsync<IndexedToDeviceBatch>(this.prefixKey('toDeviceBatches'));
return batches?.[0] || null;
}
async removeToDeviceBatch(id: number): Promise<void> {
const batches = await this.storage.getArray<IndexedToDeviceBatch>(this.prefixKey('toDeviceBatches'));
if (batches) {
const updatedBatches = batches.filter((batch) => batch.id !== id);
await this.storage.setArrayAsync(this.prefixKey('toDeviceBatches'), updatedBatches);
}
}
async destroy(): Promise<void> {
await this.deleteAllData();
}
}
export default MatrixMMKVStore; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hi,
We are currently using matrix js sdk in our react native app. As indexeddb is not supported in RN we currently use the memory store.
The problem is that when you open a notification from a room there is a loading state because nothin is stored on device. We are using TimelineWindow for our rooms with pagination etc https://matrix-org.github.io/matrix-js-sdk/classes/matrix.TimelineWindow.html.
So I have 2 questions:
Thanks
The text was updated successfully, but these errors were encountered: