From 0191282dbf01acf51d0e61378ad07b3a4e0ec181 Mon Sep 17 00:00:00 2001 From: ereio Date: Fri, 13 Nov 2020 23:59:29 -0500 Subject: [PATCH 01/25] removed hive serialization and encryption cruft --- assets/cheatsheet.md | 36 +- lib/global/cache/serializer.dart | 13 - lib/global/libs/hive/encoder.dart | 148 -------- lib/global/libs/hive/index.dart | 336 ------------------ lib/global/libs/hive/type-ids.dart | 27 -- lib/global/themes.dart | 12 - lib/main.dart | 5 - lib/store/auth/state.dart | 5 +- lib/store/crypto/keys/model.dart | 40 +-- lib/store/crypto/model.dart | 17 - lib/store/crypto/state.dart | 29 +- lib/store/media/state.dart | 24 +- .../rooms/events/ephemeral/m.read/model.dart | 11 +- lib/store/rooms/events/model.dart | 37 +- lib/store/rooms/room/model.dart | 59 +-- lib/store/rooms/state.dart | 21 +- lib/store/settings/chat-settings/model.dart | 8 - .../settings/devices-settings/model.dart | 10 - .../settings/notification-settings/model.dart | 8 +- .../notification-settings/pushers/model.dart | 9 - .../notification-settings/rules/model.dart | 10 - lib/store/settings/state.dart | 28 +- lib/store/sync/background/service.dart | 4 +- lib/store/sync/state.dart | 11 - lib/store/user/model.dart | 13 - lib/store/user/state.dart | 9 +- 26 files changed, 73 insertions(+), 857 deletions(-) delete mode 100644 lib/global/libs/hive/encoder.dart delete mode 100644 lib/global/libs/hive/index.dart delete mode 100644 lib/global/libs/hive/type-ids.dart diff --git a/assets/cheatsheet.md b/assets/cheatsheet.md index d7dc0544b..9f299ad15 100644 --- a/assets/cheatsheet.md +++ b/assets/cheatsheet.md @@ -62,4 +62,38 @@ if (true) { }, ); -``` \ No newline at end of file +``` + +```dart + // invite and membership events are different + + // {membership: join, displayname: usbfingers, avatar_url: mxc://matrix.org/RrRcMHnqXaJshyXZpGrZloyh } + // {is_direct: true, membership: invite, displayname: ereio, avatar_url: mxc://matrix.org/JllILpqzdFAUOvrTPSkDryzW} + +``` + +```dart + +/** + * OneTimeKey Data Model + * + * https://matrix.org/docs/spec/client_server/latest#id468 + * { + "failures": {}, + "one_time_keys": { + "@alice:example.com": { + "JLAFKJWSCS": { + "signed_curve25519:AAAAHg": { + "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" + } + } + } + } + } + } + } + */ + ``` \ No newline at end of file diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index 2aa019fe8..6bc4ee6ad 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -11,8 +11,6 @@ import 'package:redux_persist/redux_persist.dart'; import 'package:steel_crypt/steel_crypt.dart'; import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/cache/threadables.dart'; -import 'package:syphon/global/libs/hive/encoder.dart'; -import 'package:syphon/global/libs/hive/index.dart'; // Project imports: import 'package:syphon/store/crypto/state.dart'; @@ -131,17 +129,6 @@ class CacheSerializer implements StateSerializer { userStore, ]; - // TODO: remove after most have upgraded to 0.1.4/0.1.5 - if ((Cache.state != null || Cache.stateRooms != null) && - Cache.migration == null) { - debugPrint( - '[Legacy Cache Found] ***** FOUND ****** loading and removing cache', - ); - final legacyAppState = LegacyEncoder.decodeHive(); - deleteLegacyStorage(); - return legacyAppState; - } - // decode each store cache synchronously stores.forEach((store) { try { diff --git a/lib/global/libs/hive/encoder.dart b/lib/global/libs/hive/encoder.dart deleted file mode 100644 index e87f6aaf4..000000000 --- a/lib/global/libs/hive/encoder.dart +++ /dev/null @@ -1,148 +0,0 @@ -import 'dart:convert'; - -import 'package:flutter/material.dart'; -import 'package:syphon/global/libs/hive/index.dart'; -import 'package:syphon/store/auth/state.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/index.dart'; -import 'package:syphon/store/crypto/state.dart'; -import 'package:syphon/store/index.dart'; -import 'package:syphon/store/media/state.dart'; -import 'package:syphon/store/rooms/state.dart'; -import 'package:syphon/store/settings/state.dart'; -import 'package:syphon/store/sync/state.dart'; -import 'package:syphon/store/user/state.dart'; - -/** - * - * Ripper API (temp) - * - * One way convertion of the Hive cache to a manually encrypted / encoded state cache - */ -class LegacyEncoder { - static Future encodeHive(AppState state) async { - try { - Cache.state.put( - state.syncStore.runtimeType.toString(), - state.syncStore, - ); - // debugPrint('[Hive Storage] caching syncStore'); - } catch (error) { - debugPrint('[Hive Serializer Encode] $error'); - } - - try { - Cache.stateRooms.put( - state.roomStore.runtimeType.toString(), - state.roomStore, - ); - // debugPrint('[Hive Storage] caching roomStore'); - } catch (error) { - debugPrint('[Hive Serializer Encode] $error'); - } - - try { - Cache.state.put( - state.mediaStore.runtimeType.toString(), - state.mediaStore, - ); - // debugPrint('[Hive Storage] caching mediaStore'); - } catch (error) { - debugPrint('[Hive Serializer Encode] $error'); - } - - try { - Cache.state.put( - state.settingsStore.runtimeType.toString(), - state.settingsStore, - ); - // debugPrint('[Hive Storage] caching settingsStore'); - } catch (error) { - debugPrint('[Hive Serializer Encode] $error'); - } - - try { - Cache.state.put( - state.cryptoStore.runtimeType.toString(), - state.cryptoStore, - ); - // debugPrint('[Hive Storage] caching cryptoStore'); - } catch (error) { - debugPrint('[Hive Serializer Encode] $error'); - } - } - - static AppState decodeHive() { - AuthStore authStoreConverted = AuthStore(); - SyncStore syncStoreConverted = SyncStore(); - CryptoStore cryptoStoreConverted = CryptoStore(); - MediaStore mediaStoreConverted = MediaStore(); - RoomStore roomStoreConverted = RoomStore(); - SettingsStore settingsStoreConverted = SettingsStore(); - - try { - authStoreConverted = Cache.state.get( - authStoreConverted.runtimeType.toString(), - defaultValue: null, - ); - } catch (error) { - debugPrint('[Hive Serializer Decode] $error'); - } - - try { - syncStoreConverted = Cache.state.get( - syncStoreConverted.runtimeType.toString(), - defaultValue: null, - ); - } catch (error) { - debugPrint('[Hive Serializer Decode] $error'); - } - - try { - cryptoStoreConverted = Cache.state.get( - cryptoStoreConverted.runtimeType.toString(), - defaultValue: null, - ); - } catch (error) { - debugPrint('[Hive Serializer Decode] $error'); - } - - try { - roomStoreConverted = Cache.stateRooms.get( - roomStoreConverted.runtimeType.toString(), - defaultValue: null, - ); - } catch (error) { - debugPrint('[Hive Serializer Decode] $error'); - } - - try { - mediaStoreConverted = Cache.state.get( - mediaStoreConverted.runtimeType.toString(), - defaultValue: null, - ); - } catch (error) { - debugPrint('[Hive Serializer Decode] $error'); - } - - try { - settingsStoreConverted = Cache.state.get( - settingsStoreConverted.runtimeType.toString(), - defaultValue: null, - ); - } catch (error) { - debugPrint('[Hive Serializer Decode] $error'); - } - - return AppState( - loading: false, - authStore: authStoreConverted ?? AuthStore(), - syncStore: syncStoreConverted ?? SyncStore(), - cryptoStore: cryptoStoreConverted ?? CryptoStore(), - roomStore: roomStoreConverted ?? RoomStore(), - mediaStore: mediaStoreConverted ?? MediaStore(), - settingsStore: settingsStoreConverted ?? SettingsStore(), - userStore: UserStore(), - ); - } -} diff --git a/lib/global/libs/hive/index.dart b/lib/global/libs/hive/index.dart deleted file mode 100644 index a64834831..000000000 --- a/lib/global/libs/hive/index.dart +++ /dev/null @@ -1,336 +0,0 @@ -// Dart imports: -import 'dart:io'; - -// Flutter imports: -import 'package:flutter/material.dart'; - -// Package imports: -import 'package:convert/convert.dart'; -import 'package:flutter_secure_storage/flutter_secure_storage.dart'; -import 'package:hive/hive.dart'; -import 'package:path_provider/path_provider.dart'; - -// Project imports: -import 'package:syphon/global/themes.dart'; -import 'package:syphon/global/values.dart'; -import 'package:syphon/store/auth/state.dart'; -import 'package:syphon/store/crypto/keys/model.dart'; -import 'package:syphon/store/crypto/model.dart'; -import 'package:syphon/store/crypto/state.dart'; -import 'package:syphon/store/media/state.dart'; -import 'package:syphon/store/rooms/events/ephemeral/m.read/model.dart'; -import 'package:syphon/store/rooms/events/model.dart'; -import 'package:syphon/store/rooms/room/model.dart'; -import 'package:syphon/store/rooms/state.dart'; -import 'package:syphon/store/settings/chat-settings/model.dart'; -import 'package:syphon/store/settings/devices-settings/model.dart'; -import 'package:syphon/store/settings/state.dart'; -import 'package:syphon/store/sync/state.dart'; -import 'package:syphon/store/user/model.dart'; - -// Global cache -class Cache { - static Box state; - static Box stateRooms; - static Box stateMedia; - static LazyBox sync; - - static String migration; - - static const group_id = '${Values.appNameLabel}'; - static const encryptionKeyLocation = '${Values.appNameLabel}@publicKey'; - - static const syncKey = '${Values.appNameLabel}_sync'; - static const stateKey = '${Values.appNameLabel}_cache'; - static const stateRoomKey = '${Values.appNameLabel}_cache_2'; - - static const syncKeyUNSAFE = '${Values.appNameLabel}_sync_unsafe'; - static const stateKeyUNSAFE = '${Values.appNameLabel}_cache_unsafe'; - static const stateKeyRoomsUNSAFE = - '${Values.appNameLabel}_cache_rooms_unsafe'; - - static const backgroundKeyUNSAFE = - '${Values.appNameLabel}_background_cache_unsafe_alt'; - - static const roomNames = 'room_names'; - static const syncData = 'sync_data'; - static const protocol = 'protocol'; - static const homeserver = 'homeserver'; - static const accessTokenKey = 'accessToken'; - static const lastSinceKey = 'lastSince'; - static const currentUser = 'currentUser'; - - static const migrationKey = 'migrationKey'; -} - -/** - * TODO: Whole cache with this configuration will be deprecated after 0.1.4 - */ - -/** - * - * Init Hive - * - * For testing purposes only - should be encrypting hive - */ -Future initHive() async { - // NOTE: done in initCache - // Init storage location - final storageLocation = await initStorageLocation(); - - // Init hive cache - // Hive.init(storageLocation); - - // Init configuration - await initHiveConfiguration(); - - // ignore if already migrated cache - final storageEngine = FlutterSecureStorage(); - Cache.migration = await storageEngine.read(key: Cache.migrationKey); - if (Cache.migration != null) return; - - // otherwise, open and load the previous hive cache - if ((Platform.isAndroid || Platform.isIOS)) { - Cache.sync = await openHiveSync(); - Cache.state = await openHiveState(); - Cache.stateRooms = await openHiveStateRooms(); - } -} - -Future initStorageLocation() async { - var storageLocation; - - try { - if (Platform.isIOS || Platform.isAndroid) { - storageLocation = await getApplicationDocumentsDirectory(); - return storageLocation.path; - } - - if (Platform.isMacOS) { - storageLocation = await File('cache').create().then( - (value) => value.writeAsString( - '{}', - flush: true, - ), - ); - - return storageLocation.path; - } - - if (Platform.isLinux) { - storageLocation = await getApplicationDocumentsDirectory(); - return storageLocation.path; - } - - debugPrint('[initStorageLocation] no cache support'); - return null; - } catch (error) { - debugPrint('[initStorageLocation] $error'); - return null; - } -} - -Future initHiveConfiguration() async { -// Future initHiveConfiguration(String storageLocationPath) async { - - // Init Custom Models - Hive.registerAdapter(ThemeTypeAdapter()); - Hive.registerAdapter(ChatSettingAdapter()); - Hive.registerAdapter(RoomAdapter()); - Hive.registerAdapter(MessageAdapter()); - Hive.registerAdapter(EventAdapter()); - Hive.registerAdapter(ReadStatusAdapter()); - Hive.registerAdapter(UserAdapter()); - Hive.registerAdapter(DeviceAdapter()); - Hive.registerAdapter(DeviceKeyAdapter()); - Hive.registerAdapter(OneTimeKeyAdapter()); - // Hive.registerAdapter(AccountAdapter()); - - // Custom Store Models - Hive.registerAdapter(AuthStoreAdapter()); - Hive.registerAdapter(SyncStoreAdapter()); - Hive.registerAdapter(CryptoStoreAdapter()); - Hive.registerAdapter(RoomStoreAdapter()); - Hive.registerAdapter(MediaStoreAdapter()); - Hive.registerAdapter(SettingsStoreAdapter()); -} - -Future> unlockEncryptionKey() async { - // Check if storage has been created before - final storageEngine = FlutterSecureStorage(); - - var encryptionKey = await storageEngine.read( - key: Cache.encryptionKeyLocation, - ); - - // Create a encryptionKey if a serialized one is not found - if (encryptionKey == null) { - encryptionKey = hex.encode(Hive.generateSecureKey()); - - await storageEngine.write( - key: Cache.encryptionKeyLocation, - value: encryptionKey, - ); - } - - return hex.decode(encryptionKey); -} - -/** - * openHiveState UNSAFE - * - * For testing purposes only - should be encrypting hive - */ -Future openHiveBackgroundUnsafe() async { - var storageLocation; - - // Init storage location - try { - storageLocation = await getApplicationDocumentsDirectory(); - } catch (error) { - debugPrint('[openHiveBackgroundUnsafe] Storage Failure $error'); - } - - // Init hive cache + adapters - Hive.init(storageLocation.path); - return await Hive.openBox(Cache.backgroundKeyUNSAFE); -} - -/** - * Open Hive State - * - * Separating the rest of state from room data to - * improve performance - * Initializes encrypted storage for caching current state - */ -Future openHiveState() async { - try { - final encryptionKey = await unlockEncryptionKey(); - - return await Hive.openBox( - Cache.stateKey, - crashRecovery: false, - encryptionCipher: HiveAesCipher(encryptionKey), - compactionStrategy: (entries, deletedEntries) => deletedEntries > 1, - ); - } catch (error) { - debugPrint('[openHiveState] $error'); - return null; - } -} - -/** - * Open Hive State - * - * Initializes encrypted storage for caching current state - */ -Future openHiveStateRooms() async { - try { - final encryptionKey = await unlockEncryptionKey(); - - return await Hive.openBox( - Cache.stateRoomKey, - crashRecovery: false, - encryptionCipher: HiveAesCipher(encryptionKey), - compactionStrategy: (entries, deletedEntries) => deletedEntries > 1, - ); - } catch (error) { - debugPrint('[openHiveState] $error'); - return null; - } -} - -/** - * Open Hive Sync - * - * Initializes encrypted storage for caching sync - */ -Future openHiveSync() async { - try { - final encryptionKey = await unlockEncryptionKey(); - - return await Hive.openLazyBox( - Cache.syncKey, - crashRecovery: false, - encryptionCipher: HiveAesCipher(encryptionKey), - compactionStrategy: (entries, deletedEntries) => deletedEntries > 1, - ); - } catch (error) { - debugPrint('[openHiveState] $error'); - return null; - } -} - -// // Closes and saves storage -void closeStorage() async { - if (Cache.sync != null && Cache.sync.isOpen) { - Cache.sync.close(); - } - - if (Cache.state != null && Cache.state.isOpen) { - Cache.sync.close(); - } -} - -// Wipe all old cached files to deprecate old storage caches -Future deleteLegacyStorage() async { - try { - (await Hive.openBox(Cache.syncKeyUNSAFE)).deleteFromDisk(); - debugPrint('[deleteStorage] deleting Cache.state'); - } catch (error) { - debugPrint('[deleteStorage] ${error}'); - } - - try { - (await Hive.openBox(Cache.stateKeyUNSAFE)).deleteFromDisk(); - debugPrint('[deleteStorage] deleting Cache.stateKeyUNSAFE'); - } catch (error) { - debugPrint('[deleteStorage] ${error}'); - } - - try { - (await Hive.openBox(Cache.stateKeyRoomsUNSAFE)).deleteFromDisk(); - debugPrint('[deleteStorage] deleting Cache.stateKeyRoomsUNSAFE'); - } catch (error) { - debugPrint('[deleteStorage] ${error}'); - } - - try { - (await Hive.openBox(Cache.backgroundKeyUNSAFE)).deleteFromDisk(); - debugPrint('[deleteStorage] deleting Cache.backgroundKeyUNSAFE'); - } catch (error) { - debugPrint('[deleteStorage] ${error}'); - } - - try { - (await Hive.openLazyBox(Cache.syncKey)).deleteFromDisk(); - debugPrint('[deleteStorage] deleting Cache.syncKey'); - } catch (error) { - debugPrint('[deleteStorage] ${error}'); - } - - try { - (await Hive.openBox(Cache.stateKey)).deleteFromDisk(); - debugPrint('[deleteStorage] deleting Cache.stateKey'); - } catch (error) { - debugPrint('[deleteStorage] ${error}'); - } - - try { - (await Hive.openBox(Cache.stateRoomKey)).deleteFromDisk(); - debugPrint('[deleteStorage] deleting Cache.stateRoomKey'); - } catch (error) { - debugPrint('[deleteStorage] ${error}'); - } - - Cache.sync = null; - Cache.state = null; - Cache.stateRooms = null; - - final storageEngine = FlutterSecureStorage(); - await storageEngine.write(key: Cache.migrationKey, value: 'yes'); - - debugPrint('[deleteLegacyStorage] ran and saved migration status'); - - return true; -} diff --git a/lib/global/libs/hive/type-ids.dart b/lib/global/libs/hive/type-ids.dart deleted file mode 100644 index 06a5a7a09..000000000 --- a/lib/global/libs/hive/type-ids.dart +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Top Level accounting for hive ids assigned in the persisted cache - */ - -// Objects -const int ThemeTypeHiveId = 2; -const int ChatSettingsHiveId = 3; -const int MessageHiveId = 4; -const int EventHiveId = 5; -const int RoomHiveId = 6; -const int ReadStatusHiveId = 7; -const int UserHiveId = 8; -const int DevicesHiveId = 9; -const int NotificationSettingsHiveId = 10; -const int PusherHiveId = 11; -const int RuleHiveId = 12; -const int DeviceKeyHiveId = 13; -const int OneTimeKeyHiveId = 14; - -// Actual Redux State Stores -const int MediaStoreHiveId = 0; -const int SettingsStoreHiveId = 1; -const int RoomStoreHiveId = 100; -const int AuthStoreHiveId = 101; -const int UserStoreHiveId = 102; -const int SyncStoreHiveId = 103; -const int CryptoStoreHiveId = 104; diff --git a/lib/global/themes.dart b/lib/global/themes.dart index 48ec6c357..5375abb82 100644 --- a/lib/global/themes.dart +++ b/lib/global/themes.dart @@ -1,24 +1,12 @@ // Flutter imports: import 'package:flutter/material.dart'; -// Package imports: -import 'package:hive/hive.dart'; - -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; import 'colours.dart'; -part 'themes.g.dart'; - -@HiveType(typeId: ThemeTypeHiveId) enum ThemeType { - @HiveField(0) LIGHT, - @HiveField(1) DARK, - @HiveField(2) DARKER, - @HiveField(3) NIGHT, } diff --git a/lib/main.dart b/lib/main.dart index 9df1407f8..9b2ce2d9b 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -16,7 +16,6 @@ import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/formatters.dart'; // Project imports: -import 'package:syphon/global/libs/hive/index.dart'; import 'package:syphon/global/themes.dart'; import 'package:syphon/store/alerts/actions.dart'; import 'package:syphon/store/auth/actions.dart'; @@ -59,9 +58,6 @@ void main() async { // init cold cache (mobile only) await initCache(); - // TODO: remove after 0.1.4 - await initHive(); - // init hot cache and start runApp(Syphon(store: await initStore())); } @@ -190,7 +186,6 @@ class SyphonState extends State with WidgetsBindingObserver { @override void deactivate() { - closeStorage(); closeCache(); WidgetsBinding.instance.removeObserver(this); store.dispatch(stopAuthObserver()); diff --git a/lib/store/auth/state.dart b/lib/store/auth/state.dart index 6d60b9f08..fb4020a67 100644 --- a/lib/store/auth/state.dart +++ b/lib/store/auth/state.dart @@ -3,23 +3,20 @@ import 'dart:async'; // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; // Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; import 'package:syphon/global/values.dart'; import 'package:syphon/store/auth/credential/model.dart'; import 'package:syphon/store/user/model.dart'; part 'state.g.dart'; -@HiveType(typeId: AuthStoreHiveId) @JsonSerializable(ignoreUnannotated: true) class AuthStore extends Equatable { - @HiveField(0) @JsonKey(name: 'user') final User user; + User get currentUser => user; final StreamController authObserver; diff --git a/lib/store/crypto/keys/model.dart b/lib/store/crypto/keys/model.dart index 2e017fae0..5e112793b 100644 --- a/lib/store/crypto/keys/model.dart +++ b/lib/store/crypto/keys/model.dart @@ -1,51 +1,15 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - part 'model.g.dart'; -/** - * - * OneTimeKey Data Model - * - * - * https://matrix.org/docs/spec/client_server/latest#id468 - * { - "failures": {}, - "one_time_keys": { - "@alice:example.com": { - "JLAFKJWSCS": { - "signed_curve25519:AAAAHg": { - "key": "zKbLg+NrIjpnagy+pIY6uPL4ZwEG2v+8F9lmgsnlZzs", - "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "FLWxXqGbwrb8SM3Y795eB6OA8bwBcoMZFXBqnTn58AYWZSqiD45tlBVcDa2L7RwdKXebW/VzDlnfVJ+9jok1Bw" - } - } - } - } - } - } - } - */ -@HiveType(typeId: OneTimeKeyHiveId) @JsonSerializable() class OneTimeKey extends Equatable { - @HiveField(0) final String userId; - @HiveField(1) final String deviceId; - - // Map - @HiveField(2) - final Map keys; - - // Map - @HiveField(3) + final Map keys; // Map + // Map> final Map> signatures; const OneTimeKey({ diff --git a/lib/store/crypto/model.dart b/lib/store/crypto/model.dart index 6b64913a9..bfc3f32b7 100644 --- a/lib/store/crypto/model.dart +++ b/lib/store/crypto/model.dart @@ -1,34 +1,19 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; - -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; import 'package:syphon/global/libs/matrix/encryption.dart'; part 'model.g.dart'; -@HiveType(typeId: DeviceKeyHiveId) @JsonSerializable() class DeviceKey extends Equatable { - @HiveField(0) final String userId; - @HiveField(1) final String deviceId; - @HiveField(2) final List algorithms; - @HiveField(3) final Map keys; - @HiveField(4) final Map signatures; - @HiveField(5) final Map extras; - // DEPRRECATED - @HiveField(6) - final Map privateKeys; - const DeviceKey({ this.userId, this.deviceId, @@ -36,7 +21,6 @@ class DeviceKey extends Equatable { this.keys, this.signatures, this.extras, - this.privateKeys, }); @override @@ -47,7 +31,6 @@ class DeviceKey extends Equatable { keys, signatures, extras, - privateKeys, ]; Map toMatrix() { diff --git a/lib/store/crypto/state.dart b/lib/store/crypto/state.dart index f40f1f6bd..7102a8c67 100644 --- a/lib/store/crypto/state.dart +++ b/lib/store/crypto/state.dart @@ -1,75 +1,58 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; import 'package:olm/olm.dart'; // Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; import 'package:syphon/store/crypto/keys/model.dart'; import 'package:syphon/store/crypto/model.dart'; part 'state.g.dart'; -// Next Hive Field Number: 14 -@HiveType(typeId: CryptoStoreHiveId) @JsonSerializable() class CryptoStore extends Equatable { // Active olm account (loaded from olmAccountKey) @JsonKey(ignore: true) final Account olmAccount; + // the private key for one time keys is saved in olm? + // Map deviceKeys + @JsonKey(ignore: true) + final Map oneTimeKeysOwned; + // Serialized olm account - @HiveField(3) final String olmAccountKey; // Map // megolm - message index - @HiveField(10) final Map messageSessionIndex; // Map // megolm - index per chat - @HiveField(15) final Map> messageSessionIndexNEW; // Map // megolm - messages - @HiveField(5) final Map outboundMessageSessions; // Map // megolm - messages per chat - @HiveField(11) final Map> inboundMessageSessions; // Map // olmv1 - key-sharing per identity - @HiveField(8) final Map inboundKeySessions; // Map // olmv1 - key-sharing per identity - @HiveField(6) final Map outboundKeySessions; // Map deviceKeys - @HiveField(0) final Map> deviceKeys; // Map deviceKeysOwned - @HiveField(1) final Map deviceKeysOwned; // key is deviceId - @HiveField(2) final bool deviceKeysExist; // Track last known uploaded key amounts - @HiveField(7) final Map oneTimeKeysCounts; - @HiveField(9) - final Map oneTimeKeysClaimed; // claimed - - // @HiveField(?) TODO: consider saving generated keys? - // the private key for one time keys is saved in olm? - // Map deviceKeys - @JsonKey(ignore: true) - final Map oneTimeKeysOwned; + final Map oneTimeKeysClaimed; const CryptoStore({ this.olmAccount, diff --git a/lib/store/media/state.dart b/lib/store/media/state.dart index 324e91b58..23ac8d892 100644 --- a/lib/store/media/state.dart +++ b/lib/store/media/state.dart @@ -3,31 +3,13 @@ import 'dart:typed_data'; // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; -import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - -part 'state.g.dart'; - -// NOTE: custom json converter to allow Uint8List when in cache -// TODO: figure out how to make image-matrix.dart play nice with in component coonversions -// Would repeatedly update even if a locally cached version matched // @JsonSerializable(nullable: true, includeIfNull: true) -@HiveType(typeId: MediaStoreHiveId) class MediaStore extends Equatable { - @HiveField(0) final bool fetching; - @HiveField(1) + final Map mediaChecks; // Map final Map mediaCache; - // Map - @HiveField(2) - final Map mediaChecks; - - static const hiveBox = 'MediaStore'; - const MediaStore({ this.fetching = false, this.mediaCache = const {}, @@ -40,6 +22,7 @@ class MediaStore extends Equatable { mediaCache, mediaChecks, ]; + MediaStore copyWith({ fetching, mediaCache, @@ -51,6 +34,9 @@ class MediaStore extends Equatable { mediaChecks: mediaChecks ?? this.mediaChecks, ); + // NOTE: custom json converter to allow Uint8List when in cache + // TODO: figure out how to make image-matrix.dart play nice with in component coonversions + // Would repeatedly update even if a locally cached version matched factory MediaStore.fromJson(Map json) { return MediaStore( fetching: json['fetching'] as bool, diff --git a/lib/store/rooms/events/ephemeral/m.read/model.dart b/lib/store/rooms/events/ephemeral/m.read/model.dart index 57cabba88..f4f11cfbd 100644 --- a/lib/store/rooms/events/ephemeral/m.read/model.dart +++ b/lib/store/rooms/events/ephemeral/m.read/model.dart @@ -1,21 +1,12 @@ // Package imports: -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - part 'model.g.dart'; -@HiveType(typeId: ReadStatusHiveId) @JsonSerializable() class ReadStatus { - @HiveField(0) final int latestRead; - - // UserId -> timestamp - @HiveField(1) - final Map userReads; + final Map userReads; // UserId -> timestamp const ReadStatus({ this.latestRead = 0, diff --git a/lib/store/rooms/events/model.dart b/lib/store/rooms/events/model.dart index a7bec9dd6..9f8e94100 100644 --- a/lib/store/rooms/events/model.dart +++ b/lib/store/rooms/events/model.dart @@ -1,10 +1,6 @@ // Package imports: -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - part 'model.g.dart'; /** @@ -25,9 +21,6 @@ class EventTypes { static const creation = 'm.room.create'; static const message = 'm.room.message'; static const encrypted = 'm.room.encrypted'; - - // {membership: join, displayname: usbfingers, avatar_url: mxc://matrix.org/RrRcMHnqXaJshyXZpGrZloyh } - // {is_direct: true, membership: invite, displayname: ereio, avatar_url: mxc://matrix.org/JllILpqzdFAUOvrTPSkDryzW} static const member = 'm.room.member'; static const guestAccess = 'm.room.guest_access'; @@ -58,22 +51,14 @@ class MediumType { static const encryption = 'encryption'; } -@HiveType(typeId: EventHiveId) @JsonSerializable() class Event { - @HiveField(0) final String id; // event_id - @HiveField(1) final String userId; - @HiveField(2) final String roomId; - @HiveField(3) final String type; - @HiveField(4) final String sender; - @HiveField(5) final String stateKey; - @HiveField(6) final int timestamp; /* @@ -127,51 +112,31 @@ class Event { } // TODO: make this actually inherit Event but also allow immutability (dart says no?) -@HiveType(typeId: MessageHiveId) @JsonSerializable() class Message { - @HiveField(0) final String id; // event_id - @HiveField(1) final String userId; - @HiveField(2) final String roomId; - @HiveField(3) final String type; - @HiveField(4) final String sender; - @HiveField(5) final String stateKey; - @HiveField(6) final int timestamp; - @HiveField(7) final bool pending; - @HiveField(8) final bool syncing; - @HiveField(9) final bool failed; // Message Only - @HiveField(10) final String body; - @HiveField(11) final String msgtype; - @HiveField(12) final String format; - @HiveField(13) final String filename; - @HiveField(14) final String formattedBody; // Encrypted Messages only - @HiveField(15) final String ciphertext; - @HiveField(16) final String algorithm; - // The Curve25519 key of the device which initiated the session originally. - @HiveField(17) - final String senderKey; + final String senderKey; // Curve25519 device key which initiated the session /* * TODO: content will not always be a string? configure parsing data diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index 63c151b0d..c11e74585 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -2,19 +2,14 @@ import 'dart:collection'; // Package imports: -import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -import 'package:syphon/global/algos.dart'; // Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; import 'package:syphon/global/strings.dart'; import 'package:syphon/store/rooms/events/ephemeral/m.read/model.dart'; import 'package:syphon/store/rooms/events/model.dart'; import 'package:syphon/store/user/model.dart'; -import 'package:syphon/store/user/selectors.dart'; part 'model.g.dart'; @@ -24,85 +19,51 @@ class RoomPresets { static const public = 'public_chat'; } -@HiveType(typeId: RoomHiveId) @JsonSerializable() class Room { - @HiveField(0) final String id; - @HiveField(1) final String name; - @HiveField(2) final String alias; - @HiveField(3) final String homeserver; - @HiveField(4) final String avatarUri; - @HiveField(5) final String topic; - @HiveField(29) final String joinRule; // "public", "knock", "invite", "private" - @HiveField(28) final int namePriority; - @HiveField(6) + final int lastRead; final bool direct; - @HiveField(7) final bool syncing; - @HiveField(8) final bool sending; - @HiveField(9) final bool isDraftRoom; + final bool invite; + final bool guestEnabled; + final bool encryptionEnabled; + final bool worldReadable; - @HiveField(11) final String lastHash; // oldest hash in timeline - @HiveField(26) final String prevHash; // most recent prev_batch (not the lastHash) - @HiveField(10) final String nextHash; // most recent next_batch - @HiveField(12) final int lastUpdate; - @HiveField(13) final int totalJoinedUsers; - @HiveField(14) - final bool guestEnabled; - @HiveField(15) - final bool encryptionEnabled; - @HiveField(16) - final bool worldReadable; - // Event lists and handlers - @HiveField(17) final Message draft; - // TODO: removed until state timeline work can be done + // TODO: removed until state timeline work can be done // final List state; - - @HiveField(20) final List messages; - - @HiveField(21) final List outbox; - // Not cached + final Map users; + final Map messageReads; + @JsonKey(ignore: true) final bool userTyping; + @JsonKey(ignore: true) final List usersTyping; - @HiveField(23) - final int lastRead; - - @HiveField(24) - final Map messageReads; - - @HiveField(25) - final Map users; - - @HiveField(27) - final bool invite; - @JsonKey(ignore: true) final bool limited; diff --git a/lib/store/rooms/state.dart b/lib/store/rooms/state.dart index 07fbaa42c..aaaf23ce8 100644 --- a/lib/store/rooms/state.dart +++ b/lib/store/rooms/state.dart @@ -3,33 +3,21 @@ import 'dart:async'; // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; import './room/model.dart'; part 'state.g.dart'; -@HiveType(typeId: RoomStoreHiveId) @JsonSerializable() class RoomStore extends Equatable { - @JsonKey(ignore: true) - final bool loading; - - @HiveField(0) final bool synced; - - @HiveField(3) final int lastUpdate; // Last timestamp for actual new info + final Map rooms; // consider renaming to nextBatch - @HiveField(4) - final String lastSince; // Since we last checked for new info - - @HiveField(5) - final Map rooms; + // Since we last checked for new info + final String lastSince; @JsonKey(ignore: true) final Map archive; // TODO: actually archive @@ -39,6 +27,9 @@ class RoomStore extends Equatable { @JsonKey(ignore: true) final Timer roomObserver; + @JsonKey(ignore: true) + final bool loading; + bool get isSynced => lastUpdate != null && lastUpdate != 0; List get roomList => rooms != null ? List.from(rooms.values) : []; diff --git a/lib/store/settings/chat-settings/model.dart b/lib/store/settings/chat-settings/model.dart index 4707cef18..43d1a0a7d 100644 --- a/lib/store/settings/chat-settings/model.dart +++ b/lib/store/settings/chat-settings/model.dart @@ -1,26 +1,18 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; // Project imports: import 'package:syphon/global/colours.dart'; -import 'package:syphon/global/libs/hive/type-ids.dart'; part 'model.g.dart'; -@HiveType(typeId: ChatSettingsHiveId) @JsonSerializable() class ChatSetting extends Equatable { - @HiveField(0) final String roomId; - @HiveField(1) final int primaryColor; - @HiveField(2) final bool smsEnabled; - @HiveField(3) final bool notificationsEnabled; - @HiveField(4) final String language; const ChatSetting({ diff --git a/lib/store/settings/devices-settings/model.dart b/lib/store/settings/devices-settings/model.dart index fe06c7dbf..4cda415a7 100644 --- a/lib/store/settings/devices-settings/model.dart +++ b/lib/store/settings/devices-settings/model.dart @@ -1,25 +1,15 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - part 'model.g.dart'; -@HiveType(typeId: DevicesHiveId) @JsonSerializable() class Device extends Equatable { - @HiveField(0) final String deviceId; - @HiveField(4) final String deviceIdPrivate; - @HiveField(1) final String displayName; - @HiveField(2) final String lastSeenIp; - @HiveField(3) final int lastSeenTs; const Device({ diff --git a/lib/store/settings/notification-settings/model.dart b/lib/store/settings/notification-settings/model.dart index bd0827b7f..cacab44b8 100644 --- a/lib/store/settings/notification-settings/model.dart +++ b/lib/store/settings/notification-settings/model.dart @@ -1,23 +1,17 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; // Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; import './pushers/model.dart'; import './rules/model.dart'; part 'model.g.dart'; -@HiveType(typeId: NotificationSettingsHiveId) @JsonSerializable() class NotificationSettings extends Equatable { - @HiveField(0) - final List pushers; - - @HiveField(1) final List rules; + final List pushers; const NotificationSettings({this.pushers, this.rules}); diff --git a/lib/store/settings/notification-settings/pushers/model.dart b/lib/store/settings/notification-settings/pushers/model.dart index e9c17d409..69fefd14c 100644 --- a/lib/store/settings/notification-settings/pushers/model.dart +++ b/lib/store/settings/notification-settings/pushers/model.dart @@ -1,23 +1,14 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - part 'model.g.dart'; -@HiveType(typeId: PusherHiveId) @JsonSerializable() class Pusher extends Equatable { - @HiveField(0) final String key; - @HiveField(1) final String kind; - @HiveField(2) final String appId; - @HiveField(3) final String appDisplayName; const Pusher({ diff --git a/lib/store/settings/notification-settings/rules/model.dart b/lib/store/settings/notification-settings/rules/model.dart index 3e3b23a77..afd63b94b 100644 --- a/lib/store/settings/notification-settings/rules/model.dart +++ b/lib/store/settings/notification-settings/rules/model.dart @@ -1,23 +1,13 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - part 'model.g.dart'; -@HiveType(typeId: RuleHiveId) @JsonSerializable() class Rule extends Equatable { - @HiveField(0) final String id; // rule_id - - @HiveField(1) final bool enabled; - - @HiveField(2) final bool isDefault; // determine if these can be saved without being parsed diff --git a/lib/store/settings/state.dart b/lib/store/settings/state.dart index 47be1f261..12bfe9ead 100644 --- a/lib/store/settings/state.dart +++ b/lib/store/settings/state.dart @@ -1,12 +1,10 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; // Project imports: import "package:syphon/global/themes.dart"; import 'package:syphon/global/colours.dart'; -import 'package:syphon/global/libs/hive/type-ids.dart'; import 'package:syphon/store/settings/devices-settings/model.dart'; import 'package:syphon/store/settings/notification-settings/model.dart'; import './chat-settings/model.dart'; @@ -14,63 +12,41 @@ import './chat-settings/model.dart'; part 'state.g.dart'; // Next Field ID: 21 -@HiveType(typeId: SettingsStoreHiveId) @JsonSerializable() class SettingsStore extends Equatable { - @HiveField(0) final int primaryColor; - @HiveField(1) final int accentColor; - @HiveField(15) final int appBarColor; - @HiveField(2) final int brightness; - @HiveField(10) final ThemeType theme; - @HiveField(4) final bool enterSend; // TODO: rename *enabled - @HiveField(3) final bool smsEnabled; - @HiveField(5) final bool readReceipts; // TODO: rename *enabled - @HiveField(6) final bool typingIndicators; // TODO: rename *enabled - @HiveField(7) final bool notificationsEnabled; - @HiveField(8) final bool membershipEventsEnabled; - @HiveField(18) final bool roomTypeBadgesEnabled; - @HiveField(19) final bool timeFormat24Enabled; - @HiveField(16) final String fontName; - @HiveField(17) final String fontSize; - @HiveField(9) final String language; - @HiveField(20) final String avatarShape; - @HiveField(12) final List devices; // Map - @HiveField(11) final Map customChatSettings; - @HiveField(13) final NotificationSettings notificationSettings; - @HiveField(14) final String alphaAgreement; // a timestamp of agreement for alpha TOS - @JsonKey(ignore: true) // temp + @JsonKey(ignore: true) final String pusherToken; // NOTE: can be device token for APNS - @JsonKey(ignore: true) // temp + @JsonKey(ignore: true) final bool loading; const SettingsStore({ diff --git a/lib/store/sync/background/service.dart b/lib/store/sync/background/service.dart index 0862e4cd0..69e9fd8c1 100644 --- a/lib/store/sync/background/service.dart +++ b/lib/store/sync/background/service.dart @@ -18,7 +18,6 @@ import 'dart:math'; // Package imports: import 'package:flutter_local_notifications/flutter_local_notifications.dart'; -import 'package:hive/hive.dart'; // Project imports: import 'package:syphon/global/libs/matrix/index.dart'; @@ -178,9 +177,8 @@ void notificationSyncIsolate() async { * Save Full Sync */ FutureOr syncLoop({ - Box cache, - FlutterLocalNotificationsPlugin pluginInstance, Map params, + FlutterLocalNotificationsPlugin pluginInstance, }) async { try { final protocol = params['protocol']; diff --git a/lib/store/sync/state.dart b/lib/store/sync/state.dart index 1d8f49511..fb6e95b45 100644 --- a/lib/store/sync/state.dart +++ b/lib/store/sync/state.dart @@ -3,32 +3,23 @@ import 'dart:async'; // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - part 'state.g.dart'; -@HiveType(typeId: SyncStoreHiveId) @JsonSerializable(ignoreUnannotated: true) class SyncStore extends Equatable { - @HiveField(0) @JsonKey(name: 'synced') final bool synced; - @HiveField(3) @JsonKey(name: 'lastUpdate') final int lastUpdate; // Last timestamp for actual new info - @HiveField(4) @JsonKey(name: 'lastSince') final String lastSince; // Since we last checked for new info static const default_interval = 1; - @HiveField(5) @JsonKey(name: 'interval') final int interval = default_interval; @@ -40,11 +31,9 @@ class SyncStore extends Equatable { final bool unauthed; final Timer syncObserver; - @HiveField(6) @JsonKey(name: 'lastAttempt') final int lastAttempt; // last attempt to sync - @HiveField(7) @JsonKey(name: 'backgrounded') final bool backgrounded; diff --git a/lib/store/user/model.dart b/lib/store/user/model.dart index d1f2e9409..2068a9d05 100644 --- a/lib/store/user/model.dart +++ b/lib/store/user/model.dart @@ -1,32 +1,19 @@ // Package imports: import 'package:equatable/equatable.dart'; import 'package:flutter/material.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; -// Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; - part 'model.g.dart'; -@HiveType(typeId: UserHiveId) @JsonSerializable() class User extends Equatable { - @HiveField(0) final String userId; - @HiveField(1) final String deviceId; // current device id - @HiveField(7) final String idserver; - @HiveField(2) final String homeserver; - @HiveField(6) final String homeserverName; - @HiveField(3) final String accessToken; - @HiveField(4) final String displayName; - @HiveField(5) final String avatarUri; const User({ diff --git a/lib/store/user/state.dart b/lib/store/user/state.dart index 01c705b12..7023ad150 100644 --- a/lib/store/user/state.dart +++ b/lib/store/user/state.dart @@ -1,24 +1,19 @@ // Package imports: import 'package:equatable/equatable.dart'; -import 'package:hive/hive.dart'; import 'package:json_annotation/json_annotation.dart'; // Project imports: -import 'package:syphon/global/libs/hive/type-ids.dart'; import 'package:syphon/store/user/model.dart'; part 'state.g.dart'; -@HiveType(typeId: UserStoreHiveId) @JsonSerializable(nullable: true, includeIfNull: true) class UserStore extends Equatable { + final Map users; + @JsonKey(ignore: true) final bool loading; - @HiveField(0) - final Map users; - - @HiveField(1) @JsonKey(ignore: true) final List invites; From 076a0437d61d22368d10a082715301f14bed0d24 Mon Sep 17 00:00:00 2001 From: ereio Date: Sat, 14 Nov 2020 00:14:06 -0500 Subject: [PATCH 02/25] removing old dependency overrides --- build.yaml | 8 -------- pubspec.lock | 29 ++++------------------------- pubspec.yaml | 13 ++++--------- 3 files changed, 8 insertions(+), 42 deletions(-) delete mode 100644 build.yaml diff --git a/build.yaml b/build.yaml deleted file mode 100644 index 5e8fd297e..000000000 --- a/build.yaml +++ /dev/null @@ -1,8 +0,0 @@ -targets: - $default: - builders: - reflectable: - generate_for: - - lib/main.dart - options: - formatted: true \ No newline at end of file diff --git a/pubspec.lock b/pubspec.lock index 4d297d952..d6f117ae1 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -79,7 +79,7 @@ packages: source: hosted version: "2.1.4" build_resolvers: - dependency: "direct dev" + dependency: transitive description: name: build_resolvers url: "https://pub.dartlang.org" @@ -205,19 +205,12 @@ packages: source: hosted version: "0.1.3" dart_style: - dependency: "direct overridden" + dependency: transitive description: name: dart_style url: "https://pub.dartlang.org" source: hosted version: "1.3.4" - dartx: - dependency: transitive - description: - name: dartx - url: "https://pub.dartlang.org" - source: hosted - version: "0.5.0" device_info: dependency: "direct main" description: @@ -422,13 +415,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.3.1" - hive_generator: - dependency: "direct dev" - description: - name: hive_generator - url: "https://pub.dartlang.org" - source: hosted - version: "0.8.1" html: dependency: "direct main" description: @@ -591,12 +577,12 @@ packages: source: hosted version: "0.2.3" path: - dependency: "direct overridden" + dependency: transitive description: name: path url: "https://pub.dartlang.org" source: hosted - version: "1.7.0" + version: "1.8.0-nullsafety.1" path_drawing: dependency: transitive description: @@ -917,13 +903,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.3.11" - time: - dependency: transitive - description: - name: time - url: "https://pub.dartlang.org" - source: hosted - version: "1.3.0" timing: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 365859166..108cde202 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -44,7 +44,7 @@ description: a privacy focused matrix client # The following defines the version and build number for your application. # A version number is three numbers separated by dots, like 1.2.43 # followed by an optional build number separated by a +. -version: 0.1.4+141 +version: 0.1.5+150 environment: sdk: ">=2.9.0-13.0 <3.0.0" # <- modified to solve build_runner @@ -74,11 +74,11 @@ dependencies: steel_crypt: ^2.2.2+1 olm: 1.2.1 # flutter_olm: 1.0.1 + # cryptography: 1.2.1 # olm: # git: # url: https://gitlab.com/famedly/libraries/dart-olm # ref: 4853b5301519a50866a81b58ac3730830a4fe547 - # cryptography: 1.2.1 # Cache hive: 1.4.4 @@ -123,15 +123,10 @@ dev_dependencies: json_serializable: ^3.5.0 flutter_launcher_icons: "^0.7.5" build_runner: ^1.10.1 - hive_generator: 0.8.1 - # TODO: build fixes - remove later - build_resolvers: 1.3.10 # <- modified to solve build_runner + # build_resolvers: 1.3.10 # <- modified to solve build_runner - # TODO: build fixes - remove later dependency_overrides: - path: 1.7.0 - analyzer: 0.39.16 - dart_style: 1.3.4 + analyzer: 0.39.16 # <- override to solve build_runner flutter_icons: android: true From 20b6b48ebea9ee7e6a4e36c918826d71aeb6d1c9 Mon Sep 17 00:00:00 2001 From: ereio Date: Sat, 14 Nov 2020 00:31:25 -0500 Subject: [PATCH 03/25] testing init for sembast + sqflite --- ios/Podfile.lock | 12 +++++++++++ ios/Runner.xcodeproj/project.pbxproj | 4 ++++ lib/global/cache/index.dart | 31 +++++++++++++++++++++++++++ pubspec.lock | 32 ++++++++++++++++++++++++++-- pubspec.yaml | 4 +++- 5 files changed, 80 insertions(+), 3 deletions(-) diff --git a/ios/Podfile.lock b/ios/Podfile.lock index c0ce1cf74..ff438e804 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -41,6 +41,9 @@ PODS: - Flutter - flutter_secure_storage (3.3.1): - Flutter + - FMDB (2.7.5): + - FMDB/standard (= 2.7.5) + - FMDB/standard (2.7.5) - image_picker (0.0.1): - Flutter - OLMKit (3.1.0): @@ -60,6 +63,9 @@ PODS: - SDWebImage/Core (~> 5.6) - shared_preferences (0.0.1): - Flutter + - sqflite (0.0.2): + - Flutter + - FMDB (>= 2.7.5) - url_launcher (0.0.1): - Flutter - webview_flutter (0.0.1): @@ -76,6 +82,7 @@ DEPENDENCIES: - package_info (from `.symlinks/plugins/package_info/ios`) - path_provider (from `.symlinks/plugins/path_provider/ios`) - shared_preferences (from `.symlinks/plugins/shared_preferences/ios`) + - sqflite (from `.symlinks/plugins/sqflite/ios`) - url_launcher (from `.symlinks/plugins/url_launcher/ios`) - webview_flutter (from `.symlinks/plugins/webview_flutter/ios`) @@ -84,6 +91,7 @@ SPEC REPOS: - DKImagePickerController - DKPhotoGallery - FLAnimatedImage + - FMDB - OLMKit - SDWebImage - SDWebImageFLPlugin @@ -107,6 +115,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/path_provider/ios" shared_preferences: :path: ".symlinks/plugins/shared_preferences/ios" + sqflite: + :path: ".symlinks/plugins/sqflite/ios" url_launcher: :path: ".symlinks/plugins/url_launcher/ios" webview_flutter: @@ -121,6 +131,7 @@ SPEC CHECKSUMS: Flutter: 0e3d915762c693b495b44d77113d4970485de6ec flutter_local_notifications: 9e4738ce2471c5af910d961a6b7eadcf57c50186 flutter_secure_storage: 7953c38a04c3fdbb00571bcd87d8e3b5ceb9daec + FMDB: 2ce00b547f966261cd18927a3ddb07cb6f3db82a image_picker: 9c3312491f862b28d21ecd8fdf0ee14e601b3f09 OLMKit: 4ee0159d63feeb86d836fdcfefe418e163511639 package_info: 873975fc26034f0b863a300ad47e7f1ac6c7ec62 @@ -128,6 +139,7 @@ SPEC CHECKSUMS: SDWebImage: 84000f962cbfa70c07f19d2234cbfcf5d779b5dc SDWebImageFLPlugin: 6c2295fb1242d44467c6c87dc5db6b0a13228fd8 shared_preferences: af6bfa751691cdc24be3045c43ec037377ada40d + sqflite: 6d358c025f5b867b29ed92fc697fd34924e11904 url_launcher: 6fef411d543ceb26efce54b05a0a40bfd74cbbef webview_flutter: d2b4d6c66968ad042ad94cbb791f5b72b4678a96 diff --git a/ios/Runner.xcodeproj/project.pbxproj b/ios/Runner.xcodeproj/project.pbxproj index 1db7ac857..0b5b6a538 100644 --- a/ios/Runner.xcodeproj/project.pbxproj +++ b/ios/Runner.xcodeproj/project.pbxproj @@ -219,6 +219,7 @@ "${BUILT_PRODUCTS_DIR}/DKImagePickerController/DKImagePickerController.framework", "${BUILT_PRODUCTS_DIR}/DKPhotoGallery/DKPhotoGallery.framework", "${BUILT_PRODUCTS_DIR}/FLAnimatedImage/FLAnimatedImage.framework", + "${BUILT_PRODUCTS_DIR}/FMDB/FMDB.framework", "${PODS_ROOT}/../Flutter/Flutter.framework", "${BUILT_PRODUCTS_DIR}/OLMKit/OLMKit.framework", "${BUILT_PRODUCTS_DIR}/SDWebImage/SDWebImage.framework", @@ -231,6 +232,7 @@ "${BUILT_PRODUCTS_DIR}/package_info/package_info.framework", "${BUILT_PRODUCTS_DIR}/path_provider/path_provider.framework", "${BUILT_PRODUCTS_DIR}/shared_preferences/shared_preferences.framework", + "${BUILT_PRODUCTS_DIR}/sqflite/sqflite.framework", "${BUILT_PRODUCTS_DIR}/url_launcher/url_launcher.framework", "${BUILT_PRODUCTS_DIR}/webview_flutter/webview_flutter.framework", ); @@ -239,6 +241,7 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/DKImagePickerController.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/DKPhotoGallery.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/FLAnimatedImage.framework", + "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/FMDB.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Flutter.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/OLMKit.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/SDWebImage.framework", @@ -251,6 +254,7 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/package_info.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/path_provider.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/shared_preferences.framework", + "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sqflite.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/url_launcher.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/webview_flutter.framework", ); diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index 439d9bbac..bdd7868fc 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -1,11 +1,16 @@ +import 'dart:convert'; import 'dart:io'; import 'package:flutter/material.dart'; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; import 'package:hive/hive.dart'; import 'package:path_provider/path_provider.dart'; +import 'package:sembast/sembast.dart'; +import 'package:sembast_sqflite/sembast_sqflite.dart'; import 'package:steel_crypt/steel_crypt.dart'; import 'package:syphon/global/values.dart'; +import 'package:sqflite/sqflite.dart' as sqflite; +import 'package:syphon/store/auth/state.dart'; class CacheSecure { // encryption references (in memory only) @@ -18,6 +23,11 @@ class CacheSecure { static Box cacheRooms; static Box cacheCrypto; + // cache database references + static Database cacheMainSql; + static Database cacheRoomSql; + static Database cacheCryptoSql; + // cache storage identifiers static const cacheKeyMain = '${Values.appNameLabel}-main-cache'; static const cacheKeyRooms = '${Values.appNameLabel}-room-cache'; @@ -41,6 +51,27 @@ Future initCache() async { // Init storage location final String storageLocation = await initStorageLocation(); + /// Supports iOS/Android/MacOS for now. + final factory = getDatabaseFactorySqflite(sqflite.databaseFactory); + + // Open sqlflit + final sqlite = await factory.openDatabase('${CacheSecure.cacheKeyMain}.db'); + + // Define the store, key is a string, value is a string + final store = StoreRef.main(); + + // Define the record + final record = store.record(AuthStore().runtimeType.toString()); + + // Write a record + await record.put(sqlite, json.encode(AuthStore())); + + // print store content + print(await store.stream(sqlite).first); + + // Close the database + await sqlite.close(); + // Init configuration Hive.init(storageLocation); diff --git a/pubspec.lock b/pubspec.lock index d6f117ae1..a2fdb7edb 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -730,6 +730,20 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.3.0" + sembast: + dependency: "direct main" + description: + name: sembast + url: "https://pub.dartlang.org" + source: hosted + version: "2.4.8" + sembast_sqflite: + dependency: "direct main" + description: + name: sembast_sqflite + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.0" shared_preferences: dependency: transitive description: @@ -833,6 +847,20 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.7.0" + sqflite: + dependency: "direct main" + description: + name: sqflite + url: "https://pub.dartlang.org" + source: hosted + version: "1.3.2+1" + sqflite_common: + dependency: transitive + description: + name: sqflite_common + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.2+1" stack_trace: dependency: transitive description: @@ -1044,5 +1072,5 @@ packages: source: hosted version: "2.2.1" sdks: - dart: ">=2.10.0-110 <2.11.0" - flutter: ">=1.18.0-6.0.pre <2.0.0" + dart: ">=2.10.2 <2.11.0" + flutter: ">=1.22.2 <2.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index 108cde202..c5d727ab7 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -86,7 +86,9 @@ dependencies: flutter_secure_storage: 3.3.3 json_annotation: ^3.1.0 checked_yaml: 1.0.2 - # sembast: 2.4.7+7 + sqflite: ^1.3.2 + sembast: ^2.4.8 + sembast_sqflite: ^1.0.0 # isolate_handler: 0.3.1 # flutter_isolate: 1.0.0+14 From 04788305dd1ef4adabae6de65d9206eca7287893 Mon Sep 17 00:00:00 2001 From: ereio Date: Mon, 23 Nov 2020 23:20:29 -0500 Subject: [PATCH 04/25] attemting to cache to sembast --- .../kotlin/org/tether/tether/Application.kt | 16 +++++ lib/global/cache/index.dart | 29 +++++--- lib/global/cache/serializer.dart | 40 ++++++++--- lib/global/cache/threadables.dart | 38 ++++++++++ lib/store/auth/state.dart | 7 +- lib/store/rooms/state.dart | 7 -- lib/store/sync/state.dart | 69 ++++++++----------- lib/views/signup/index.dart | 1 - 8 files changed, 138 insertions(+), 69 deletions(-) create mode 100644 android/app/src/main/kotlin/org/tether/tether/Application.kt diff --git a/android/app/src/main/kotlin/org/tether/tether/Application.kt b/android/app/src/main/kotlin/org/tether/tether/Application.kt new file mode 100644 index 000000000..3c6a1d953 --- /dev/null +++ b/android/app/src/main/kotlin/org/tether/tether/Application.kt @@ -0,0 +1,16 @@ +import io.flutter.app.FlutterApplication +import io.flutter.plugin.common.PluginRegistry +import io.flutter.plugin.common.PluginRegistry.PluginRegistrantCallback +import io.flutter.plugins.firebasemessaging.FirebaseMessagingPlugin +import io.flutter.plugins.firebasemessaging.FlutterFirebaseMessagingService + +class Application : FlutterApplication(), PluginRegistrantCallback { + override fun onCreate() { + super.onCreate() + FlutterFirebaseMessagingService.setPluginRegistrant(this) + } + + override fun registerWith(registry: PluginRegistry?) { + SqflitePlugin.registerWith(registry.registrarFor("com.tekartik.sqflite.SqflitePlugin")); + } +} \ No newline at end of file diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index bdd7868fc..e35e06172 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -11,6 +11,7 @@ import 'package:steel_crypt/steel_crypt.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite/sqflite.dart' as sqflite; import 'package:syphon/store/auth/state.dart'; +import 'package:syphon/store/user/model.dart'; class CacheSecure { // encryption references (in memory only) @@ -54,23 +55,27 @@ Future initCache() async { /// Supports iOS/Android/MacOS for now. final factory = getDatabaseFactorySqflite(sqflite.databaseFactory); - // Open sqlflit - final sqlite = await factory.openDatabase('${CacheSecure.cacheKeyMain}.db'); + // Example + // final example = AuthStore(user: User(userId: 'testing123')); // Define the store, key is a string, value is a string - final store = StoreRef.main(); - - // Define the record - final record = store.record(AuthStore().runtimeType.toString()); + // final store = StoreRef.main(); - // Write a record - await record.put(sqlite, json.encode(AuthStore())); + // Define and write a record + // final record = await store + // .record(example.runtimeType.toString()) + // .put(sqlite, json.encode(example)); // print store content - print(await store.stream(sqlite).first); + // print('[CacheSecure] database read'); + // print(await store.stream(sqlite).first); // Close the database - await sqlite.close(); + // await sqlite.close(); + + // Open sqlflit + CacheSecure.cacheMainSql = + await factory.openDatabase('${CacheSecure.cacheKeyMain}.db'); // Init configuration Hive.init(storageLocation); @@ -126,6 +131,10 @@ void closeCache() async { if (CacheSecure.cacheCrypto != null && CacheSecure.cacheCrypto.isOpen) { CacheSecure.cacheCrypto.close(); } + + if (CacheSecure.cacheMainSql != null) { + CacheSecure.cacheMainSql.close(); + } } String createIVKey() { diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index 6bc4ee6ad..d4a887062 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -8,6 +8,7 @@ import 'package:flutter/material.dart'; // Package imports: import 'package:redux_persist/redux_persist.dart'; +import 'package:sembast/sembast.dart'; import 'package:steel_crypt/steel_crypt.dart'; import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/cache/threadables.dart'; @@ -30,7 +31,7 @@ import 'package:syphon/store/settings/state.dart'; class CacheSerializer implements StateSerializer { @override Uint8List encode(AppState state) { - final stores = [ + final List stores = [ state.authStore, state.syncStore, state.cryptoStore, @@ -45,6 +46,7 @@ class CacheSerializer implements StateSerializer { Future.microtask(() async { // // create a new IV for the encrypted cache CacheSecure.ivKey = createIVKey(); + // // backup the IV in case the app is force closed before caching finishes await saveIVKeyNext(CacheSecure.ivKey); @@ -54,16 +56,34 @@ class CacheSerializer implements StateSerializer { var jsonEncoded; var jsonEncrypted; - // encode the store contents to json - // HACK: unable to pass both listed stores direct to an isolate - // final sensitiveStorage = [AuthStore, SyncStore, CryptoStore]; - // if (!sensitiveStorage.contains(store.runtimeType)) { - // jsonEncoded = await compute(jsonEncode, store); - // } else { - jsonEncoded = json.encode(store); - // } + // serialize the store contents + try { + // HACK: unable to pass both listed stores direct to an isolate + final sensitiveStorage = [AuthStore, SyncStore, CryptoStore]; + if (!sensitiveStorage.contains(store.runtimeType)) { + jsonEncoded = await compute(jsonEncode, store); + } else { + jsonEncoded = json.encode(store); + } + } catch (error) { + jsonEncoded = json.encode(store); + print( + '[Cache] Background Encoding Failed ${store.runtimeType.toString()} $error', + ); + } + + // encrypt the store contents + try { + final cache = CacheSecure.cacheMainSql; + final storeRef = StoreRef.main(); + final jsonEncrypted = json.encode(store); + await storeRef + .record(store.runtimeType.toString()) + .put(cache, jsonEncrypted); + } catch (error) { + print('[Cache] ERROR $error'); + } - // encrypt the store contents previously converted to json jsonEncrypted = await compute(encryptJsonBackground, { 'ivKey': CacheSecure.ivKey, 'cryptKey': CacheSecure.cryptKey, diff --git a/lib/global/cache/threadables.dart b/lib/global/cache/threadables.dart index 3600623bd..98baf18e1 100644 --- a/lib/global/cache/threadables.dart +++ b/lib/global/cache/threadables.dart @@ -4,8 +4,11 @@ import 'dart:ui'; import 'dart:async'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; +import 'package:sqflite/sqflite.dart' as sqflite; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; +import 'package:sembast/sembast.dart'; +import 'package:sembast_sqflite/sembast_sqflite.dart'; import 'package:steel_crypt/steel_crypt.dart'; import 'package:syphon/global/cache/index.dart'; @@ -56,3 +59,38 @@ Future serializeJsonBackground(Object store) async { return null; } } + +// responsibile for both json serialization and encryption +Future encryptSerializeObjectBackground(Object store) async { + WidgetsFlutterBinding.ensureInitialized(); + window.onPlatformMessage = + ServicesBinding.instance.defaultBinaryMessenger.handlePlatformMessage; + + try { + final storageEngine = FlutterSecureStorage(); + + final ivKey = await storageEngine.read(key: CacheSecure.ivKeyLocation); + final cryptKey = + await storageEngine.read(key: CacheSecure.cryptKeyLocation); + + final cryptor = AesCrypt(key: cryptKey, padding: PaddingAES.pkcs7); + + final jsonEncrypted = cryptor.ctr.encrypt(inp: store, iv: ivKey); + + /// Supports iOS/Android/MacOS for now. + final factory = getDatabaseFactorySqflite(sqflite.databaseFactory); + + // Open sqlflite + final cacheSql = + await factory.openDatabase('${CacheSecure.cacheKeyMain}.db'); + + // Define and write a record + final storeRef = StoreRef.main(); + await storeRef.record(store.runtimeType.toString()).put(cacheSql, store); + + // + } catch (error) { + debugPrint('[serializeJsonBackground] $error'); + return null; + } +} diff --git a/lib/store/auth/state.dart b/lib/store/auth/state.dart index fb4020a67..178bb7b4b 100644 --- a/lib/store/auth/state.dart +++ b/lib/store/auth/state.dart @@ -12,14 +12,17 @@ import 'package:syphon/store/user/model.dart'; part 'state.g.dart'; -@JsonSerializable(ignoreUnannotated: true) +@JsonSerializable() class AuthStore extends Equatable { - @JsonKey(name: 'user') final User user; + @JsonKey(ignore: true) User get currentUser => user; + @JsonKey(ignore: true) final StreamController authObserver; + + @JsonKey(ignore: true) Stream get onAuthStateChanged => authObserver != null ? authObserver.stream : null; diff --git a/lib/store/rooms/state.dart b/lib/store/rooms/state.dart index aaaf23ce8..252210769 100644 --- a/lib/store/rooms/state.dart +++ b/lib/store/rooms/state.dart @@ -24,9 +24,6 @@ class RoomStore extends Equatable { @JsonKey(ignore: true) final List roomsHidden; - @JsonKey(ignore: true) - final Timer roomObserver; - @JsonKey(ignore: true) final bool loading; @@ -40,7 +37,6 @@ class RoomStore extends Equatable { this.loading = false, this.lastUpdate = 0, this.lastSince, - this.roomObserver, this.roomsHidden = const [], }); @@ -51,7 +47,6 @@ class RoomStore extends Equatable { archive, lastUpdate, lastSince, - roomObserver, roomsHidden, ]; @@ -62,7 +57,6 @@ class RoomStore extends Equatable { loading, lastUpdate, lastSince, - roomObserver, roomsHidden, }) => RoomStore( @@ -72,7 +66,6 @@ class RoomStore extends Equatable { loading: loading ?? this.loading, lastUpdate: lastUpdate ?? this.lastUpdate, lastSince: lastSince ?? this.lastSince, - roomObserver: roomObserver ?? this.roomObserver, roomsHidden: roomsHidden ?? this.roomsHidden, ); diff --git a/lib/store/sync/state.dart b/lib/store/sync/state.dart index fb6e95b45..db01e3d7f 100644 --- a/lib/store/sync/state.dart +++ b/lib/store/sync/state.dart @@ -7,35 +7,30 @@ import 'package:json_annotation/json_annotation.dart'; part 'state.g.dart'; -@JsonSerializable(ignoreUnannotated: true) +@JsonSerializable() class SyncStore extends Equatable { - @JsonKey(name: 'synced') final bool synced; + final bool offline; + final bool backgrounded; - @JsonKey(name: 'lastUpdate') final int lastUpdate; // Last timestamp for actual new info - - @JsonKey(name: 'lastSince') + final int lastAttempt; // last attempt to sync final String lastSince; // Since we last checked for new info - static const default_interval = 1; - - @JsonKey(name: 'interval') - final int interval = default_interval; - - @JsonKey(name: 'offline') - final bool offline; + @JsonKey(ignore: true) + final int interval; + @JsonKey(ignore: true) final int backoff; + + @JsonKey(ignore: true) final bool syncing; - final bool unauthed; - final Timer syncObserver; - @JsonKey(name: 'lastAttempt') - final int lastAttempt; // last attempt to sync + @JsonKey(ignore: true) + final bool unauthed; - @JsonKey(name: 'backgrounded') - final bool backgrounded; + @JsonKey(ignore: true) + final Timer syncObserver; const SyncStore({ this.synced = false, @@ -46,6 +41,7 @@ class SyncStore extends Equatable { this.lastUpdate = 0, this.lastAttempt = 0, this.backoff = 0, + this.interval = 2, // default_interval this.lastSince, this.syncObserver, }); @@ -72,27 +68,22 @@ class SyncStore extends Equatable { bool unauthed, bool backgrounded, int lastUpdate, - lastAttempt, - syncObserver, - lastSince, - }) { - return SyncStore( - synced: synced ?? this.synced, - syncing: syncing ?? this.syncing, - offline: offline ?? this.offline, - unauthed: unauthed ?? this.unauthed, - lastUpdate: lastUpdate ?? this.lastUpdate, - lastAttempt: lastAttempt ?? - this.lastAttempt ?? - 0, // TODO: remove after version 0.1.4 - lastSince: lastSince ?? this.lastSince, - syncObserver: syncObserver ?? this.syncObserver, - backgrounded: backgrounded ?? - this.backgrounded ?? - false, // TODO: remove after version 0.1.4 - backoff: backoff ?? this.backoff, - ); - } + int lastAttempt, + Timer syncObserver, + String lastSince, + }) => + SyncStore( + synced: synced ?? this.synced, + syncing: syncing ?? this.syncing, + offline: offline ?? this.offline, + unauthed: unauthed ?? this.unauthed, + lastUpdate: lastUpdate ?? this.lastUpdate, + lastAttempt: lastAttempt ?? this.lastAttempt, + lastSince: lastSince ?? this.lastSince, + syncObserver: syncObserver ?? this.syncObserver, + backgrounded: backgrounded ?? this.backgrounded, + backoff: backoff ?? this.backoff, + ); Map toJson() => _$SyncStoreToJson(this); diff --git a/lib/views/signup/index.dart b/lib/views/signup/index.dart index f8f94ac9c..0ef988b73 100644 --- a/lib/views/signup/index.dart +++ b/lib/views/signup/index.dart @@ -29,7 +29,6 @@ import './step-password.dart'; import './step-username.dart'; // Styling Widgets - final Duration nextAnimationDuration = Duration( milliseconds: Values.animationDurationDefault, ); From 4319a24fcb3b1ff61f649e4933405ef61b38695c Mon Sep 17 00:00:00 2001 From: ereio Date: Wed, 2 Dec 2020 20:15:17 -0500 Subject: [PATCH 05/25] sembast (without sql) working as a cache replacement --- .../kotlin/org/tether/tether/Application.kt | 16 -- assets/cheatsheet.md | 42 +++- lib/global/cache/index.dart | 98 ++++------ lib/global/cache/serializer.dart | 183 ++++++------------ lib/global/cache/storage.dart | 85 ++++++++ lib/global/cache/threadables.dart | 85 ++++---- lib/store/index.dart | 8 +- lib/store/rooms/actions.dart | 3 +- lib/store/rooms/room/model.dart | 2 + lib/store/user/parser.dart | 16 ++ pubspec.lock | 14 ++ pubspec.yaml | 3 +- 12 files changed, 298 insertions(+), 257 deletions(-) delete mode 100644 android/app/src/main/kotlin/org/tether/tether/Application.kt create mode 100644 lib/global/cache/storage.dart create mode 100644 lib/store/user/parser.dart diff --git a/android/app/src/main/kotlin/org/tether/tether/Application.kt b/android/app/src/main/kotlin/org/tether/tether/Application.kt deleted file mode 100644 index 3c6a1d953..000000000 --- a/android/app/src/main/kotlin/org/tether/tether/Application.kt +++ /dev/null @@ -1,16 +0,0 @@ -import io.flutter.app.FlutterApplication -import io.flutter.plugin.common.PluginRegistry -import io.flutter.plugin.common.PluginRegistry.PluginRegistrantCallback -import io.flutter.plugins.firebasemessaging.FirebaseMessagingPlugin -import io.flutter.plugins.firebasemessaging.FlutterFirebaseMessagingService - -class Application : FlutterApplication(), PluginRegistrantCallback { - override fun onCreate() { - super.onCreate() - FlutterFirebaseMessagingService.setPluginRegistrant(this) - } - - override fun registerWith(registry: PluginRegistry?) { - SqflitePlugin.registerWith(registry.registrarFor("com.tekartik.sqflite.SqflitePlugin")); - } -} \ No newline at end of file diff --git a/assets/cheatsheet.md b/assets/cheatsheet.md index 9f299ad15..61b976afe 100644 --- a/assets/cheatsheet.md +++ b/assets/cheatsheet.md @@ -96,4 +96,44 @@ if (true) { } } */ - ``` \ No newline at end of file + ``` + + + ```dart +/* + Opening storage path on mobile devices (main thread only) +*/ + Future initStorageLocation() async { + var storageLocation; + + try { + if (Platform.isIOS || Platform.isAndroid) { + storageLocation = await getApplicationDocumentsDirectory(); + return storageLocation.path; + } + + if (Platform.isMacOS) { + storageLocation = await File('cache').create().then( + (value) => value.writeAsString( + '{}', + flush: true, + ), + ); + + return storageLocation.path; + } + + if (Platform.isLinux) { + storageLocation = await getApplicationDocumentsDirectory(); + return storageLocation.path; + } + + debugPrint('[initStorageLocation] no cache support'); + return null; + } catch (error) { + debugPrint('[initStorageLocation] $error'); + return null; + } +} + +``` \ No newline at end of file diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index e35e06172..bc34b58d1 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -1,17 +1,19 @@ -import 'dart:convert'; import 'dart:io'; +import 'package:path/path.dart'; +import 'package:path_provider/path_provider.dart'; + import 'package:flutter/material.dart'; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; import 'package:hive/hive.dart'; import 'package:path_provider/path_provider.dart'; import 'package:sembast/sembast.dart'; +import 'package:sembast/sembast_io.dart'; import 'package:sembast_sqflite/sembast_sqflite.dart'; import 'package:steel_crypt/steel_crypt.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite/sqflite.dart' as sqflite; -import 'package:syphon/store/auth/state.dart'; -import 'package:syphon/store/user/model.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; class CacheSecure { // encryption references (in memory only) @@ -26,11 +28,13 @@ class CacheSecure { // cache database references static Database cacheMainSql; - static Database cacheRoomSql; - static Database cacheCryptoSql; + + // inital store caches for reload + static Map cacheStoreDecoded = {}; // cache storage identifiers static const cacheKeyMain = '${Values.appNameLabel}-main-cache'; + static const cacheKeyMainAlt = '${Values.appNameLabel}-main-cache-alt'; static const cacheKeyRooms = '${Values.appNameLabel}-room-cache'; static const cacheKeyCrypto = '${Values.appNameLabel}-crypto-cache'; @@ -49,73 +53,41 @@ class CacheSecure { } Future initCache() async { - // Init storage location - final String storageLocation = await initStorageLocation(); - - /// Supports iOS/Android/MacOS for now. - final factory = getDatabaseFactorySqflite(sqflite.databaseFactory); - - // Example - // final example = AuthStore(user: User(userId: 'testing123')); - - // Define the store, key is a string, value is a string - // final store = StoreRef.main(); - - // Define and write a record - // final record = await store - // .record(example.runtimeType.toString()) - // .put(sqlite, json.encode(example)); - - // print store content - // print('[CacheSecure] database read'); - // print(await store.stream(sqlite).first); - - // Close the database - // await sqlite.close(); - - // Open sqlflit - CacheSecure.cacheMainSql = - await factory.openDatabase('${CacheSecure.cacheKeyMain}.db'); - - // Init configuration - Hive.init(storageLocation); - - CacheSecure.cacheMain = await unlockMainCache(); - CacheSecure.cacheRooms = await unlockRoomCache(); - CacheSecure.cacheCrypto = await unlockCryptoCache(); -} - -Future initStorageLocation() async { - var storageLocation; - try { - if (Platform.isIOS || Platform.isAndroid) { - storageLocation = await getApplicationDocumentsDirectory(); - return storageLocation.path; + var factory; + var databasePath = '${CacheSecure.cacheKeyMainAlt}.db'; + + if (Platform.isAndroid || Platform.isIOS) { + var directory = await getApplicationDocumentsDirectory(); + await directory.create(recursive: true); + databasePath = join(directory.path, '${CacheSecure.cacheKeyMain}.db'); + factory = databaseFactoryIo; + // factory = getDatabaseFactorySqflite(sqflite.databaseFactory); } - if (Platform.isMacOS) { - storageLocation = await File('cache').create().then( - (value) => value.writeAsString( - '{}', - flush: true, - ), - ); - - return storageLocation.path; + /// Supports Windows/Linux/MacOS for now. + if (Platform.isLinux || Platform.isWindows || Platform.isMacOS) { + factory = getDatabaseFactorySqflite(sqflite_ffi.databaseFactoryFfi); } - if (Platform.isLinux) { - storageLocation = await getApplicationDocumentsDirectory(); - return storageLocation.path; + if (factory == null) { + throw UnsupportedError( + 'Sorry, Syphon does not support your platform yet. Hope to do so soon!', + ); } - debugPrint('[initStorageLocation] no cache support'); - return null; + // open sqlflite + CacheSecure.cacheMainSql = await factory.openDatabase( + databasePath, + ); } catch (error) { - debugPrint('[initStorageLocation] $error'); - return null; + debugPrint('[initCache] ${error}'); } + + // Configure cache encryption/decryption instance + CacheSecure.ivKey = await unlockIVKey(); + CacheSecure.ivKeyNext = await unlockIVKeyNext(); + CacheSecure.cryptKey = await unlockCryptKey(); } // // Closes and saves storage diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index d4a887062..eddabab74 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -9,7 +9,6 @@ import 'package:flutter/material.dart'; // Package imports: import 'package:redux_persist/redux_persist.dart'; import 'package:sembast/sembast.dart'; -import 'package:steel_crypt/steel_crypt.dart'; import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/cache/threadables.dart'; @@ -53,12 +52,14 @@ class CacheSerializer implements StateSerializer { // run through all redux stores for encryption and encoding await Future.wait(stores.map((store) async { try { - var jsonEncoded; - var jsonEncrypted; + String jsonEncoded; + String jsonEncrypted; + String type = store.runtimeType.toString(); // serialize the store contents + Stopwatch stopwatchSerialize = new Stopwatch()..start(); try { - // HACK: unable to pass both listed stores direct to an isolate + // HACK: unable to pass certain stores directly to an isolate final sensitiveStorage = [AuthStore, SyncStore, CryptoStore]; if (!sensitiveStorage.contains(store.runtimeType)) { jsonEncoded = await compute(jsonEncode, store); @@ -68,60 +69,54 @@ class CacheSerializer implements StateSerializer { } catch (error) { jsonEncoded = json.encode(store); print( - '[Cache] Background Encoding Failed ${store.runtimeType.toString()} $error', + '[CacheSerializer] ${type} failed $error', ); } + debugPrint( + '[CacheSerializer] ${stopwatchSerialize.elapsed} ${type} serialize', + ); + + Stopwatch stopwatchEncrypt = new Stopwatch()..start(); // encrypt the store contents + jsonEncrypted = await compute( + encryptJsonBackground, + { + 'ivKey': CacheSecure.ivKey, + 'cryptKey': CacheSecure.cryptKey, + 'type': type, + 'json': jsonEncoded, + }, + debugLabel: 'encryptJsonBackground', + ); + + debugPrint( + '[CacheSerializer] ${stopwatchEncrypt.elapsed} ${type} encrypt', + ); + try { + Stopwatch stopwatchSave = new Stopwatch()..start(); final cache = CacheSecure.cacheMainSql; final storeRef = StoreRef.main(); - final jsonEncrypted = json.encode(store); - await storeRef - .record(store.runtimeType.toString()) - .put(cache, jsonEncrypted); - } catch (error) { - print('[Cache] ERROR $error'); - } + await storeRef.record(type).put(cache, jsonEncrypted); - jsonEncrypted = await compute(encryptJsonBackground, { - 'ivKey': CacheSecure.ivKey, - 'cryptKey': CacheSecure.cryptKey, - 'type': store.runtimeType.toString(), - 'json': jsonEncoded, - }); - - // cache redux store to main cache storage - // caching room and crypto stores with additional hive level error handling - switch (store.runtimeType) { - case RoomStore: - await CacheSecure.cacheRooms.put( - store.runtimeType.toString(), - jsonEncrypted, - ); - break; - case CryptoStore: - await CacheSecure.cacheCrypto.put( - store.runtimeType.toString(), - jsonEncrypted, - ); - break; - default: - await CacheSecure.cacheMain.put( - store.runtimeType.toString(), - jsonEncrypted, - ); - break; + debugPrint( + '[CacheSerializer] ${stopwatchSave.elapsed} ${type} saved', + ); + } catch (error) { + print('[CacheSerializer] ERROR $error'); } } catch (error) { debugPrint( - '[CacheSerializer.encode] $error', + '[CacheSerializer] $error', ); } })); // Rotate encryption for the next save await saveIVKey(CacheSecure.ivKey); + + return Future.value(null); }); // Disregard redux persist storage saving @@ -129,8 +124,6 @@ class CacheSerializer implements StateSerializer { } AppState decode(Uint8List data) { - final aes = AesCrypt(key: CacheSecure.cryptKey, padding: PaddingAES.pkcs7); - AuthStore authStore = AuthStore(); SyncStore syncStore = SyncStore(); CryptoStore cryptoStore = CryptoStore(); @@ -139,100 +132,40 @@ class CacheSerializer implements StateSerializer { SettingsStore settingsStore = SettingsStore(); UserStore userStore = UserStore(); - final List stores = [ - authStore, - syncStore, - mediaStore, - roomStore, - cryptoStore, - settingsStore, - userStore, - ]; + // final aes = AesCrypt(key: CacheSecure.cryptKey, padding: PaddingAES.pkcs7); + + // Load stores previously fetched from cache, + // mutable global due to redux_presist not extendable beyond Uint8List + final stores = CacheSecure.cacheStoreDecoded; // decode each store cache synchronously - stores.forEach((store) { + stores.forEach((type, store) { try { - Map decodedJson = {}; - - var encryptedJson; - - // fetch from main cache storage - // fetching room and crypto store has additional hive level error handling - switch (store.runtimeType) { - case RoomStore: - encryptedJson = CacheSecure.cacheRooms.get( - store.runtimeType.toString(), - defaultValue: null, - ); - break; - case CryptoStore: - encryptedJson = CacheSecure.cacheCrypto.get( - store.runtimeType.toString(), - defaultValue: null, - ); - break; - default: - encryptedJson = CacheSecure.cacheMain.get( - store.runtimeType.toString(), - defaultValue: null, - ); - break; - } - - // attempt to decrypt encrypted state after loaded from RAM - if (encryptedJson != null) { - try { - final decryptedJson = aes.ctr.decrypt( - enc: encryptedJson, - iv: CacheSecure.ivKey, - ); - decodedJson = json.decode(decryptedJson); - } catch (error) { - debugPrint('[CacheSerializer.decode] $error'); - decodedJson = {}; - } - } - - // decryption may fail if force closed, attempt with new iv generated before close - if (decodedJson.isEmpty) { - try { - // decrypt encrypted state after loaded from RAM - final decryptedJson = aes.ctr.decrypt( - enc: encryptedJson, - iv: CacheSecure.ivKeyNext, - ); - decodedJson = json.decode(decryptedJson); - } catch (error) { - debugPrint('[CacheSerializer.decode] $error'); - decodedJson = {}; - } - } - // if all else fails, just pass back a fresh store to avoid a crash - if (decodedJson.isEmpty) return; + if (store == null || store.isEmpty) return; // this stinks, but dart doesn't allow reflection for factories/contructors - switch (store.runtimeType) { - case AuthStore: - authStore = AuthStore.fromJson(decodedJson); + switch (type) { + case 'AuthStore': + authStore = AuthStore.fromJson(store); break; - case SyncStore: - syncStore = SyncStore.fromJson(decodedJson); + case 'SyncStore': + syncStore = SyncStore.fromJson(store); break; - case CryptoStore: - cryptoStore = CryptoStore.fromJson(decodedJson); + case 'CryptoStore': + cryptoStore = CryptoStore.fromJson(store); break; - case MediaStore: - mediaStore = MediaStore.fromJson(decodedJson); + case 'MediaStore': + mediaStore = MediaStore.fromJson(store); break; - case RoomStore: - roomStore = RoomStore.fromJson(decodedJson); + case 'RoomStore': + roomStore = RoomStore.fromJson(store); break; - case SettingsStore: - settingsStore = SettingsStore.fromJson(decodedJson); + case 'SettingsStore': + settingsStore = SettingsStore.fromJson(store); break; - case UserStore: - userStore = UserStore.fromJson(decodedJson); + case 'UserStore': + userStore = UserStore.fromJson(store); break; default: break; diff --git a/lib/global/cache/storage.dart b/lib/global/cache/storage.dart new file mode 100644 index 000000000..377be585c --- /dev/null +++ b/lib/global/cache/storage.dart @@ -0,0 +1,85 @@ +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:redux_persist/redux_persist.dart'; +import 'package:sembast/sembast.dart'; +import 'package:syphon/global/cache/index.dart'; +import 'package:syphon/global/cache/threadables.dart'; +import 'package:syphon/global/print.dart'; +import 'package:syphon/store/auth/state.dart'; +import 'package:syphon/store/crypto/state.dart'; +import 'package:syphon/store/media/state.dart'; +import 'package:syphon/store/rooms/state.dart'; +import 'package:syphon/store/settings/state.dart'; +import 'package:syphon/store/sync/state.dart'; +import 'package:syphon/store/user/state.dart'; + +final List stores = [ + AuthStore(), + SyncStore(), + MediaStore(), + RoomStore(), + CryptoStore(), + SettingsStore(), + UserStore(), +]; + +class CacheStorage implements StorageEngine { + @override + Future load() async { + try { + Stopwatch stopwatchTotal = new Stopwatch()..start(); + + final cache = CacheSecure.cacheMainSql; + + await Future.wait(stores.map((store) async { + Stopwatch stopwatchStore = new Stopwatch()..start(); + // Fetch from database + final type = store.runtimeType.toString(); + final table = StoreRef.main(); + final record = table.record(store.runtimeType.toString()); + final jsonEncrypted = await record.get(cache); + + debugPrint('[CacheStorage] load ${stopwatchStore.elapsed}'); + + // Decrypt from database + final jsonDecoded = await compute( + decryptJsonBackground, + { + 'ivKey': CacheSecure.ivKey, + 'ivKeyNext': CacheSecure.ivKeyNext, + 'cryptKey': CacheSecure.cryptKey, + 'type': type, + 'json': jsonEncrypted, + }, + debugLabel: 'decryptJsonBackground', + ); + + debugPrint('[CacheStorage] decrypt ${stopwatchStore.elapsed}'); + + // Load for CacheSerializer to use later + CacheSecure.cacheStoreDecoded[type] = jsonDecoded; + })); + + debugPrint('[CacheStorage] load total time ${stopwatchTotal.elapsed} '); + } catch (error) { + printError(error, title: 'CacheStorage.load'); + } + + // unlock redux_persist after cache loaded from sqflite + return null; + } + + @override + Future save(Uint8List data) { + return null; + } + + static Future saveOffload(String jsonEncrypted, {String type}) async { + final cache = CacheSecure.cacheMainSql; + final table = StoreRef.main(); + final record = table.record(type); + await record.put(cache, jsonEncrypted); + } +} diff --git a/lib/global/cache/threadables.dart b/lib/global/cache/threadables.dart index 98baf18e1..3b3469cab 100644 --- a/lib/global/cache/threadables.dart +++ b/lib/global/cache/threadables.dart @@ -4,11 +4,8 @@ import 'dart:ui'; import 'dart:async'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; -import 'package:sqflite/sqflite.dart' as sqflite; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; -import 'package:sembast/sembast.dart'; -import 'package:sembast_sqflite/sembast_sqflite.dart'; import 'package:steel_crypt/steel_crypt.dart'; import 'package:syphon/global/cache/index.dart'; @@ -22,15 +19,52 @@ Future encryptJsonBackground(Map params) async { return cryptor.ctr.encrypt(inp: json, iv: ivKey); } -// TODO: deserialization is required synchronous by redux_persist :/ -Future decryptJsonBackground(Map params) async { +Future decryptJsonBackground(Map params) async { String ivKey = params['ivKey']; + String ivKeyNext = params['ivKeyNext']; + String type = params['type']; String cryptKey = params['cryptKey']; - String json = params['json']; + String jsonEncrypted = params['json']; - final cryptor = AesCrypt(key: cryptKey, padding: PaddingAES.pkcs7); + String jsonDecrypted; + Map jsonDecoded = {}; + + final aes = AesCrypt(key: cryptKey, padding: PaddingAES.pkcs7); + + if (jsonEncrypted == null) return null; + + try { + jsonDecrypted = aes.ctr.decrypt( + enc: jsonEncrypted, + iv: ivKey, + ); + } catch (error) { + debugPrint('[decryptJsonBackground] error $error'); + } + + if (jsonDecoded.isEmpty) { + try { + jsonDecrypted = aes.ctr.decrypt( + enc: jsonEncrypted, + iv: ivKeyNext, + ); + } catch (error) { + debugPrint('[decryptJsonBackground] error $error'); + jsonDecoded = {}; + } + } + + // Failed to decrypt data + if (jsonDecrypted == null) { + debugPrint('[decryptJsonBackground] decryption failed ${type}'); + return null; + } + + // decode serialized object + jsonDecoded = json.decode(jsonDecrypted); - return cryptor.ctr.decrypt(enc: json, iv: ivKey); + debugPrint('[decryptJsonBackground] decryption succeed ${type}'); + return jsonDecoded; } // TODO: needs plugins that work in isolates, still having @@ -59,38 +93,3 @@ Future serializeJsonBackground(Object store) async { return null; } } - -// responsibile for both json serialization and encryption -Future encryptSerializeObjectBackground(Object store) async { - WidgetsFlutterBinding.ensureInitialized(); - window.onPlatformMessage = - ServicesBinding.instance.defaultBinaryMessenger.handlePlatformMessage; - - try { - final storageEngine = FlutterSecureStorage(); - - final ivKey = await storageEngine.read(key: CacheSecure.ivKeyLocation); - final cryptKey = - await storageEngine.read(key: CacheSecure.cryptKeyLocation); - - final cryptor = AesCrypt(key: cryptKey, padding: PaddingAES.pkcs7); - - final jsonEncrypted = cryptor.ctr.encrypt(inp: store, iv: ivKey); - - /// Supports iOS/Android/MacOS for now. - final factory = getDatabaseFactorySqflite(sqflite.databaseFactory); - - // Open sqlflite - final cacheSql = - await factory.openDatabase('${CacheSecure.cacheKeyMain}.db'); - - // Define and write a record - final storeRef = StoreRef.main(); - await storeRef.record(store.runtimeType.toString()).put(cacheSql, store); - - // - } catch (error) { - debugPrint('[serializeJsonBackground] $error'); - return null; - } -} diff --git a/lib/store/index.dart b/lib/store/index.dart index 384380812..d10187a3d 100644 --- a/lib/store/index.dart +++ b/lib/store/index.dart @@ -8,6 +8,7 @@ import 'package:redux/redux.dart'; import 'package:redux_persist/redux_persist.dart'; import 'package:redux_thunk/redux_thunk.dart'; import 'package:syphon/global/cache/index.dart'; +import 'package:syphon/global/cache/storage.dart'; // Project imports: import 'package:syphon/store/alerts/model.dart'; @@ -100,7 +101,7 @@ AppState appReducer(AppState state, action) => AppState( Future initStore() async { // Configure redux persist instance final persistor = Persistor( - storage: MemoryStorage(), + storage: CacheStorage(), serializer: CacheSerializer(), throttleDuration: Duration(milliseconds: 4500), shouldSave: (Store store, dynamic action) { @@ -125,11 +126,6 @@ Future initStore() async { }, ); - // Configure cache encryption/decryption instance - CacheSecure.ivKey = await unlockIVKey(); - CacheSecure.ivKeyNext = await unlockIVKeyNext(); - CacheSecure.cryptKey = await unlockCryptKey(); - // Finally load persisted store var initialState; diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index 90af3ef61..206cd82f5 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -417,10 +417,9 @@ ThunkAction createRoom({ value: (user) => user, ); + // generate user invite map to cache recent users room = room.copyWith(users: userInviteMap); - printJson(userInviteMap); - if (avatarFile != null) { await store.dispatch( updateRoomAvatar(roomId: room.id, localFile: avatarFile), diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index c11e74585..67c2d226c 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -55,7 +55,9 @@ class Room { final List messages; final List outbox; + @JsonKey(ignore: true) final Map users; + final Map messageReads; @JsonKey(ignore: true) diff --git a/lib/store/user/parser.dart b/lib/store/user/parser.dart new file mode 100644 index 000000000..d069ef392 --- /dev/null +++ b/lib/store/user/parser.dart @@ -0,0 +1,16 @@ +/**/ +Map parseUsers(AppState state) { + final rooms = state.roomStore.rooms.values as Iterable; + final roomsDirect = rooms.where((room) => room.direct); + final roomsDirectUsers = roomsDirect.map((room) => room.users); + + final allDirectUsers = roomsDirectUsers.fold( + {}, + (usersAll, users) { + (usersAll as Map).addAll(users); + return usersAll; + }, + ); + + return List.from(allDirectUsers.values); +} diff --git a/pubspec.lock b/pubspec.lock index a2fdb7edb..d3c5a6309 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -861,6 +861,20 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.0.2+1" + sqflite_common_ffi: + dependency: "direct main" + description: + name: sqflite_common_ffi + url: "https://pub.dartlang.org" + source: hosted + version: "1.1.1" + sqlite3: + dependency: transitive + description: + name: sqlite3 + url: "https://pub.dartlang.org" + source: hosted + version: "0.1.8" stack_trace: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index c5d727ab7..47aadaeb3 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -88,7 +88,8 @@ dependencies: checked_yaml: 1.0.2 sqflite: ^1.3.2 sembast: ^2.4.8 - sembast_sqflite: ^1.0.0 + sembast_sqflite: 1.0.0 + sqflite_common_ffi: 1.1.1 # isolate_handler: 0.3.1 # flutter_isolate: 1.0.0+14 From 3048d904b2225ea6cfa19c24434d5f4d365968ac Mon Sep 17 00:00:00 2001 From: ereio Date: Wed, 2 Dec 2020 22:40:33 -0500 Subject: [PATCH 06/25] seperated hot cache and cold storage --- lib/global/cache/index.dart | 107 +++++++------------------ lib/global/cache/serializer.dart | 11 ++- lib/global/cache/storage.dart | 12 +-- lib/global/print.dart | 23 ++++-- lib/global/storage/index.dart | 65 +++++++++++++++ lib/main.dart | 14 +++- lib/store/index.dart | 12 +-- lib/store/rooms/actions.dart | 12 ++- lib/store/rooms/events/parsers.dart | 37 +++++++++ lib/store/rooms/room/model.dart | 11 ++- lib/store/rooms/state.dart | 1 + lib/store/sync/background/service.dart | 8 +- lib/store/user/parser.dart | 16 ---- lib/store/user/parsers.dart | 8 ++ lib/store/user/storage.dart | 47 +++++++++++ 15 files changed, 255 insertions(+), 129 deletions(-) create mode 100644 lib/global/storage/index.dart create mode 100644 lib/store/rooms/events/parsers.dart delete mode 100644 lib/store/user/parser.dart create mode 100644 lib/store/user/parsers.dart create mode 100644 lib/store/user/storage.dart diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index bc34b58d1..f5614827b 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -5,14 +5,12 @@ import 'package:path_provider/path_provider.dart'; import 'package:flutter/material.dart'; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; -import 'package:hive/hive.dart'; -import 'package:path_provider/path_provider.dart'; import 'package:sembast/sembast.dart'; import 'package:sembast/sembast_io.dart'; import 'package:sembast_sqflite/sembast_sqflite.dart'; import 'package:steel_crypt/steel_crypt.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/global/values.dart'; -import 'package:sqflite/sqflite.dart' as sqflite; import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; class CacheSecure { @@ -21,22 +19,17 @@ class CacheSecure { static String ivKeyNext; static String cryptKey; - // cache refrences - static Box cacheMain; - static Box cacheRooms; - static Box cacheCrypto; + // hot cachee refrences + static Database cacheMain; - // cache database references - static Database cacheMainSql; + // cold storage references + static Database storageMain; // inital store caches for reload - static Map cacheStoreDecoded = {}; + static Map cacheStores = {}; // cache storage identifiers static const cacheKeyMain = '${Values.appNameLabel}-main-cache'; - static const cacheKeyMainAlt = '${Values.appNameLabel}-main-cache-alt'; - static const cacheKeyRooms = '${Values.appNameLabel}-room-cache'; - static const cacheKeyCrypto = '${Values.appNameLabel}-crypto-cache'; // cache key identifiers static const ivKeyLocation = '${Values.appNameLabel}@ivKey'; @@ -44,30 +37,38 @@ class CacheSecure { static const cryptKeyLocation = '${Values.appNameLabel}@cryptKey'; // background data identifiers - static const roomNamesKey = 'roomNamesKey'; + static const userIdKey = 'userId'; static const protocolKey = 'protocol'; + static const lastSinceKey = 'lastSince'; static const homeserverKey = 'homeserver'; + static const roomNamesKey = 'roomNamesKey'; static const accessTokenKey = 'accessToken'; - static const lastSinceKey = 'lastSince'; - static const userIdKey = 'userId'; } +/** + * Init Cache + * + * (needs cold storage extracted as it's own entity) + */ Future initCache() async { try { - var factory; - var databasePath = '${CacheSecure.cacheKeyMainAlt}.db'; + var cacheFactory; + + var cachePath = '${CacheSecure.cacheKeyMain}.db'; if (Platform.isAndroid || Platform.isIOS) { var directory = await getApplicationDocumentsDirectory(); - await directory.create(recursive: true); - databasePath = join(directory.path, '${CacheSecure.cacheKeyMain}.db'); - factory = databaseFactoryIo; - // factory = getDatabaseFactorySqflite(sqflite.databaseFactory); + await directory.create(); + cachePath = join(directory.path, '${CacheSecure.cacheKeyMain}.db'); + cacheFactory = databaseFactoryIo; } /// Supports Windows/Linux/MacOS for now. if (Platform.isLinux || Platform.isWindows || Platform.isMacOS) { - factory = getDatabaseFactorySqflite(sqflite_ffi.databaseFactoryFfi); + // open cache w/ sqflite ffi for desktop compat + cacheFactory = getDatabaseFactorySqflite( + sqflite_ffi.databaseFactoryFfi, + ); } if (factory == null) { @@ -76,10 +77,11 @@ Future initCache() async { ); } - // open sqlflite - CacheSecure.cacheMainSql = await factory.openDatabase( - databasePath, + CacheSecure.cacheMain = await cacheFactory.openDatabase( + cachePath, ); + printDebug('CacheSecure.cacheMain'); + return Future.sync(() => null); } catch (error) { debugPrint('[initCache] ${error}'); } @@ -92,21 +94,9 @@ Future initCache() async { // // Closes and saves storage void closeCache() async { - if (CacheSecure.cacheMain != null && CacheSecure.cacheMain.isOpen) { + if (CacheSecure.cacheMain != null) { CacheSecure.cacheMain.close(); } - - if (CacheSecure.cacheRooms != null && CacheSecure.cacheRooms.isOpen) { - CacheSecure.cacheRooms.close(); - } - - if (CacheSecure.cacheCrypto != null && CacheSecure.cacheCrypto.isOpen) { - CacheSecure.cacheCrypto.close(); - } - - if (CacheSecure.cacheMainSql != null) { - CacheSecure.cacheMainSql.close(); - } } String createIVKey() { @@ -179,42 +169,3 @@ Future unlockCryptKey() async { return cryptKey; } - -Future unlockMainCache() async { - try { - return await Hive.openBox( - CacheSecure.cacheKeyMain, - crashRecovery: true, - compactionStrategy: (entries, deletedEntries) => deletedEntries > 1, - ); - } catch (error) { - debugPrint('[unlockMainCache] $error'); - return null; - } -} - -Future unlockRoomCache() async { - try { - return await Hive.openBox( - CacheSecure.cacheKeyRooms, - crashRecovery: true, - compactionStrategy: (entries, deletedEntries) => deletedEntries > 1, - ); - } catch (error) { - debugPrint('[unlockRoomCache] $error'); - return null; - } -} - -Future unlockCryptoCache() async { - try { - return await Hive.openBox( - CacheSecure.cacheKeyCrypto, - crashRecovery: true, - compactionStrategy: (entries, deletedEntries) => deletedEntries > 1, - ); - } catch (error) { - debugPrint('[unlockCryptoCache] $error'); - return null; - } -} diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index eddabab74..3cc38e508 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -11,6 +11,7 @@ import 'package:redux_persist/redux_persist.dart'; import 'package:sembast/sembast.dart'; import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/cache/threadables.dart'; +import 'package:syphon/global/storage/index.dart'; // Project imports: import 'package:syphon/store/crypto/state.dart'; @@ -96,7 +97,7 @@ class CacheSerializer implements StateSerializer { try { Stopwatch stopwatchSave = new Stopwatch()..start(); - final cache = CacheSecure.cacheMainSql; + final cache = CacheSecure.cacheMain; final storeRef = StoreRef.main(); await storeRef.record(type).put(cache, jsonEncrypted); @@ -132,11 +133,9 @@ class CacheSerializer implements StateSerializer { SettingsStore settingsStore = SettingsStore(); UserStore userStore = UserStore(); - // final aes = AesCrypt(key: CacheSecure.cryptKey, padding: PaddingAES.pkcs7); - // Load stores previously fetched from cache, // mutable global due to redux_presist not extendable beyond Uint8List - final stores = CacheSecure.cacheStoreDecoded; + final stores = CacheSecure.cacheStores; // decode each store cache synchronously stores.forEach((type, store) { @@ -166,6 +165,10 @@ class CacheSerializer implements StateSerializer { break; case 'UserStore': userStore = UserStore.fromJson(store); + // TODO: rehydrating users from cold storage + userStore = userStore.copyWith( + users: StorageSecure.storageData['users'] ?? {}, + ); break; default: break; diff --git a/lib/global/cache/storage.dart b/lib/global/cache/storage.dart index 377be585c..d02cb16fd 100644 --- a/lib/global/cache/storage.dart +++ b/lib/global/cache/storage.dart @@ -31,7 +31,7 @@ class CacheStorage implements StorageEngine { try { Stopwatch stopwatchTotal = new Stopwatch()..start(); - final cache = CacheSecure.cacheMainSql; + final cache = CacheSecure.cacheMain; await Future.wait(stores.map((store) async { Stopwatch stopwatchStore = new Stopwatch()..start(); @@ -59,16 +59,16 @@ class CacheStorage implements StorageEngine { debugPrint('[CacheStorage] decrypt ${stopwatchStore.elapsed}'); // Load for CacheSerializer to use later - CacheSecure.cacheStoreDecoded[type] = jsonDecoded; + CacheSecure.cacheStores[type] = jsonDecoded; })); - debugPrint('[CacheStorage] load total time ${stopwatchTotal.elapsed} '); + debugPrint('[CacheStorage] total time ${stopwatchTotal.elapsed} '); } catch (error) { - printError(error, title: 'CacheStorage.load'); + printError(error, title: 'CacheStorage'); } // unlock redux_persist after cache loaded from sqflite - return null; + return Uint8List(0); } @override @@ -77,7 +77,7 @@ class CacheStorage implements StorageEngine { } static Future saveOffload(String jsonEncrypted, {String type}) async { - final cache = CacheSecure.cacheMainSql; + final cache = CacheSecure.cacheMain; final table = StoreRef.main(); final record = table.record(type); await record.put(cache, jsonEncrypted); diff --git a/lib/global/print.dart b/lib/global/print.dart index 789dea1fa..0c94b4965 100644 --- a/lib/global/print.dart +++ b/lib/global/print.dart @@ -1,19 +1,30 @@ -void printInfo(String content, {String title}) { +import 'package:flutter/material.dart'; + +typedef PrintDebug = void Function(String message, {String title}); +typedef PrintError = void Function(String message, {String title}); + +void _printInfo(String content, {String title}) { final body = title != null ? '[$title] $content' : content; print('\u001b[32m$body\u001b[0m'); } -void printWarning(String content, {String title}) { +void _printWarning(String content, {String title}) { final body = title != null ? '[$title] $content' : content; print('\u001b[34m$body\u001b[0m'); } -void printError(String content, {String title}) { +void _printError(String content, {String title}) { final body = title != null ? '[$title] $content' : content; - print('\u001b[31m$body\u001b[0m'); + debugPrint('\u001b[31m$body\u001b[0m'); } -void printDebug(String content, {String title}) { +void _printDebug(String content, {String title}) { final body = title != null ? '[$title] $content' : content; - print(body); + debugPrint(body); } + +PrintDebug printInfo = _printInfo; +PrintDebug printDebug = _printDebug; + +PrintError printError = _printError; +PrintError printWarning = _printWarning; diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart new file mode 100644 index 000000000..ac1ecabdf --- /dev/null +++ b/lib/global/storage/index.dart @@ -0,0 +1,65 @@ +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:sembast/sembast.dart'; +import 'package:sembast_sqflite/sembast_sqflite.dart'; +import 'package:syphon/global/values.dart'; +import 'package:sqflite/sqflite.dart' as sqflite; +import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; +import 'package:syphon/store/user/storage.dart'; + +class StorageSecure { + // cold storage references + static Database storageMain; + + // preloaded cold storage data + static Map storageData = {}; + + // storage identifiers + static const storageKeyMain = '${Values.appNameLabel}-main-storage'; +} + +Future initStorage() async { + try { + var storageFactory; + + var storagePath = '${StorageSecure.storageKeyMain}.db'; + + if (Platform.isAndroid || Platform.isIOS) { + // always open cold storage as sqflite + storageFactory = getDatabaseFactorySqflite( + sqflite.databaseFactory, + ); + } + + /// Supports Windows/Linux/MacOS for now. + if (Platform.isLinux || Platform.isWindows || Platform.isMacOS) { + storageFactory = getDatabaseFactorySqflite( + sqflite_ffi.databaseFactoryFfi, + ); + } + + if (factory == null) { + throw UnsupportedError( + 'Sorry, Syphon does not support your platform yet. Hope to do so soon!', + ); + } + + StorageSecure.storageMain = await storageFactory.openDatabase( + storagePath, + ); + } catch (error) { + debugPrint('[initCache] ${error}'); + } +} + +Future loadStorage() async { + StorageSecure.storageData['users'] = await loadUsers(); +} + +// // Closes and saves storage +void closeStorage() async { + if (StorageSecure.storageMain != null) { + StorageSecure.storageMain.close(); + } +} diff --git a/lib/main.dart b/lib/main.dart index 9b2ce2d9b..470929a65 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -14,6 +14,8 @@ import 'package:flutter_redux/flutter_redux.dart'; import 'package:redux/redux.dart'; import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/formatters.dart'; +import 'package:syphon/global/print.dart'; +import 'package:syphon/global/storage/index.dart'; // Project imports: import 'package:syphon/global/themes.dart'; @@ -37,6 +39,7 @@ void main() async { // disable debugPrint when in release mode if (kReleaseMode) { debugPrint = (String message, {int wrapWidth}) {}; + printDebug = (String message, {String title}) {}; } // init platform overrides for compatability with dart libs @@ -55,9 +58,18 @@ void main() async { debugPrint('[main] background service started $backgroundSyncStatus'); } - // init cold cache (mobile only) + printDebug('await initCache();'); + // init hot cache and cold storage await initCache(); + printDebug('await initStorage();'); + // init cold storage and load to data + await initStorage(); + + printDebug('await loadStorage();'); + // actually load storage to memory (to rehydrate cache for now) + await loadStorage(); + // init hot cache and start runApp(Syphon(store: await initStore())); } diff --git a/lib/store/index.dart b/lib/store/index.dart index d10187a3d..0399d588b 100644 --- a/lib/store/index.dart +++ b/lib/store/index.dart @@ -7,7 +7,6 @@ import 'package:equatable/equatable.dart'; import 'package:redux/redux.dart'; import 'package:redux_persist/redux_persist.dart'; import 'package:redux_thunk/redux_thunk.dart'; -import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/cache/storage.dart'; // Project imports: @@ -90,22 +89,17 @@ AppState appReducer(AppState state, action) => AppState( /** * Initialize Store - * - Hot redux state cache for top level data - * * Consider still using hive here - * - * PLEASE NOTE redux persist manages when the store - * should persist and if it can, not where it's persisting too - * this is why the "storage: MemoryStore()" property is set and - * the Hive Serializer has been impliemented + * - Hot redux state cache for top level data */ Future initStore() async { // Configure redux persist instance final persistor = Persistor( storage: CacheStorage(), serializer: CacheSerializer(), + // TODO: can remove once cold storage is in place throttleDuration: Duration(milliseconds: 4500), shouldSave: (Store store, dynamic action) { - // TODO: can remove once sqlcipher storage is in place + // TODO: can remove once cold storage is in place switch (action.runtimeType) { case SetSynced: if (action.synced) { diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index 206cd82f5..973b3e83a 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -10,6 +10,8 @@ import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:redux/redux.dart'; import 'package:redux_thunk/redux_thunk.dart'; import 'package:syphon/global/algos.dart'; +import 'package:syphon/global/cache/index.dart'; +import 'package:syphon/store/rooms/events/parsers.dart'; // Project imports: import 'package:syphon/store/rooms/selectors.dart' as roomSelectors; @@ -23,7 +25,9 @@ import 'package:syphon/store/media/actions.dart'; import 'package:syphon/store/rooms/events/actions.dart'; import 'package:syphon/store/rooms/events/selectors.dart'; import 'package:syphon/store/sync/actions.dart'; +import 'package:syphon/store/user/storage.dart'; import 'package:syphon/store/user/model.dart'; +import 'package:syphon/store/user/parsers.dart'; import 'events/model.dart'; import 'room/model.dart'; @@ -153,13 +157,19 @@ ThunkAction syncRooms(Map roomData) { json['timeline']['events'] = decryptedTimelineEvents; } - // Filter through parsers + // filter through parsers room = room.fromSync( json: json, currentUser: user, lastSince: lastSince, ); + // -- COLD STORAGE -- + await saveUsers( + room.users, + storage: CacheSecure.storageMain, + ); + // fetch avatar if a uri was found if (room.avatarUri != null) { store.dispatch(fetchThumbnail( diff --git a/lib/store/rooms/events/parsers.dart b/lib/store/rooms/events/parsers.dart new file mode 100644 index 000000000..a78f4331b --- /dev/null +++ b/lib/store/rooms/events/parsers.dart @@ -0,0 +1,37 @@ +import 'package:sembast/sembast.dart'; +import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/user/model.dart'; + +Future> parseStateEvents( + Map json, { + Database database, +}) { + List stateEvents = []; + + if (json['state'] != null) { + final List stateEventsRaw = json['state']['events']; + + stateEvents = + stateEventsRaw.map((event) => Event.fromMatrix(event)).toList(); + } + + if (json['invite_state'] != null) { + final List stateEventsRaw = json['invite_state']['events']; + + stateEvents = + stateEventsRaw.map((event) => Event.fromMatrix(event)).toList(); + } + + if (json['timeline'] != null) { + final List timelineEventsRaw = json['timeline']['events']; + + for (dynamic event in timelineEventsRaw) { + if (!(event['type'] == EventTypes.message || + event['type'] == EventTypes.encrypted)) { + stateEvents.add(Event.fromMatrix(event)); + } + } + } + + return Future.value(); +} diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index 67c2d226c..419134870 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -50,16 +50,19 @@ class Room { // Event lists and handlers final Message draft; + // Associated user ids + final List userIds; + // TODO: removed until state timeline work can be done // final List state; final List messages; final List outbox; + final Map messageReads; + @JsonKey(ignore: true) final Map users; - final Map messageReads; - @JsonKey(ignore: true) final bool userTyping; @@ -101,6 +104,7 @@ class Room { this.limited = false, this.draft, this.users, + this.userIds = const [], this.outbox = const [], this.messages = const [], this.lastRead = 0, @@ -143,6 +147,7 @@ class Room { isDraftRoom, draft, users, + userIds, events, outbox, messages, @@ -178,6 +183,7 @@ class Room { outbox: outbox ?? this.outbox, messages: messages ?? this.messages, users: users ?? this.users, + userIds: userIds ?? this.userIds, messageReads: messageReads ?? this.messageReads, lastHash: lastHash ?? this.lastHash, prevHash: prevHash ?? this.prevHash, @@ -459,6 +465,7 @@ class Room { name: name ?? this.name ?? Strings.labelRoomNameDefault, topic: topic ?? this.topic, users: users ?? this.users, + userIds: users.keys.toList(), direct: direct ?? this.direct, invite: invite ?? this.invite, limited: limited ?? this.limited, diff --git a/lib/store/rooms/state.dart b/lib/store/rooms/state.dart index 252210769..e963b03cb 100644 --- a/lib/store/rooms/state.dart +++ b/lib/store/rooms/state.dart @@ -21,6 +21,7 @@ class RoomStore extends Equatable { @JsonKey(ignore: true) final Map archive; // TODO: actually archive + @JsonKey(ignore: true) final List roomsHidden; diff --git a/lib/store/sync/background/service.dart b/lib/store/sync/background/service.dart index 69e9fd8c1..07519bcfd 100644 --- a/lib/store/sync/background/service.dart +++ b/lib/store/sync/background/service.dart @@ -133,8 +133,6 @@ void notificationSyncIsolate() async { roomNames = jsonDecode( await secureStorage.read(key: CacheSecure.roomNamesKey), ); - - // Init hive cache + adapters } catch (error) { print('[notificationSyncIsolate] $error'); } @@ -201,15 +199,14 @@ FutureOr syncLoop({ lastSinceNew = await secureStorage.read( key: CacheSecure.lastSinceKey, ); - // Init hive cache + adapters } catch (error) { print('[syncLoop] $error'); } /** * Check last since and see if any new messages arrived in the payload - * No need to update the hive store for now, just do not save the lastSince - * to the store and the next foreground fetchSync will update the state + * do not save the lastSince to the store and + * the next foreground fetchSync will update the state */ final data = await MatrixApi.sync( protocol: protocol, @@ -231,7 +228,6 @@ FutureOr syncLoop({ key: CacheSecure.lastSinceKey, value: lastSinceNew, ); - // Init hive cache + adapters } catch (error) { print('[syncLoop] $error'); } diff --git a/lib/store/user/parser.dart b/lib/store/user/parser.dart deleted file mode 100644 index d069ef392..000000000 --- a/lib/store/user/parser.dart +++ /dev/null @@ -1,16 +0,0 @@ -/**/ -Map parseUsers(AppState state) { - final rooms = state.roomStore.rooms.values as Iterable; - final roomsDirect = rooms.where((room) => room.direct); - final roomsDirectUsers = roomsDirect.map((room) => room.users); - - final allDirectUsers = roomsDirectUsers.fold( - {}, - (usersAll, users) { - (usersAll as Map).addAll(users); - return usersAll; - }, - ); - - return List.from(allDirectUsers.values); -} diff --git a/lib/store/user/parsers.dart b/lib/store/user/parsers.dart new file mode 100644 index 000000000..8adac0f38 --- /dev/null +++ b/lib/store/user/parsers.dart @@ -0,0 +1,8 @@ +import 'package:sembast/sembast.dart'; +import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/user/model.dart'; + +/**/ +Future parseUsers(List stateEvents, {Database database}) { + return Future.value(); +} diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart new file mode 100644 index 000000000..38fbbbd7b --- /dev/null +++ b/lib/store/user/storage.dart @@ -0,0 +1,47 @@ +import 'dart:convert'; + +import 'package:sembast/sembast.dart'; +import 'package:syphon/global/print.dart'; +import 'package:syphon/store/user/model.dart'; + +Future saveUsers( + Map users, { + Database cache, + Database storage, +}) async { + final store = StoreRef('users'); + + await storage.transaction((txn) async { + for (User user in users.values) { + final record = store.record(user.userId); + await record.put(txn, jsonEncode(user)); + } + }); + + return Future.value(); +} + +Future> loadUsers({ + Database cache, + Database storage, +}) async { + final Map userMap = {}; + try { + final store = StoreRef('users'); + printDebug('store'); + final allUsers = await store.find(storage); + printDebug('allUsers'); + + if (allUsers.isEmpty) { + return userMap; + } + printDebug('allUsers.isEmpty'); + + for (RecordSnapshot record in allUsers) { + userMap[record.key] = json.decode(record.value); + } + } catch (error) { + printDebug(error); + } + return userMap; +} From 11e8560172abae9b2a89b780f98d46593ee3d10a Mon Sep 17 00:00:00 2001 From: ereio Date: Wed, 2 Dec 2020 22:51:48 -0500 Subject: [PATCH 07/25] half way to extracting user cache to cold storage --- lib/global/cache/index.dart | 13 ++++--------- lib/global/storage/index.dart | 4 +++- lib/main.dart | 3 --- lib/store/rooms/actions.dart | 6 ++---- lib/store/user/storage.dart | 4 +--- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index f5614827b..3508f91e1 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -22,9 +22,6 @@ class CacheSecure { // hot cachee refrences static Database cacheMain; - // cold storage references - static Database storageMain; - // inital store caches for reload static Map cacheStores = {}; @@ -51,6 +48,10 @@ class CacheSecure { * (needs cold storage extracted as it's own entity) */ Future initCache() async { + // Configure cache encryption/decryption instance + CacheSecure.ivKey = await unlockIVKey(); + CacheSecure.ivKeyNext = await unlockIVKeyNext(); + CacheSecure.cryptKey = await unlockCryptKey(); try { var cacheFactory; @@ -80,16 +81,10 @@ Future initCache() async { CacheSecure.cacheMain = await cacheFactory.openDatabase( cachePath, ); - printDebug('CacheSecure.cacheMain'); return Future.sync(() => null); } catch (error) { debugPrint('[initCache] ${error}'); } - - // Configure cache encryption/decryption instance - CacheSecure.ivKey = await unlockIVKey(); - CacheSecure.ivKeyNext = await unlockIVKeyNext(); - CacheSecure.cryptKey = await unlockCryptKey(); } // // Closes and saves storage diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index ac1ecabdf..f6f5aa84a 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -54,7 +54,9 @@ Future initStorage() async { } Future loadStorage() async { - StorageSecure.storageData['users'] = await loadUsers(); + StorageSecure.storageData['users'] = await loadUsers( + storage: StorageSecure.storageMain, + ); } // // Closes and saves storage diff --git a/lib/main.dart b/lib/main.dart index 470929a65..993a5c73b 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -58,15 +58,12 @@ void main() async { debugPrint('[main] background service started $backgroundSyncStatus'); } - printDebug('await initCache();'); // init hot cache and cold storage await initCache(); - printDebug('await initStorage();'); // init cold storage and load to data await initStorage(); - printDebug('await loadStorage();'); // actually load storage to memory (to rehydrate cache for now) await loadStorage(); diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index 973b3e83a..0c2a17967 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -9,9 +9,8 @@ import 'package:flutter/foundation.dart'; import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:redux/redux.dart'; import 'package:redux_thunk/redux_thunk.dart'; -import 'package:syphon/global/algos.dart'; import 'package:syphon/global/cache/index.dart'; -import 'package:syphon/store/rooms/events/parsers.dart'; +import 'package:syphon/global/storage/index.dart'; // Project imports: import 'package:syphon/store/rooms/selectors.dart' as roomSelectors; @@ -27,7 +26,6 @@ import 'package:syphon/store/rooms/events/selectors.dart'; import 'package:syphon/store/sync/actions.dart'; import 'package:syphon/store/user/storage.dart'; import 'package:syphon/store/user/model.dart'; -import 'package:syphon/store/user/parsers.dart'; import 'events/model.dart'; import 'room/model.dart'; @@ -167,7 +165,7 @@ ThunkAction syncRooms(Map roomData) { // -- COLD STORAGE -- await saveUsers( room.users, - storage: CacheSecure.storageMain, + storage: StorageSecure.storageMain, ); // fetch avatar if a uri was found diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index 38fbbbd7b..717088885 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -26,16 +26,14 @@ Future> loadUsers({ Database storage, }) async { final Map userMap = {}; + try { final store = StoreRef('users'); - printDebug('store'); final allUsers = await store.find(storage); - printDebug('allUsers'); if (allUsers.isEmpty) { return userMap; } - printDebug('allUsers.isEmpty'); for (RecordSnapshot record in allUsers) { userMap[record.key] = json.decode(record.value); From 680cc31053f86f77c43aede9cbda8af7bb0b0dfc Mon Sep 17 00:00:00 2001 From: ereio Date: Fri, 4 Dec 2020 00:12:13 -0500 Subject: [PATCH 08/25] saving/loading users from cold storage only --- lib/global/cache/index.dart | 1 - lib/global/cache/serializer.dart | 4 +-- lib/store/user/parsers.dart | 8 ----- lib/store/user/storage.dart | 53 +++++++++++++++++++++++++++++--- 4 files changed, 49 insertions(+), 17 deletions(-) delete mode 100644 lib/store/user/parsers.dart diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index 3508f91e1..16b084fab 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -9,7 +9,6 @@ import 'package:sembast/sembast.dart'; import 'package:sembast/sembast_io.dart'; import 'package:sembast_sqflite/sembast_sqflite.dart'; import 'package:steel_crypt/steel_crypt.dart'; -import 'package:syphon/global/print.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index 3cc38e508..6804e1dff 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -38,7 +38,6 @@ class CacheSerializer implements StateSerializer { state.roomStore, state.mediaStore, state.settingsStore, - state.userStore, ]; // Queue up a cache saving will wait @@ -164,8 +163,7 @@ class CacheSerializer implements StateSerializer { settingsStore = SettingsStore.fromJson(store); break; case 'UserStore': - userStore = UserStore.fromJson(store); - // TODO: rehydrating users from cold storage + // --- cold storage only --- userStore = userStore.copyWith( users: StorageSecure.storageData['users'] ?? {}, ); diff --git a/lib/store/user/parsers.dart b/lib/store/user/parsers.dart deleted file mode 100644 index 8adac0f38..000000000 --- a/lib/store/user/parsers.dart +++ /dev/null @@ -1,8 +0,0 @@ -import 'package:sembast/sembast.dart'; -import 'package:syphon/store/rooms/events/model.dart'; -import 'package:syphon/store/user/model.dart'; - -/**/ -Future parseUsers(List stateEvents, {Database database}) { - return Future.value(); -} diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index 717088885..fd0c8c2e4 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'package:sembast/sembast.dart'; import 'package:syphon/global/print.dart'; +import 'package:syphon/global/storage/index.dart'; import 'package:syphon/store/user/model.dart'; Future saveUsers( @@ -21,25 +22,67 @@ Future saveUsers( return Future.value(); } +Future loadUser( + String userId, { + Database cache, + Database storage, +}) async { + try { + final store = StoreRef('users'); + final user = await store.findKey( + storage, + finder: Finder( + filter: Filter.byKey(userId), + ), + ); + + return jsonDecode(user); + } catch (error) { + printDebug(error); + } + return null; +} + Future> loadUsers({ Database cache, Database storage, + int offset = 0, }) async { final Map userMap = {}; try { + const limit = 2000; final store = StoreRef('users'); - final allUsers = await store.find(storage); + final count = await store.count(storage); + + final finder = Finder( + limit: limit, + offset: offset, + ); - if (allUsers.isEmpty) { + final usersPaginated = await store.find( + storage, + finder: finder, + ); + + if (usersPaginated.isEmpty) { return userMap; } - for (RecordSnapshot record in allUsers) { - userMap[record.key] = json.decode(record.value); + for (RecordSnapshot record in usersPaginated) { + userMap[record.key] = User.fromJson(json.decode(record.value)); + } + + if (offset < count) { + printDebug( + '[userMap] cur ${userMap.length.toString()} off ${offset} total ${count}'); + userMap.addAll(await loadUsers( + offset: offset + limit, + storage: storage, + )); } } catch (error) { - printDebug(error); + printDebug(error.toString()); } return userMap; } From 71a3500dc842aa5be0af0b0020b4aa53e211d212 Mon Sep 17 00:00:00 2001 From: ereio Date: Fri, 4 Dec 2020 00:30:31 -0500 Subject: [PATCH 09/25] bug fixes for user cold storage --- lib/global/cache/serializer.dart | 11 ++++------- lib/store/user/storage.dart | 1 + lib/views/home/chat/index.dart | 10 +++++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index 6804e1dff..0588b9a55 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -130,7 +130,6 @@ class CacheSerializer implements StateSerializer { MediaStore mediaStore = MediaStore(); RoomStore roomStore = RoomStore(); SettingsStore settingsStore = SettingsStore(); - UserStore userStore = UserStore(); // Load stores previously fetched from cache, // mutable global due to redux_presist not extendable beyond Uint8List @@ -163,11 +162,7 @@ class CacheSerializer implements StateSerializer { settingsStore = SettingsStore.fromJson(store); break; case 'UserStore': - // --- cold storage only --- - userStore = userStore.copyWith( - users: StorageSecure.storageData['users'] ?? {}, - ); - break; + // --- cold storage only --- default: break; } @@ -182,9 +177,11 @@ class CacheSerializer implements StateSerializer { syncStore: syncStore ?? SyncStore(), cryptoStore: cryptoStore ?? CryptoStore(), roomStore: roomStore ?? RoomStore(), - userStore: userStore ?? UserStore(), mediaStore: mediaStore ?? MediaStore(), settingsStore: settingsStore ?? SettingsStore(), + userStore: UserStore().copyWith( + users: StorageSecure.storageData['users'] ?? {}, + ), ); } } diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index fd0c8c2e4..4169fa162 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -84,5 +84,6 @@ Future> loadUsers({ } catch (error) { printDebug(error.toString()); } + printDebug('[userMap] loaded ${userMap.length.toString()}'); return userMap; } diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index 8a3192c0a..f6b035e15 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -19,6 +19,7 @@ import 'package:syphon/global/assets.dart'; // Project imports: import 'package:syphon/global/colours.dart'; import 'package:syphon/global/dimensions.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/global/strings.dart'; import 'package:syphon/global/themes.dart'; import 'package:syphon/store/crypto/actions.dart'; @@ -29,6 +30,7 @@ import 'package:syphon/store/rooms/events/model.dart'; import 'package:syphon/store/rooms/events/selectors.dart'; import 'package:syphon/store/rooms/room/model.dart'; import 'package:syphon/store/rooms/selectors.dart' as roomSelectors; +import 'package:syphon/store/user/model.dart'; import 'package:syphon/views/home/chat/chat-input.dart'; import 'package:syphon/views/home/chat/dialog-encryption.dart'; import 'package:syphon/views/home/chat/dialog-invite.dart'; @@ -453,7 +455,9 @@ class ChatViewState extends State { ? this.selectedMessage.id : null; - final avatarUri = props.room.users[message.sender]?.avatarUri; + printDebug(message.sender); + printDebug(props.users[message.sender]?.avatarUri); + final avatarUri = props.users[message.sender]?.avatarUri; return MessageWidget( message: message, @@ -654,6 +658,7 @@ class _Props extends Equatable { final String userId; final bool loading; final ThemeType theme; + final Map users; final List messages; final Color roomPrimaryColor; final bool timeFormat24Enabled; @@ -676,6 +681,7 @@ class _Props extends Equatable { @required this.room, @required this.theme, @required this.userId, + @required this.users, @required this.messages, @required this.loading, @required this.roomPrimaryColor, @@ -698,6 +704,7 @@ class _Props extends Equatable { @override List get props => [ userId, + users, messages, room, roomPrimaryColor, @@ -713,6 +720,7 @@ class _Props extends Equatable { store.state.settingsStore.timeFormat24Enabled ?? false, loading: (store.state.roomStore.rooms[roomId] ?? Room()).syncing, room: roomSelectors.room(id: roomId, state: store.state), + users: store.state.userStore.users, messages: latestMessages( wrapOutboxMessages( messages: roomSelectors.room(id: roomId, state: store.state).messages, From ec0202b8df2588233e91c5cd4102c8c03393f825 Mon Sep 17 00:00:00 2001 From: ereio Date: Fri, 4 Dec 2020 00:44:20 -0500 Subject: [PATCH 10/25] clean up --- lib/store/user/storage.dart | 4 +--- lib/views/home/chat/index.dart | 2 -- lib/views/widgets/modals/modal-user-details.dart | 8 ++++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index 4169fa162..b994ee3d9 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -51,7 +51,7 @@ Future> loadUsers({ final Map userMap = {}; try { - const limit = 2000; + const limit = 5000; final store = StoreRef('users'); final count = await store.count(storage); @@ -74,8 +74,6 @@ Future> loadUsers({ } if (offset < count) { - printDebug( - '[userMap] cur ${userMap.length.toString()} off ${offset} total ${count}'); userMap.addAll(await loadUsers( offset: offset + limit, storage: storage, diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index f6b035e15..8f473abbc 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -455,8 +455,6 @@ class ChatViewState extends State { ? this.selectedMessage.id : null; - printDebug(message.sender); - printDebug(props.users[message.sender]?.avatarUri); final avatarUri = props.users[message.sender]?.avatarUri; return MessageWidget( diff --git a/lib/views/widgets/modals/modal-user-details.dart b/lib/views/widgets/modals/modal-user-details.dart index 946ccfb90..6d5ee84ad 100644 --- a/lib/views/widgets/modals/modal-user-details.dart +++ b/lib/views/widgets/modals/modal-user-details.dart @@ -285,11 +285,11 @@ class _Props extends Equatable { return user; } - final room = store.state.roomStore.rooms[roomId]; - if (room != null) { - return room.users[userId]; + if (userId == null) { + return null; } - return null; + + return store.state.userStore.users[userId]; }(), onDisabled: () => store.dispatch(addInProgress()), onCreateChatDirect: ({User user}) async => store.dispatch( From 5fa45819f7504594746c7c4ce25da051042a1598 Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 6 Dec 2020 13:00:31 -0500 Subject: [PATCH 11/25] replacing room.users with room.userIds --- android/app/proguard-rules.pro | 1 + assets/cheatsheet.md | 14 +++++++++++++- lib/store/crypto/actions.dart | 21 ++++++++++++--------- lib/store/rooms/actions.dart | 16 +++++++++------- lib/store/rooms/room/model.dart | 4 +++- lib/store/user/selectors.dart | 17 ++++++----------- lib/views/home/chat/details-chat.dart | 2 +- lib/views/home/chat/index.dart | 7 ++++--- pubspec.lock | 7 +++++++ pubspec.yaml | 1 + 10 files changed, 57 insertions(+), 33 deletions(-) create mode 100644 android/app/proguard-rules.pro diff --git a/android/app/proguard-rules.pro b/android/app/proguard-rules.pro new file mode 100644 index 000000000..d0e0fbc9b --- /dev/null +++ b/android/app/proguard-rules.pro @@ -0,0 +1 @@ +-keep class net.sqlcipher.** { *; } \ No newline at end of file diff --git a/assets/cheatsheet.md b/assets/cheatsheet.md index 61b976afe..a51ce6ea7 100644 --- a/assets/cheatsheet.md +++ b/assets/cheatsheet.md @@ -136,4 +136,16 @@ if (true) { } } -``` \ No newline at end of file +``` + +```dart + // reduce several maps to one map + final allDirectUsers = roomsDirectUsers.fold( + {}, + (usersAll, users) { + (usersAll as Map).addAll(users); + return usersAll; + }, + ); + + ``` \ No newline at end of file diff --git a/lib/store/crypto/actions.dart b/lib/store/crypto/actions.dart index 4bbaea6c1..da996461c 100644 --- a/lib/store/crypto/actions.dart +++ b/lib/store/crypto/actions.dart @@ -591,14 +591,14 @@ ThunkAction claimOneTimeKeys({ }) { return (Store store) async { try { - final roomUsers = room.users.values; + final roomUserIds = room.userIds; final deviceKeys = store.state.cryptoStore.deviceKeys; final outboundKeySessions = store.state.cryptoStore.outboundKeySessions; final currentUser = store.state.authStore.user; // get deviceKeys for every user present in the chat - final List roomDeviceKeys = List.from(roomUsers - .map((user) => (deviceKeys[user.userId] ?? {}).values) + final List roomDeviceKeys = List.from(roomUserIds + .map((userId) => (deviceKeys[userId] ?? {}).values) .expand((x) => x)); // Create a map of all the oneTimeKeys to claim @@ -1012,17 +1012,22 @@ ThunkAction exportMessageSession({String roomId}) { * fetches the keys uploaded to the matrix homeserver * by other users */ -ThunkAction fetchDeviceKeys({Map users}) { +ThunkAction fetchDeviceKeys( + {Map users, List userIds}) { return (Store store) async { try { - final userMap = users.map((userId, user) => MapEntry(userId, const [])); + final Map userIdMap = Map.fromIterable( + userIds, + key: (userId) => userId, + value: (userId) => const [], + ); final data = await MatrixApi.fetchKeys( protocol: protocol, homeserver: store.state.authStore.user.homeserver, accessToken: store.state.authStore.user.accessToken, lastSince: store.state.syncStore.lastSince, - users: userMap, + users: userIdMap, ); final Map deviceKeys = data['device_keys']; @@ -1054,9 +1059,7 @@ ThunkAction fetchDeviceKeys({Map users}) { ThunkAction fetchDeviceKeysOwned(User user) { return (Store store) async { final deviceKeys = await store.dispatch( - fetchDeviceKeys(users: { - user.userId: user, - }), + fetchDeviceKeys(userIds: [user.userId]), ); return deviceKeys[user.userId]; }; diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index 0c2a17967..389806d64 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -597,27 +597,29 @@ ThunkAction toggleDirectRoom({Room room, bool enabled}) { // Find the other user in the direct room final currentUser = store.state.authStore.user; - final otherUser = room.users.values.firstWhere( - (user) => user.userId != currentUser.userId, + + // only the other user id, and not the user object, is needed here + final otherUserId = room.userIds.firstWhere( + (userId) => userId != currentUser.userId, ); - if (otherUser == null) { + if (otherUserId == null) { throw 'Cannot toggle room to direct without other users'; } // Pull the direct room for that specific user Map directRoomUsers = data as Map; - final usersDirectRooms = directRoomUsers[otherUser.userId] ?? []; + final usersDirectRooms = directRoomUsers[otherUserId] ?? []; if (usersDirectRooms.isEmpty && enabled) { - directRoomUsers[otherUser.userId] = [room.id]; + directRoomUsers[otherUserId] = [room.id]; } // Toggle the direct room data based on user actions directRoomUsers = directRoomUsers.map((userId, rooms) { List updatedRooms = List.from(rooms ?? []); - if (userId != otherUser.userId) { + if (userId != otherUserId) { return MapEntry(userId, updatedRooms); } @@ -1030,7 +1032,7 @@ ThunkAction archiveRoom({Room room}) { // createRoom( // name: room.name, // topic: room.topic, -// invites: room.users, +// invites: room.userIds, // isDirect: room.direct, // ), // ); diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index 419134870..7c1460635 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -461,14 +461,16 @@ class Room { } } catch (error) {} + final userIds = this.userIds..addAll(users.keys ?? []); + return this.copyWith( name: name ?? this.name ?? Strings.labelRoomNameDefault, topic: topic ?? this.topic, users: users ?? this.users, - userIds: users.keys.toList(), direct: direct ?? this.direct, invite: invite ?? this.invite, limited: limited ?? this.limited, + userIds: userIds ?? this.userIds, avatarUri: avatarUri ?? this.avatarUri, joinRule: joinRule ?? this.joinRule, lastUpdate: lastUpdate > 0 ? lastUpdate : this.lastUpdate, diff --git a/lib/store/user/selectors.dart b/lib/store/user/selectors.dart index 27b44e135..3b01480fd 100644 --- a/lib/store/user/selectors.dart +++ b/lib/store/user/selectors.dart @@ -12,19 +12,14 @@ dynamic homeserver(AppState state) { // Users the authed user has dm'ed List friendlyUsers(AppState state) { - final rooms = state.roomStore.rooms.values as Iterable; + final rooms = state.roomStore.rooms.values; + final users = state.userStore.users; final roomsDirect = rooms.where((room) => room.direct); - final roomsDirectUsers = roomsDirect.map((room) => room.users); + final roomUserIdsList = roomsDirect.map((room) => room.userIds); + final roomDirectUserIds = roomUserIdsList.expand((pair) => pair).toList(); + final roomsDirectUsers = roomDirectUserIds.map((userId) => users[userId]); - final allDirectUsers = roomsDirectUsers.fold( - {}, - (usersAll, users) { - (usersAll as Map).addAll(users); - return usersAll; - }, - ); - - return List.from(allDirectUsers.values); + return List.from(roomsDirectUsers); } /* diff --git a/lib/views/home/chat/details-chat.dart b/lib/views/home/chat/details-chat.dart index 05c92cac1..45a3b101b 100644 --- a/lib/views/home/chat/details-chat.dart +++ b/lib/views/home/chat/details-chat.dart @@ -247,7 +247,7 @@ class ChatDetailsState extends State { ), Container( child: Text( - ' (${props.room.users.length})', + ' (${props.room.userIds.length})', textAlign: TextAlign.start, ), ), diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index 8f473abbc..06b3979b3 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -423,7 +423,7 @@ class ChatViewState extends State { MessageTypingWidget( typing: props.room.userTyping, usersTyping: props.room.usersTyping, - roomUsers: props.room.users, + roomUsers: props.users, selectedMessageId: this.selectedMessage != null ? this.selectedMessage.id : null, @@ -739,7 +739,7 @@ class _Props extends Equatable { final room = store.state.roomStore.rooms[roomId]; final usersDeviceKeys = await store.dispatch( - fetchDeviceKeys(users: room.users), + fetchDeviceKeys(userIds: room.userIds), ); store.dispatch(setDeviceKeys(usersDeviceKeys)); @@ -832,8 +832,9 @@ class _Props extends Equatable { store.dispatch(updateKeySessions(room: room)); final usersDeviceKeys = await store.dispatch( - fetchDeviceKeys(users: room.users), + fetchDeviceKeys(userIds: room.userIds), ); + printJson(usersDeviceKeys); }); } diff --git a/pubspec.lock b/pubspec.lock index d3c5a6309..a3f66f18e 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -868,6 +868,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.1.1" + sqflite_sqlcipher: + dependency: "direct main" + description: + name: sqflite_sqlcipher + url: "https://pub.dartlang.org" + source: hosted + version: "1.1.2" sqlite3: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 47aadaeb3..32e90e646 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -89,6 +89,7 @@ dependencies: sqflite: ^1.3.2 sembast: ^2.4.8 sembast_sqflite: 1.0.0 + sqflite_sqlcipher: 1.1.2 sqflite_common_ffi: 1.1.1 # isolate_handler: 0.3.1 # flutter_isolate: 1.0.0+14 From da4013988795a83469ce4cb7098f5c40372181a8 Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 6 Dec 2020 13:47:01 -0500 Subject: [PATCH 12/25] changes for userIds --- ios/Podfile.lock | 17 +++++++++++++++++ ios/Runner.xcodeproj/project.pbxproj | 4 ++++ lib/store/rooms/room/model.dart | 5 +++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ios/Podfile.lock b/ios/Podfile.lock index ff438e804..173406c5c 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -43,6 +43,8 @@ PODS: - Flutter - FMDB (2.7.5): - FMDB/standard (= 2.7.5) + - FMDB/SQLCipher (2.7.5): + - SQLCipher - FMDB/standard (2.7.5) - image_picker (0.0.1): - Flutter @@ -66,6 +68,15 @@ PODS: - sqflite (0.0.2): - Flutter - FMDB (>= 2.7.5) + - sqflite_sqlcipher (0.0.1): + - Flutter + - FMDB/SQLCipher (~> 2.7.5) + - SQLCipher (= 4.4.1) + - SQLCipher (4.4.1): + - SQLCipher/standard (= 4.4.1) + - SQLCipher/common (4.4.1) + - SQLCipher/standard (4.4.1): + - SQLCipher/common - url_launcher (0.0.1): - Flutter - webview_flutter (0.0.1): @@ -83,6 +94,7 @@ DEPENDENCIES: - path_provider (from `.symlinks/plugins/path_provider/ios`) - shared_preferences (from `.symlinks/plugins/shared_preferences/ios`) - sqflite (from `.symlinks/plugins/sqflite/ios`) + - sqflite_sqlcipher (from `.symlinks/plugins/sqflite_sqlcipher/ios`) - url_launcher (from `.symlinks/plugins/url_launcher/ios`) - webview_flutter (from `.symlinks/plugins/webview_flutter/ios`) @@ -95,6 +107,7 @@ SPEC REPOS: - OLMKit - SDWebImage - SDWebImageFLPlugin + - SQLCipher EXTERNAL SOURCES: device_info: @@ -117,6 +130,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/shared_preferences/ios" sqflite: :path: ".symlinks/plugins/sqflite/ios" + sqflite_sqlcipher: + :path: ".symlinks/plugins/sqflite_sqlcipher/ios" url_launcher: :path: ".symlinks/plugins/url_launcher/ios" webview_flutter: @@ -140,6 +155,8 @@ SPEC CHECKSUMS: SDWebImageFLPlugin: 6c2295fb1242d44467c6c87dc5db6b0a13228fd8 shared_preferences: af6bfa751691cdc24be3045c43ec037377ada40d sqflite: 6d358c025f5b867b29ed92fc697fd34924e11904 + sqflite_sqlcipher: cc00ecef02643b799857eb5c66c7074660aa658a + SQLCipher: 0b81a39b21247c559c52c3bd2234808e626c008b url_launcher: 6fef411d543ceb26efce54b05a0a40bfd74cbbef webview_flutter: d2b4d6c66968ad042ad94cbb791f5b72b4678a96 diff --git a/ios/Runner.xcodeproj/project.pbxproj b/ios/Runner.xcodeproj/project.pbxproj index 0b5b6a538..c51e6c44e 100644 --- a/ios/Runner.xcodeproj/project.pbxproj +++ b/ios/Runner.xcodeproj/project.pbxproj @@ -224,6 +224,7 @@ "${BUILT_PRODUCTS_DIR}/OLMKit/OLMKit.framework", "${BUILT_PRODUCTS_DIR}/SDWebImage/SDWebImage.framework", "${BUILT_PRODUCTS_DIR}/SDWebImageFLPlugin/SDWebImageFLPlugin.framework", + "${BUILT_PRODUCTS_DIR}/SQLCipher/SQLCipher.framework", "${BUILT_PRODUCTS_DIR}/device_info/device_info.framework", "${BUILT_PRODUCTS_DIR}/file_picker/file_picker.framework", "${BUILT_PRODUCTS_DIR}/flutter_local_notifications/flutter_local_notifications.framework", @@ -233,6 +234,7 @@ "${BUILT_PRODUCTS_DIR}/path_provider/path_provider.framework", "${BUILT_PRODUCTS_DIR}/shared_preferences/shared_preferences.framework", "${BUILT_PRODUCTS_DIR}/sqflite/sqflite.framework", + "${BUILT_PRODUCTS_DIR}/sqflite_sqlcipher/sqflite_sqlcipher.framework", "${BUILT_PRODUCTS_DIR}/url_launcher/url_launcher.framework", "${BUILT_PRODUCTS_DIR}/webview_flutter/webview_flutter.framework", ); @@ -246,6 +248,7 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/OLMKit.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/SDWebImage.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/SDWebImageFLPlugin.framework", + "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/SQLCipher.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/device_info.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/file_picker.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/flutter_local_notifications.framework", @@ -255,6 +258,7 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/path_provider.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/shared_preferences.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sqflite.framework", + "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sqflite_sqlcipher.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/url_launcher.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/webview_flutter.framework", ); diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index 7c1460635..d0c476adb 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -358,6 +358,7 @@ class Room { int lastUpdate = this.lastUpdate; int namePriority = this.namePriority != 4 ? this.namePriority : 4; Map users = this.users ?? Map(); + Set userIds = Set.from(this.userIds) ?? Set(); try { events.forEach((event) { @@ -461,7 +462,7 @@ class Room { } } catch (error) {} - final userIds = this.userIds..addAll(users.keys ?? []); + userIds = userIds..addAll(users.keys ?? []); return this.copyWith( name: name ?? this.name ?? Strings.labelRoomNameDefault, @@ -470,7 +471,7 @@ class Room { direct: direct ?? this.direct, invite: invite ?? this.invite, limited: limited ?? this.limited, - userIds: userIds ?? this.userIds, + userIds: userIds.toList() ?? this.userIds, avatarUri: avatarUri ?? this.avatarUri, joinRule: joinRule ?? this.joinRule, lastUpdate: lastUpdate > 0 ? lastUpdate : this.lastUpdate, From e1541bbd5a68f040974cd5a6ef2f906abd6681c7 Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 6 Dec 2020 19:00:29 -0500 Subject: [PATCH 13/25] replaced encryption library due to memory leak suspicion --- lib/global/cache/index.dart | 17 ++++----- lib/global/cache/serializer.dart | 24 ++++++------ lib/global/cache/storage.dart | 10 ++--- lib/global/cache/threadables.dart | 63 ++++++++++++++++++------------- lib/global/storage/index.dart | 2 + pubspec.lock | 37 ++++++++++-------- pubspec.yaml | 2 +- 7 files changed, 86 insertions(+), 69 deletions(-) diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index 16b084fab..82fdcefe1 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -1,14 +1,14 @@ import 'dart:io'; +import 'package:encrypt/encrypt.dart'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; -import 'package:flutter/material.dart'; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; import 'package:sembast/sembast.dart'; import 'package:sembast/sembast_io.dart'; import 'package:sembast_sqflite/sembast_sqflite.dart'; -import 'package:steel_crypt/steel_crypt.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; @@ -52,9 +52,8 @@ Future initCache() async { CacheSecure.ivKeyNext = await unlockIVKeyNext(); CacheSecure.cryptKey = await unlockCryptKey(); try { - var cacheFactory; - var cachePath = '${CacheSecure.cacheKeyMain}.db'; + var cacheFactory; if (Platform.isAndroid || Platform.isIOS) { var directory = await getApplicationDocumentsDirectory(); @@ -71,7 +70,7 @@ Future initCache() async { ); } - if (factory == null) { + if (cacheFactory == null) { throw UnsupportedError( 'Sorry, Syphon does not support your platform yet. Hope to do so soon!', ); @@ -82,7 +81,7 @@ Future initCache() async { ); return Future.sync(() => null); } catch (error) { - debugPrint('[initCache] ${error}'); + printDebug('[initCache] ${error}'); } } @@ -94,7 +93,7 @@ void closeCache() async { } String createIVKey() { - return CryptKey().genDart(); + return Key.fromSecureRandom(16).base64; } Future saveIVKey(String ivKey) async { @@ -148,12 +147,12 @@ Future unlockCryptKey() async { key: CacheSecure.cryptKeyLocation, ); } catch (error) { - debugPrint('[unlockCryptKey] ${error}'); + printDebug('[unlockCryptKey] ${error}'); } // Create a encryptionKey if a serialized one is not found if (cryptKey == null) { - cryptKey = CryptKey().genFortuna(len: 32); // 256 bits + cryptKey = Key.fromSecureRandom(32).base64; await storageEngine.write( key: CacheSecure.cryptKeyLocation, diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index 0588b9a55..15e8c5af7 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -57,7 +57,7 @@ class CacheSerializer implements StateSerializer { String type = store.runtimeType.toString(); // serialize the store contents - Stopwatch stopwatchSerialize = new Stopwatch()..start(); + // Stopwatch stopwatchSerialize = new Stopwatch()..start(); try { // HACK: unable to pass certain stores directly to an isolate final sensitiveStorage = [AuthStore, SyncStore, CryptoStore]; @@ -73,11 +73,11 @@ class CacheSerializer implements StateSerializer { ); } - debugPrint( - '[CacheSerializer] ${stopwatchSerialize.elapsed} ${type} serialize', - ); + // debugPrint( + // '[CacheSerializer] ${stopwatchSerialize.elapsed} ${type} serialize', + // ); - Stopwatch stopwatchEncrypt = new Stopwatch()..start(); + // Stopwatch stopwatchEncrypt = new Stopwatch()..start(); // encrypt the store contents jsonEncrypted = await compute( encryptJsonBackground, @@ -90,19 +90,19 @@ class CacheSerializer implements StateSerializer { debugLabel: 'encryptJsonBackground', ); - debugPrint( - '[CacheSerializer] ${stopwatchEncrypt.elapsed} ${type} encrypt', - ); + // debugPrint( + // '[CacheSerializer] ${stopwatchEncrypt.elapsed} ${type} encrypt', + // ); try { - Stopwatch stopwatchSave = new Stopwatch()..start(); + // Stopwatch stopwatchSave = new Stopwatch()..start(); final cache = CacheSecure.cacheMain; final storeRef = StoreRef.main(); await storeRef.record(type).put(cache, jsonEncrypted); - debugPrint( - '[CacheSerializer] ${stopwatchSave.elapsed} ${type} saved', - ); + // debugPrint( + // '[CacheSerializer] ${stopwatchSave.elapsed} ${type} saved', + // ); } catch (error) { print('[CacheSerializer] ERROR $error'); } diff --git a/lib/global/cache/storage.dart b/lib/global/cache/storage.dart index d02cb16fd..96322f6ca 100644 --- a/lib/global/cache/storage.dart +++ b/lib/global/cache/storage.dart @@ -29,19 +29,19 @@ class CacheStorage implements StorageEngine { @override Future load() async { try { - Stopwatch stopwatchTotal = new Stopwatch()..start(); + // Stopwatch stopwatchTotal = new Stopwatch()..start(); final cache = CacheSecure.cacheMain; await Future.wait(stores.map((store) async { - Stopwatch stopwatchStore = new Stopwatch()..start(); + // Stopwatch stopwatchStore = new Stopwatch()..start(); // Fetch from database final type = store.runtimeType.toString(); final table = StoreRef.main(); final record = table.record(store.runtimeType.toString()); final jsonEncrypted = await record.get(cache); - debugPrint('[CacheStorage] load ${stopwatchStore.elapsed}'); + // printDebug('[CacheStorage] load ${stopwatchStore.elapsed}'); // Decrypt from database final jsonDecoded = await compute( @@ -56,13 +56,13 @@ class CacheStorage implements StorageEngine { debugLabel: 'decryptJsonBackground', ); - debugPrint('[CacheStorage] decrypt ${stopwatchStore.elapsed}'); + // printDebug('[CacheStorage] decrypt ${stopwatchStore.elapsed}'); // Load for CacheSerializer to use later CacheSecure.cacheStores[type] = jsonDecoded; })); - debugPrint('[CacheStorage] total time ${stopwatchTotal.elapsed} '); + // printDebug('[CacheStorage] total time ${stopwatchTotal.elapsed} '); } catch (error) { printError(error, title: 'CacheStorage'); } diff --git a/lib/global/cache/threadables.dart b/lib/global/cache/threadables.dart index 3b3469cab..2c3bb8d36 100644 --- a/lib/global/cache/threadables.dart +++ b/lib/global/cache/threadables.dart @@ -1,22 +1,23 @@ import 'dart:convert'; - -import 'dart:ui'; import 'dart:async'; -import 'package:flutter/material.dart'; -import 'package:flutter/services.dart'; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; -import 'package:steel_crypt/steel_crypt.dart'; +import 'package:encrypt/encrypt.dart'; import 'package:syphon/global/cache/index.dart'; +import 'package:syphon/global/print.dart'; Future encryptJsonBackground(Map params) async { String ivKey = params['ivKey']; String cryptKey = params['cryptKey']; String json = params['json']; - final cryptor = AesCrypt(key: cryptKey, padding: PaddingAES.pkcs7); + final iv = IV.fromBase64(ivKey); + final key = Key.fromBase64(cryptKey); + + final encrypter = Encrypter(AES(key, mode: AESMode.ctr, padding: null)); + final encrypted = encrypter.encrypt(json, iv: iv); - return cryptor.ctr.encrypt(inp: json, iv: ivKey); + return encrypted.base64; } Future decryptJsonBackground(Map params) async { @@ -29,41 +30,45 @@ Future decryptJsonBackground(Map params) async { String jsonDecrypted; Map jsonDecoded = {}; - final aes = AesCrypt(key: cryptKey, padding: PaddingAES.pkcs7); + final iv = IV.fromBase64(ivKey); + final ivNext = IV.fromBase64(ivKeyNext); + final key = Key.fromBase64(cryptKey); + + final encrypter = Encrypter(AES(key, mode: AESMode.ctr, padding: null)); if (jsonEncrypted == null) return null; try { - jsonDecrypted = aes.ctr.decrypt( - enc: jsonEncrypted, - iv: ivKey, + jsonDecrypted = encrypter.decrypt64( + jsonEncrypted, + iv: iv, ); } catch (error) { - debugPrint('[decryptJsonBackground] error $error'); + printDebug('[decryptJsonBackground] error $error'); } if (jsonDecoded.isEmpty) { try { - jsonDecrypted = aes.ctr.decrypt( - enc: jsonEncrypted, - iv: ivKeyNext, + jsonDecrypted = encrypter.decrypt64( + jsonEncrypted, + iv: ivNext, ); } catch (error) { - debugPrint('[decryptJsonBackground] error $error'); + printDebug('[decryptJsonBackground] error $error'); jsonDecoded = {}; } } // Failed to decrypt data if (jsonDecrypted == null) { - debugPrint('[decryptJsonBackground] decryption failed ${type}'); + printDebug('[decryptJsonBackground] decryption failed ${type}'); return null; } // decode serialized object jsonDecoded = json.decode(jsonDecrypted); - debugPrint('[decryptJsonBackground] decryption succeed ${type}'); + printDebug('[decryptJsonBackground] decryption succeed ${type}'); return jsonDecoded; } @@ -73,23 +78,27 @@ Future decryptJsonBackground(Map params) async { // to the isolate // responsibile for both json serialization and encryption Future serializeJsonBackground(Object store) async { - WidgetsFlutterBinding.ensureInitialized(); - window.onPlatformMessage = BinaryMessages.handlePlatformMessage; - try { final storageEngine = FlutterSecureStorage(); - final ivKey = await storageEngine.read(key: CacheSecure.ivKeyLocation); - final cryptKey = - await storageEngine.read(key: CacheSecure.cryptKeyLocation); + final ivKey = await storageEngine.read( + key: CacheSecure.ivKeyLocation, + ); + final cryptKey = await storageEngine.read( + key: CacheSecure.cryptKeyLocation, + ); + + final iv = IV.fromBase64(ivKey); + final key = Key.fromBase64(cryptKey); final jsonEncoded = jsonEncode(store); - final cryptor = AesCrypt(key: cryptKey, padding: PaddingAES.pkcs7); + final encrypter = Encrypter(AES(key, mode: AESMode.ctr)); + final encrypted = encrypter.encrypt(jsonEncoded, iv: iv); - return cryptor.ctr.encrypt(inp: jsonEncoded, iv: ivKey); + return encrypted.base64; } catch (error) { - debugPrint('[serializeJsonBackground] $error'); + printDebug('[serializeJsonBackground] $error'); return null; } } diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index f6f5aa84a..53de8ed11 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -6,11 +6,13 @@ import 'package:sembast_sqflite/sembast_sqflite.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite/sqflite.dart' as sqflite; import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; +import 'package:sqflite_sqlcipher/sqflite.dart' as sqflite_sqlcipher; import 'package:syphon/store/user/storage.dart'; class StorageSecure { // cold storage references static Database storageMain; + static sqflite_sqlcipher.Database storageMainEncrypted; // preloaded cold storage data static Map storageData = {}; diff --git a/pubspec.lock b/pubspec.lock index a3f66f18e..d7865ac52 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -42,7 +42,7 @@ packages: name: asn1lib url: "https://pub.dartlang.org" source: hosted - version: "0.6.5" + version: "0.8.1" async: dependency: transitive description: @@ -148,6 +148,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.1.4" + clock: + dependency: transitive + description: + name: clock + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.1" code_builder: dependency: transitive description: @@ -232,6 +239,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.3.3" + encrypt: + dependency: "direct main" + description: + name: encrypt + url: "https://pub.dartlang.org" + source: hosted + version: "4.1.0" equatable: dependency: "direct main" description: @@ -625,13 +639,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.0.3" - pc_steelcrypt: - dependency: transitive - description: - name: pc_steelcrypt - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.1" pedantic: dependency: transitive description: @@ -667,6 +674,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.0.2" + pointycastle: + dependency: transitive + description: + name: pointycastle + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.0" pool: dependency: transitive description: @@ -889,13 +903,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.9.5" - steel_crypt: - dependency: "direct main" - description: - name: steel_crypt - url: "https://pub.dartlang.org" - source: hosted - version: "2.2.2+1" stream_channel: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 32e90e646..788a0aca1 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -71,7 +71,7 @@ dependencies: # Encryption crypt: 3.0.1 crypto: ^2.1.5 - steel_crypt: ^2.2.2+1 + encrypt: 4.1.0 olm: 1.2.1 # flutter_olm: 1.0.1 # cryptography: 1.2.1 From ea3dd17dae7af2c07c78d60563e1aa2d4d6a8dc1 Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 6 Dec 2020 20:26:40 -0500 Subject: [PATCH 14/25] pub cleanup --- pubspec.lock | 14 ------------- pubspec.yaml | 57 +++++++++++----------------------------------------- 2 files changed, 12 insertions(+), 59 deletions(-) diff --git a/pubspec.lock b/pubspec.lock index d7865ac52..f5c6e8435 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -415,20 +415,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.2.0" - hive: - dependency: "direct main" - description: - name: hive - url: "https://pub.dartlang.org" - source: hosted - version: "1.4.4" - hive_flutter: - dependency: "direct main" - description: - name: hive_flutter - url: "https://pub.dartlang.org" - source: hosted - version: "0.3.1" html: dependency: "direct main" description: diff --git a/pubspec.yaml b/pubspec.yaml index 788a0aca1..efe1a1cd8 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -53,7 +53,7 @@ dependencies: flutter: sdk: flutter - # State Management + # state expandable: 3.0.1 equatable: 1.2.4 canonical_json: 1.0.0 @@ -68,33 +68,23 @@ dependencies: path_provider: 1.6.14 package_info: 0.4.3 - # Encryption + # encryption + olm: 1.2.1 crypt: 3.0.1 crypto: ^2.1.5 encrypt: 4.1.0 - olm: 1.2.1 - # flutter_olm: 1.0.1 - # cryptography: 1.2.1 - # olm: - # git: - # url: https://gitlab.com/famedly/libraries/dart-olm - # ref: 4853b5301519a50866a81b58ac3730830a4fe547 - - # Cache - hive: 1.4.4 - hive_flutter: 0.3.1 - flutter_secure_storage: 3.3.3 - json_annotation: ^3.1.0 - checked_yaml: 1.0.2 - sqflite: ^1.3.2 + + # cache/storage sembast: ^2.4.8 + sqflite: ^1.3.2 sembast_sqflite: 1.0.0 sqflite_sqlcipher: 1.1.2 sqflite_common_ffi: 1.1.1 - # isolate_handler: 0.3.1 - # flutter_isolate: 1.0.0+14 + checked_yaml: 1.0.2 + json_annotation: ^3.1.0 + flutter_secure_storage: 3.3.3 - # Services + # services http: ^0.12.0+2 html: ^0.13.3+3 intl: ^0.16.1 @@ -127,10 +117,9 @@ dev_dependencies: json_serializable: ^3.5.0 flutter_launcher_icons: "^0.7.5" build_runner: ^1.10.1 - # build_resolvers: 1.3.10 # <- modified to solve build_runner dependency_overrides: - analyzer: 0.39.16 # <- override to solve build_runner + analyzer: 0.39.16 # <- override for json_serializable build_runner flutter_icons: android: true @@ -146,6 +135,7 @@ flutter: # the material Icons class. uses-material-design: true + # see https://flutter.dev/custom-fonts/#from-packages fonts: - family: Poppins fonts: @@ -206,26 +196,3 @@ flutter: - assets/icons/global/being-send-unlock.svg - assets/icons/global/being-chevrons-right.svg - assets/icons/global/feather-message-circle.svg - # An image asset can refer to one or more resolution-specific "variants", see - # https://flutter.dev/assets-and-images/#resolution-aware. - # For details regarding adding assets from package dependencies, see - # https://flutter.dev/assets-and-images/#from-packages - # To add custom fonts to your application, add a fonts section here, - # in this "flutter" section. Each entry in this list should have a - # "family" key with the font family name, and a "fonts" key with a - # list giving the asset and other descriptors for the font. For - # example: - # fonts: - # - family: Schyler - # fonts: - # - asset: fonts/Schyler-Regular.ttf - # - asset: fonts/Schyler-Italic.ttf - # style: italic - # - family: Trajan Pro - # fonts: - # - asset: fonts/TrajanPro.ttf - # - asset: fonts/TrajanPro_Bold.ttf - # weight: 700 - # - # For details regarding fonts from package dependencies, - # see https://flutter.dev/custom-fonts/#from-packages From 0d5ccba07f064d4b1f478ffd87ccd24a16cbf3ad Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 6 Dec 2020 20:46:39 -0500 Subject: [PATCH 15/25] trying out codec for cold storage, better error handling --- lib/global/cache/storage.dart | 22 +++--- lib/global/storage/codec.dart | 118 ++++++++++++++++++++++++++++++++ lib/global/storage/index.dart | 17 +++-- lib/store/rooms/room/model.dart | 12 ++-- 4 files changed, 144 insertions(+), 25 deletions(-) create mode 100644 lib/global/storage/codec.dart diff --git a/lib/global/cache/storage.dart b/lib/global/cache/storage.dart index 96322f6ca..379c48e9e 100644 --- a/lib/global/cache/storage.dart +++ b/lib/global/cache/storage.dart @@ -28,15 +28,14 @@ final List stores = [ class CacheStorage implements StorageEngine { @override Future load() async { - try { - // Stopwatch stopwatchTotal = new Stopwatch()..start(); - - final cache = CacheSecure.cacheMain; + final cache = CacheSecure.cacheMain; - await Future.wait(stores.map((store) async { + await Future.wait(stores.map((store) async { + final type = store.runtimeType.toString(); + try { + // Stopwatch stopwatchTotal = new Stopwatch()..start(); // Stopwatch stopwatchStore = new Stopwatch()..start(); // Fetch from database - final type = store.runtimeType.toString(); final table = StoreRef.main(); final record = table.record(store.runtimeType.toString()); final jsonEncrypted = await record.get(cache); @@ -60,12 +59,11 @@ class CacheStorage implements StorageEngine { // Load for CacheSerializer to use later CacheSecure.cacheStores[type] = jsonDecoded; - })); - - // printDebug('[CacheStorage] total time ${stopwatchTotal.elapsed} '); - } catch (error) { - printError(error, title: 'CacheStorage'); - } + // printDebug('[CacheStorage] total time ${stopwatchTotal.elapsed} '); + } catch (error) { + printError(error.toString(), title: 'CacheStorage|$type'); + } + })); // unlock redux_persist after cache loaded from sqflite return Uint8List(0); diff --git a/lib/global/storage/codec.dart b/lib/global/storage/codec.dart new file mode 100644 index 000000000..baee043db --- /dev/null +++ b/lib/global/storage/codec.dart @@ -0,0 +1,118 @@ +import 'dart:convert'; +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; +import 'package:encrypt/encrypt.dart'; +import 'package:meta/meta.dart'; +import 'package:sembast/src/api/v2/sembast.dart'; + +var _random = Random.secure(); + +/// Random bytes generator +Uint8List _randBytes(int length) { + return Uint8List.fromList( + List.generate(length, (i) => _random.nextInt(256))); +} + +/// Generate an encryption password based on a user input password +/// +/// It uses MD5 which generates a 16 bytes blob, size needed for Salsa20 +Uint8List _generateEncryptPassword(String password) { + var blob = Uint8List.fromList(md5.convert(utf8.encode(password)).bytes); + assert(blob.length == 16); + return blob; +} + +/// Salsa20 based encoder +class _EncryptEncoder extends Converter { + final Salsa20 salsa20; + + _EncryptEncoder(this.salsa20); + + @override + String convert(dynamic input) { + // Generate random initial value + final iv = _randBytes(8); + final ivEncoded = base64.encode(iv); + assert(ivEncoded.length == 12); + + // Encode the input value + final encoded = + Encrypter(salsa20).encrypt(json.encode(input), iv: IV(iv)).base64; + + // Prepend the initial value + return '$ivEncoded$encoded'; + } +} + +/// Salsa20 based decoder +class _EncryptDecoder extends Converter { + final Salsa20 salsa20; + + _EncryptDecoder(this.salsa20); + + @override + dynamic convert(String input) { + // Read the initial value that was prepended + assert(input.length >= 12); + final iv = base64.decode(input.substring(0, 12)); + + // Extract the real input + input = input.substring(12); + + // Decode the input + var decoded = json.decode(Encrypter(salsa20).decrypt64(input, iv: IV(iv))); + if (decoded is Map) { + return decoded.cast(); + } + return decoded; + } +} + +/// Salsa20 based Codec +class _EncryptCodec extends Codec { + _EncryptEncoder _encoder; + _EncryptDecoder _decoder; + + _EncryptCodec(Uint8List passwordBytes) { + var salsa20 = Salsa20(Key(passwordBytes)); + _encoder = _EncryptEncoder(salsa20); + _decoder = _EncryptDecoder(salsa20); + } + + @override + Converter get decoder => _decoder; + + @override + Converter get encoder => _encoder; +} + +/// Our plain text signature +const _encryptCodecSignature = 'encrypt'; + +/// Create a codec to use to open a database with encrypted stored data. +/// +/// Hash (md5) of the password is used (but never stored) as a key to encrypt +/// the data using the Salsa20 algorithm with a random (8 bytes) initial value +/// +/// This is just used as a demonstration and should not be considered as a +/// reference since its implementation (and storage format) might change. +/// +/// No performance metrics has been made to check whether this is a viable +/// solution for big databases. +/// +/// The usage is then +/// +/// ```dart +/// // Initialize the encryption codec with a user password +/// var codec = getEncryptSembastCodec(password: '[your_user_password]'); +/// // Open the database with the codec +/// Database db = await factory.openDatabase(dbPath, codec: codec); +/// +/// // ...your database is ready to use +/// ``` +SembastCodec getEncryptSembastCodec({@required String password}) => + SembastCodec( + signature: _encryptCodecSignature, + codec: _EncryptCodec(_generateEncryptPassword(password))); diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index 53de8ed11..1933fa473 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -1,31 +1,29 @@ import 'dart:io'; +import 'package:meta/meta.dart'; import 'package:flutter/material.dart'; import 'package:sembast/sembast.dart'; import 'package:sembast_sqflite/sembast_sqflite.dart'; +import 'package:syphon/global/storage/codec.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite/sqflite.dart' as sqflite; import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; -import 'package:sqflite_sqlcipher/sqflite.dart' as sqflite_sqlcipher; import 'package:syphon/store/user/storage.dart'; class StorageSecure { // cold storage references static Database storageMain; - static sqflite_sqlcipher.Database storageMainEncrypted; // preloaded cold storage data static Map storageData = {}; // storage identifiers - static const storageKeyMain = '${Values.appNameLabel}-main-storage'; + static const storageKeyMain = '${Values.appNameLabel}-main-storage.db'; } Future initStorage() async { try { - var storageFactory; - - var storagePath = '${StorageSecure.storageKeyMain}.db'; + DatabaseFactory storageFactory; if (Platform.isAndroid || Platform.isIOS) { // always open cold storage as sqflite @@ -47,8 +45,13 @@ Future initStorage() async { ); } + var codec = getEncryptSembastCodec(password: 'testing123'); + + await storageFactory.deleteDatabase(StorageSecure.storageKeyMain); + StorageSecure.storageMain = await storageFactory.openDatabase( - storagePath, + StorageSecure.storageKeyMain, + codec: codec, ); } catch (error) { debugPrint('[initCache] ${error}'); diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index d0c476adb..2604d1c12 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -360,8 +360,8 @@ class Room { Map users = this.users ?? Map(); Set userIds = Set.from(this.userIds) ?? Set(); - try { - events.forEach((event) { + events.forEach((event) { + try { final timestamp = event.timestamp ?? 0; lastUpdate = timestamp > lastUpdate ? event.timestamp : lastUpdate; @@ -423,10 +423,10 @@ class Room { default: break; } - }); - } catch (error) { - debugPrint('[Room.fromStateEvents] ${error}'); - } + } catch (error) { + debugPrint('[Room.fromStateEvents] ${error} ${event.type}'); + } + }); try { // checks to make sure someone didn't name the room after the authed user From 8eb88ff5a50a5fbf9e9224deacbc09c2f75eec58 Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 6 Dec 2020 22:00:18 -0500 Subject: [PATCH 16/25] testing rooms within cold storage only until messages have been extracted from rooms --- lib/global/cache/serializer.dart | 10 ++--- lib/global/cache/storage.dart | 3 -- lib/global/storage/index.dart | 19 ++++++---- lib/store/rooms/actions.dart | 9 ++--- lib/store/rooms/room/model.dart | 4 +- lib/store/rooms/state.dart | 25 ++----------- lib/store/rooms/storage.dart | 63 ++++++++++++++++++++++++++++++++ lib/store/sync/actions.dart | 2 +- lib/store/user/storage.dart | 24 ------------ 9 files changed, 91 insertions(+), 68 deletions(-) create mode 100644 lib/store/rooms/storage.dart diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index 15e8c5af7..779d54a08 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -35,7 +35,6 @@ class CacheSerializer implements StateSerializer { state.authStore, state.syncStore, state.cryptoStore, - state.roomStore, state.mediaStore, state.settingsStore, ]; @@ -128,7 +127,6 @@ class CacheSerializer implements StateSerializer { SyncStore syncStore = SyncStore(); CryptoStore cryptoStore = CryptoStore(); MediaStore mediaStore = MediaStore(); - RoomStore roomStore = RoomStore(); SettingsStore settingsStore = SettingsStore(); // Load stores previously fetched from cache, @@ -155,12 +153,10 @@ class CacheSerializer implements StateSerializer { case 'MediaStore': mediaStore = MediaStore.fromJson(store); break; - case 'RoomStore': - roomStore = RoomStore.fromJson(store); - break; case 'SettingsStore': settingsStore = SettingsStore.fromJson(store); break; + case 'RoomStore': case 'UserStore': // --- cold storage only --- default: @@ -176,9 +172,11 @@ class CacheSerializer implements StateSerializer { authStore: authStore ?? AuthStore(), syncStore: syncStore ?? SyncStore(), cryptoStore: cryptoStore ?? CryptoStore(), - roomStore: roomStore ?? RoomStore(), mediaStore: mediaStore ?? MediaStore(), settingsStore: settingsStore ?? SettingsStore(), + roomStore: RoomStore().copyWith( + rooms: StorageSecure.storageData['rooms'] ?? {}, + ), userStore: UserStore().copyWith( users: StorageSecure.storageData['users'] ?? {}, ), diff --git a/lib/global/cache/storage.dart b/lib/global/cache/storage.dart index 379c48e9e..9504aa043 100644 --- a/lib/global/cache/storage.dart +++ b/lib/global/cache/storage.dart @@ -13,16 +13,13 @@ import 'package:syphon/store/media/state.dart'; import 'package:syphon/store/rooms/state.dart'; import 'package:syphon/store/settings/state.dart'; import 'package:syphon/store/sync/state.dart'; -import 'package:syphon/store/user/state.dart'; final List stores = [ AuthStore(), SyncStore(), MediaStore(), - RoomStore(), CryptoStore(), SettingsStore(), - UserStore(), ]; class CacheStorage implements StorageEngine { diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index 1933fa473..9c252953c 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -4,10 +4,12 @@ import 'package:meta/meta.dart'; import 'package:flutter/material.dart'; import 'package:sembast/sembast.dart'; import 'package:sembast_sqflite/sembast_sqflite.dart'; +import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/storage/codec.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite/sqflite.dart' as sqflite; import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; +import 'package:syphon/store/rooms/storage.dart'; import 'package:syphon/store/user/storage.dart'; class StorageSecure { @@ -45,23 +47,26 @@ Future initStorage() async { ); } - var codec = getEncryptSembastCodec(password: 'testing123'); - - await storageFactory.deleteDatabase(StorageSecure.storageKeyMain); + final codec = getEncryptSembastCodec(password: CacheSecure.cryptKey); StorageSecure.storageMain = await storageFactory.openDatabase( StorageSecure.storageKeyMain, codec: codec, ); } catch (error) { - debugPrint('[initCache] ${error}'); + debugPrint('[initStorage] $error'); } } Future loadStorage() async { - StorageSecure.storageData['users'] = await loadUsers( - storage: StorageSecure.storageMain, - ); + StorageSecure.storageData = { + 'users': await loadUsers( + storage: StorageSecure.storageMain, + ), + 'rooms': await loadRooms( + storage: StorageSecure.storageMain, + ) + }; } // // Closes and saves storage diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index 389806d64..a51e48420 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -23,6 +23,7 @@ import 'package:syphon/store/index.dart'; import 'package:syphon/store/media/actions.dart'; import 'package:syphon/store/rooms/events/actions.dart'; import 'package:syphon/store/rooms/events/selectors.dart'; +import 'package:syphon/store/rooms/storage.dart'; import 'package:syphon/store/sync/actions.dart'; import 'package:syphon/store/user/storage.dart'; import 'package:syphon/store/user/model.dart'; @@ -162,11 +163,9 @@ ThunkAction syncRooms(Map roomData) { lastSince: lastSince, ); - // -- COLD STORAGE -- - await saveUsers( - room.users, - storage: StorageSecure.storageMain, - ); + // save cold storage objects + saveUsers(room.users, storage: StorageSecure.storageMain); + saveRooms({room.id: room}, storage: StorageSecure.storageMain); // fetch avatar if a uri was found if (room.avatarUri != null) { diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index 2604d1c12..a0fd0956b 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -58,6 +58,8 @@ class Room { final List messages; final List outbox; + // TODO: offload messageReads, for large rooms these are ridiculously large + @JsonKey(ignore: true) final Map messageReads; @JsonKey(ignore: true) @@ -358,7 +360,7 @@ class Room { int lastUpdate = this.lastUpdate; int namePriority = this.namePriority != 4 ? this.namePriority : 4; Map users = this.users ?? Map(); - Set userIds = Set.from(this.userIds) ?? Set(); + Set userIds = Set.from(this.userIds ?? []); events.forEach((event) { try { diff --git a/lib/store/rooms/state.dart b/lib/store/rooms/state.dart index e963b03cb..9701074ff 100644 --- a/lib/store/rooms/state.dart +++ b/lib/store/rooms/state.dart @@ -11,62 +11,45 @@ part 'state.g.dart'; @JsonSerializable() class RoomStore extends Equatable { - final bool synced; - final int lastUpdate; // Last timestamp for actual new info final Map rooms; - // consider renaming to nextBatch - // Since we last checked for new info - final String lastSince; - @JsonKey(ignore: true) - final Map archive; // TODO: actually archive + final List archive; // TODO: archive by ID @JsonKey(ignore: true) - final List roomsHidden; + final List roomsHidden; // TODO: hidden by ID @JsonKey(ignore: true) final bool loading; - bool get isSynced => lastUpdate != null && lastUpdate != 0; + @JsonKey(ignore: true) List get roomList => rooms != null ? List.from(rooms.values) : []; const RoomStore({ this.rooms = const {}, - this.archive = const {}, - this.synced = false, + this.archive = const [], this.loading = false, - this.lastUpdate = 0, - this.lastSince, this.roomsHidden = const [], }); @override List get props => [ rooms, - synced, archive, - lastUpdate, - lastSince, roomsHidden, ]; RoomStore copyWith({ rooms, - synced, archive, loading, - lastUpdate, lastSince, roomsHidden, }) => RoomStore( rooms: rooms ?? this.rooms, - synced: synced ?? this.synced, archive: archive ?? this.archive, loading: loading ?? this.loading, - lastUpdate: lastUpdate ?? this.lastUpdate, - lastSince: lastSince ?? this.lastSince, roomsHidden: roomsHidden ?? this.roomsHidden, ); diff --git a/lib/store/rooms/storage.dart b/lib/store/rooms/storage.dart new file mode 100644 index 000000000..e22bb3fdc --- /dev/null +++ b/lib/store/rooms/storage.dart @@ -0,0 +1,63 @@ +import 'dart:convert'; + +import 'package:sembast/sembast.dart'; +import 'package:syphon/global/print.dart'; +import 'package:syphon/store/rooms/room/model.dart'; + +Future saveRooms( + Map rooms, { + Database cache, + Database storage, +}) async { + final store = StoreRef('rooms'); + + await storage.transaction((txn) async { + for (Room room in rooms.values) { + final record = store.record(room.id); + await record.put(txn, jsonEncode(room)); + } + }); +} + +Future> loadRooms({ + Database cache, + Database storage, + int offset = 0, +}) async { + final Map rooms = {}; + + try { + const limit = 10; + final store = StoreRef('rooms'); + final count = await store.count(storage); + + final finder = Finder( + limit: limit, + offset: offset, + ); + + final usersPaginated = await store.find( + storage, + finder: finder, + ); + + if (usersPaginated.isEmpty) { + return rooms; + } + + for (RecordSnapshot record in usersPaginated) { + rooms[record.key] = Room.fromJson(json.decode(record.value)); + } + + if (offset < count) { + rooms.addAll(await loadRooms( + offset: offset + limit, + storage: storage, + )); + } + } catch (error) { + printDebug(error.toString()); + } + printDebug('[rooms] loaded ${rooms.length.toString()}'); + return rooms; +} diff --git a/lib/store/sync/actions.dart b/lib/store/sync/actions.dart index 4ddc04dfa..0e3b60cc8 100644 --- a/lib/store/sync/actions.dart +++ b/lib/store/sync/actions.dart @@ -205,7 +205,7 @@ ThunkAction fetchSync({String since, bool forceFull = false}) { 'homeserver': store.state.authStore.user.homeserver, 'accessToken': store.state.authStore.user.accessToken, 'fullState': forceFull || store.state.roomStore.rooms == null, - 'since': forceFull ? null : since ?? store.state.roomStore.lastSince, + 'since': forceFull ? null : since ?? store.state.syncStore.lastSince, 'filter': filterId, 'timeout': 10000 }); diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index b994ee3d9..a1064e110 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -2,7 +2,6 @@ import 'dart:convert'; import 'package:sembast/sembast.dart'; import 'package:syphon/global/print.dart'; -import 'package:syphon/global/storage/index.dart'; import 'package:syphon/store/user/model.dart'; Future saveUsers( @@ -18,29 +17,6 @@ Future saveUsers( await record.put(txn, jsonEncode(user)); } }); - - return Future.value(); -} - -Future loadUser( - String userId, { - Database cache, - Database storage, -}) async { - try { - final store = StoreRef('users'); - final user = await store.findKey( - storage, - finder: Finder( - filter: Filter.byKey(userId), - ), - ); - - return jsonDecode(user); - } catch (error) { - printDebug(error); - } - return null; } Future> loadUsers({ From 394ad57b624d34a0951ad2829aa1d856980a2317 Mon Sep 17 00:00:00 2001 From: ereio Date: Mon, 7 Dec 2020 06:34:44 -0500 Subject: [PATCH 17/25] drafting events store and cold storage --- lib/global/cache/index.dart | 1 + lib/global/libs/matrix/events.dart | 2 +- lib/global/libs/matrix/user.dart | 2 +- lib/main.dart | 7 +- lib/store/crypto/actions.dart | 2 +- lib/store/crypto/events/actions.dart | 2 +- lib/store/{rooms => }/events/actions.dart | 40 +++++++++-- .../events/ephemeral/m.read/model.dart | 0 lib/store/{rooms => }/events/model.dart | 0 lib/store/{rooms => }/events/parsers.dart | 2 +- lib/store/events/reducer.dart | 26 +++++++ lib/store/{rooms => }/events/selectors.dart | 2 +- lib/store/events/state.dart | 40 +++++++++++ lib/store/events/storage.dart | 68 +++++++++++++++++++ lib/store/index.dart | 6 ++ lib/store/rooms/actions.dart | 8 ++- lib/store/rooms/reducer.dart | 2 +- lib/store/rooms/room/model.dart | 4 +- lib/store/rooms/room/selectors.dart | 4 +- lib/store/rooms/storage.dart | 8 +-- lib/store/user/actions.dart | 2 +- lib/store/user/storage.dart | 14 ++-- lib/views/home/chat/chat-input.dart | 2 +- lib/views/home/chat/details-chat.dart | 4 +- lib/views/home/chat/details-message.dart | 4 +- lib/views/home/chat/index.dart | 6 +- lib/views/home/index.dart | 4 +- .../appbars/appbar-options-message.dart | 4 +- lib/views/widgets/messages/message.dart | 2 +- 29 files changed, 223 insertions(+), 45 deletions(-) rename lib/store/{rooms => }/events/actions.dart (92%) rename lib/store/{rooms => }/events/ephemeral/m.read/model.dart (100%) rename lib/store/{rooms => }/events/model.dart (100%) rename lib/store/{rooms => }/events/parsers.dart (94%) create mode 100644 lib/store/events/reducer.dart rename lib/store/{rooms => }/events/selectors.dart (93%) create mode 100644 lib/store/events/state.dart create mode 100644 lib/store/events/storage.dart diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index 82fdcefe1..12f794645 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -51,6 +51,7 @@ Future initCache() async { CacheSecure.ivKey = await unlockIVKey(); CacheSecure.ivKeyNext = await unlockIVKeyNext(); CacheSecure.cryptKey = await unlockCryptKey(); + try { var cachePath = '${CacheSecure.cacheKeyMain}.db'; var cacheFactory; diff --git a/lib/global/libs/matrix/events.dart b/lib/global/libs/matrix/events.dart index e142ff632..78bb8f0b5 100644 --- a/lib/global/libs/matrix/events.dart +++ b/lib/global/libs/matrix/events.dart @@ -8,7 +8,7 @@ import 'package:syphon/global/algos.dart'; // Project imports: import 'package:syphon/global/libs/matrix/encryption.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; abstract class Events { /** diff --git a/lib/global/libs/matrix/user.dart b/lib/global/libs/matrix/user.dart index 82fa39bc0..ed645ee60 100644 --- a/lib/global/libs/matrix/user.dart +++ b/lib/global/libs/matrix/user.dart @@ -6,7 +6,7 @@ import 'dart:convert'; import 'package:http/http.dart' as http; // Project imports: -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; abstract class Users { /** diff --git a/lib/main.dart b/lib/main.dart index 993a5c73b..8303093fb 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -12,6 +12,7 @@ import 'package:flutter/scheduler.dart'; import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:flutter_redux/flutter_redux.dart'; import 'package:redux/redux.dart'; +import 'package:sembast/sembast.dart'; import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/formatters.dart'; import 'package:syphon/global/print.dart'; @@ -72,8 +73,12 @@ void main() async { } class Syphon extends StatefulWidget { + final Database cache; + final Database storage; final Store store; - const Syphon({Key key, this.store}) : super(key: key); + + const Syphon({Key key, this.store, this.cache, this.storage}) + : super(key: key); @override SyphonState createState() => SyphonState(store: store); diff --git a/lib/store/crypto/actions.dart b/lib/store/crypto/actions.dart index da996461c..ab9fc4534 100644 --- a/lib/store/crypto/actions.dart +++ b/lib/store/crypto/actions.dart @@ -24,7 +24,7 @@ import 'package:syphon/store/crypto/events/actions.dart'; import 'package:syphon/store/crypto/keys/model.dart'; import 'package:syphon/store/crypto/model.dart'; import 'package:syphon/store/index.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/store/rooms/room/model.dart'; import 'package:syphon/store/user/model.dart'; diff --git a/lib/store/crypto/events/actions.dart b/lib/store/crypto/events/actions.dart index 099539742..61ea2f3c3 100644 --- a/lib/store/crypto/events/actions.dart +++ b/lib/store/crypto/events/actions.dart @@ -17,7 +17,7 @@ import 'package:syphon/store/alerts/actions.dart'; import 'package:syphon/store/crypto/actions.dart'; import 'package:syphon/store/crypto/model.dart'; import 'package:syphon/store/index.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; /** * Encrypt event content with loaded outbound session for room diff --git a/lib/store/rooms/events/actions.dart b/lib/store/events/actions.dart similarity index 92% rename from lib/store/rooms/events/actions.dart rename to lib/store/events/actions.dart index 31f1cbf4b..0c6781eba 100644 --- a/lib/store/rooms/events/actions.dart +++ b/lib/store/events/actions.dart @@ -12,27 +12,57 @@ import 'package:redux_thunk/redux_thunk.dart'; // Project imports: import 'package:syphon/global/libs/matrix/index.dart'; +import 'package:syphon/global/print.dart'; +import 'package:syphon/global/storage/index.dart'; import 'package:syphon/store/alerts/actions.dart'; import 'package:syphon/store/crypto/actions.dart'; import 'package:syphon/store/crypto/events/actions.dart'; +import 'package:syphon/store/events/storage.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/rooms/actions.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/store/rooms/room/model.dart'; final protocol = DotEnv().env['PROTOCOL']; +class ResetEvents {} + +class SetMessages { + final String roomId; + final List messages; + SetMessages({this.roomId, this.messages}); +} + +class SetState { + final String roomId; + final List states; + SetState({this.roomId, this.states}); +} + /** - * Load Message Events + * Init Messages * - * Pulls next message events from cold storage + * Pulls initial messages from storage or paginates through + * those existing in cold storage */ -ThunkAction loadMessageEvents({Room room}) { +ThunkAction loadMessageEvents({ + Room room, + int offset = 0, + int limit = 20, +}) { return (Store store) async { try { store.dispatch(UpdateRoom(id: room.id, syncing: true)); + + final messages = await loadMessages( + storage: StorageSecure.storageMain, + offset: offset, + limit: !room.encryptionEnabled ? limit : null, + ); + + store.dispatch(SetMessages(roomId: room.id, messages: messages.values)); } catch (error) { - debugPrint('[fetchMessageEvents] $error'); + printDebug('[fetchMessageEvents] $error'); } finally { store.dispatch(UpdateRoom(id: room.id, syncing: false)); } diff --git a/lib/store/rooms/events/ephemeral/m.read/model.dart b/lib/store/events/ephemeral/m.read/model.dart similarity index 100% rename from lib/store/rooms/events/ephemeral/m.read/model.dart rename to lib/store/events/ephemeral/m.read/model.dart diff --git a/lib/store/rooms/events/model.dart b/lib/store/events/model.dart similarity index 100% rename from lib/store/rooms/events/model.dart rename to lib/store/events/model.dart diff --git a/lib/store/rooms/events/parsers.dart b/lib/store/events/parsers.dart similarity index 94% rename from lib/store/rooms/events/parsers.dart rename to lib/store/events/parsers.dart index a78f4331b..738720c1f 100644 --- a/lib/store/rooms/events/parsers.dart +++ b/lib/store/events/parsers.dart @@ -1,5 +1,5 @@ import 'package:sembast/sembast.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/store/user/model.dart'; Future> parseStateEvents( diff --git a/lib/store/events/reducer.dart b/lib/store/events/reducer.dart new file mode 100644 index 000000000..bcb39326c --- /dev/null +++ b/lib/store/events/reducer.dart @@ -0,0 +1,26 @@ +// Project imports: +import './actions.dart'; +import '../events/model.dart'; +import './state.dart'; + +EventStore eventReducer( + [EventStore state = const EventStore(), dynamic action]) { + switch (action.runtimeType) { + case SetMessages: + final roomId = action.roomId; + final messages = Map>.from(state.messages); + messages[roomId] = action.messages; + return state.copyWith(messages: messages); + + case SetState: + final roomId = action.roomId; + final states = Map>.from(state.states); + states[roomId] = action.state; + return state.copyWith(states: states); + + case ResetEvents: + return EventStore(); + default: + return state; + } +} diff --git a/lib/store/rooms/events/selectors.dart b/lib/store/events/selectors.dart similarity index 93% rename from lib/store/rooms/events/selectors.dart rename to lib/store/events/selectors.dart index d58533a8d..3e58bcb15 100644 --- a/lib/store/rooms/events/selectors.dart +++ b/lib/store/events/selectors.dart @@ -1,5 +1,5 @@ // Project imports: -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/store/rooms/room/model.dart'; List latestMessages(List messages) { diff --git a/lib/store/events/state.dart b/lib/store/events/state.dart new file mode 100644 index 000000000..016d29a1a --- /dev/null +++ b/lib/store/events/state.dart @@ -0,0 +1,40 @@ +// Dart imports: +import 'dart:async'; + +// Package imports: +import 'package:equatable/equatable.dart'; +import 'package:json_annotation/json_annotation.dart'; + +import 'model.dart'; + +part 'state.g.dart'; + +@JsonSerializable() +class EventStore extends Equatable { + final Map> states; // indexed by roomId + final Map> messages; // indexed by roomId + + const EventStore({ + this.states = const {}, + this.messages = const {}, + }); + + @override + List get props => [ + states, + messages, + ]; + + EventStore copyWith({ + states, + messages, + }) => + EventStore( + states: states ?? this.states, + messages: messages ?? this.messages, + ); + + Map toJson() => _$EventStoreToJson(this); + factory EventStore.fromJson(Map json) => + _$EventStoreFromJson(json); +} diff --git a/lib/store/events/storage.dart b/lib/store/events/storage.dart new file mode 100644 index 000000000..4fea36885 --- /dev/null +++ b/lib/store/events/storage.dart @@ -0,0 +1,68 @@ +import 'dart:convert'; + +import 'package:sembast/sembast.dart'; +import 'package:syphon/global/print.dart'; +import 'package:syphon/store/events/model.dart'; + +const String MESSAGES = 'messages'; + +Future saveMessages( + List messages, { + Database cache, + Database storage, +}) async { + final store = StoreRef(MESSAGES); + + await storage.transaction((txn) async { + for (Message message in messages) { + final record = store.record(message.id); + await record.put(txn, json.encode(message)); + } + }); +} + +Future> loadMessages({ + List eventIds, + bool encrypted, + Database cache, + Database storage, + int offset = 0, + int page = 20, // default amount loaded + int limit, +}) async { + final Map messages = {}; + + try { + final store = StoreRef(MESSAGES); + final count = limit ?? await store.count(storage); + + final finder = Finder( + limit: page, + offset: offset, + ); + + final messagesPaginated = await store.find( + storage, + finder: finder, + ); + + if (messagesPaginated.isEmpty) { + return messages; + } + + for (RecordSnapshot record in messagesPaginated) { + messages[record.key] = Message.fromJson(json.decode(record.value)); + } + + if (offset < count) { + messages.addAll(await loadMessages( + offset: offset + limit, + storage: storage, + )); + } + } catch (error) { + printDebug(error.toString()); + } + printDebug('[messages] loaded ${messages.length.toString()}'); + return messages; +} diff --git a/lib/store/index.dart b/lib/store/index.dart index 0399d588b..f50414bc8 100644 --- a/lib/store/index.dart +++ b/lib/store/index.dart @@ -16,6 +16,8 @@ import 'package:syphon/store/auth/reducer.dart'; import 'package:syphon/store/crypto/actions.dart'; import 'package:syphon/store/crypto/reducer.dart'; import 'package:syphon/store/crypto/state.dart'; +import 'package:syphon/store/events/reducer.dart'; +import 'package:syphon/store/events/state.dart'; import 'package:syphon/store/media/reducer.dart'; import 'package:syphon/global/cache/serializer.dart'; import 'package:syphon/store/sync/actions.dart'; @@ -42,6 +44,7 @@ class AppState extends Equatable { final MediaStore mediaStore; final SettingsStore settingsStore; final RoomStore roomStore; + final EventStore eventStore; final UserStore userStore; final SyncStore syncStore; final CryptoStore cryptoStore; @@ -52,6 +55,7 @@ class AppState extends Equatable { this.alertsStore = const AlertsStore(), this.syncStore = const SyncStore(), this.roomStore = const RoomStore(), + this.eventStore = const EventStore(), this.userStore = const UserStore(), this.mediaStore = const MediaStore(), this.searchStore = const SearchStore(), @@ -68,6 +72,7 @@ class AppState extends Equatable { roomStore, userStore, mediaStore, + eventStore, searchStore, settingsStore, cryptoStore, @@ -80,6 +85,7 @@ AppState appReducer(AppState state, action) => AppState( alertsStore: alertsReducer(state.alertsStore, action), mediaStore: mediaReducer(state.mediaStore, action), roomStore: roomReducer(state.roomStore, action), + eventStore: eventReducer(state.eventStore, action), syncStore: syncReducer(state.syncStore, action), userStore: userReducer(state.userStore, action), searchStore: searchReducer(state.searchStore, action), diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index a51e48420..1a26c895b 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -11,6 +11,7 @@ import 'package:redux/redux.dart'; import 'package:redux_thunk/redux_thunk.dart'; import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/storage/index.dart'; +import 'package:syphon/store/events/storage.dart'; // Project imports: import 'package:syphon/store/rooms/selectors.dart' as roomSelectors; @@ -21,13 +22,13 @@ import 'package:syphon/store/alerts/actions.dart'; import 'package:syphon/store/crypto/events/actions.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/media/actions.dart'; -import 'package:syphon/store/rooms/events/actions.dart'; -import 'package:syphon/store/rooms/events/selectors.dart'; +import 'package:syphon/store/events/actions.dart'; +import 'package:syphon/store/events/selectors.dart'; import 'package:syphon/store/rooms/storage.dart'; import 'package:syphon/store/sync/actions.dart'; import 'package:syphon/store/user/storage.dart'; import 'package:syphon/store/user/model.dart'; -import 'events/model.dart'; +import '../events/model.dart'; import 'room/model.dart'; final protocol = DotEnv().env['PROTOCOL']; @@ -166,6 +167,7 @@ ThunkAction syncRooms(Map roomData) { // save cold storage objects saveUsers(room.users, storage: StorageSecure.storageMain); saveRooms({room.id: room}, storage: StorageSecure.storageMain); + saveEvents(room.messages, storage: StorageSecure.storageMain); // fetch avatar if a uri was found if (room.avatarUri != null) { diff --git a/lib/store/rooms/reducer.dart b/lib/store/rooms/reducer.dart index 09c50383e..2a246e39c 100644 --- a/lib/store/rooms/reducer.dart +++ b/lib/store/rooms/reducer.dart @@ -1,6 +1,6 @@ // Project imports: import './actions.dart'; -import './events/model.dart'; +import '../events/model.dart'; import './room/model.dart'; import './state.dart'; diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index a0fd0956b..2282fb06a 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -7,8 +7,8 @@ import 'package:json_annotation/json_annotation.dart'; // Project imports: import 'package:syphon/global/strings.dart'; -import 'package:syphon/store/rooms/events/ephemeral/m.read/model.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/ephemeral/m.read/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/store/user/model.dart'; part 'model.g.dart'; diff --git a/lib/store/rooms/room/selectors.dart b/lib/store/rooms/room/selectors.dart index dbe9cfa28..4ba8a75cf 100644 --- a/lib/store/rooms/room/selectors.dart +++ b/lib/store/rooms/room/selectors.dart @@ -1,7 +1,7 @@ // Project imports: import 'package:syphon/global/strings.dart'; -import 'package:syphon/store/rooms/events/model.dart'; -import 'package:syphon/store/rooms/events/selectors.dart'; +import 'package:syphon/store/events/model.dart'; +import 'package:syphon/store/events/selectors.dart'; import 'package:syphon/store/rooms/room/model.dart'; List availableRooms(List rooms, {List hidden = const []}) { diff --git a/lib/store/rooms/storage.dart b/lib/store/rooms/storage.dart index e22bb3fdc..2ed6209c9 100644 --- a/lib/store/rooms/storage.dart +++ b/lib/store/rooms/storage.dart @@ -23,11 +23,11 @@ Future> loadRooms({ Database cache, Database storage, int offset = 0, + int limit = 10, }) async { final Map rooms = {}; try { - const limit = 10; final store = StoreRef('rooms'); final count = await store.count(storage); @@ -36,16 +36,16 @@ Future> loadRooms({ offset: offset, ); - final usersPaginated = await store.find( + final roomsPaginated = await store.find( storage, finder: finder, ); - if (usersPaginated.isEmpty) { + if (roomsPaginated.isEmpty) { return rooms; } - for (RecordSnapshot record in usersPaginated) { + for (RecordSnapshot record in roomsPaginated) { rooms[record.key] = Room.fromJson(json.decode(record.value)); } diff --git a/lib/store/user/actions.dart b/lib/store/user/actions.dart index 31711a197..6ebc17e64 100644 --- a/lib/store/user/actions.dart +++ b/lib/store/user/actions.dart @@ -5,7 +5,7 @@ import 'package:redux_thunk/redux_thunk.dart'; import 'package:syphon/global/libs/matrix/index.dart'; import 'package:syphon/store/alerts/actions.dart'; import 'package:syphon/store/index.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/store/rooms/room/model.dart'; import 'package:syphon/store/user/model.dart'; diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index a1064e110..4304071b0 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -23,11 +23,11 @@ Future> loadUsers({ Database cache, Database storage, int offset = 0, + int limit = 5000, }) async { - final Map userMap = {}; + final Map users = {}; try { - const limit = 5000; final store = StoreRef('users'); final count = await store.count(storage); @@ -42,15 +42,15 @@ Future> loadUsers({ ); if (usersPaginated.isEmpty) { - return userMap; + return users; } for (RecordSnapshot record in usersPaginated) { - userMap[record.key] = User.fromJson(json.decode(record.value)); + users[record.key] = User.fromJson(json.decode(record.value)); } if (offset < count) { - userMap.addAll(await loadUsers( + users.addAll(await loadUsers( offset: offset + limit, storage: storage, )); @@ -58,6 +58,6 @@ Future> loadUsers({ } catch (error) { printDebug(error.toString()); } - printDebug('[userMap] loaded ${userMap.length.toString()}'); - return userMap; + printDebug('[users] loaded ${users.length.toString()}'); + return users; } diff --git a/lib/views/home/chat/chat-input.dart b/lib/views/home/chat/chat-input.dart index 3cb31abb7..1142af0bb 100644 --- a/lib/views/home/chat/chat-input.dart +++ b/lib/views/home/chat/chat-input.dart @@ -7,7 +7,7 @@ import 'package:syphon/global/assets.dart'; // Project imports: import 'package:syphon/global/colours.dart'; import 'package:syphon/global/strings.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; class ChatInput extends StatelessWidget { final bool sendable; diff --git a/lib/views/home/chat/details-chat.dart b/lib/views/home/chat/details-chat.dart index 45a3b101b..f9ccae0c3 100644 --- a/lib/views/home/chat/details-chat.dart +++ b/lib/views/home/chat/details-chat.dart @@ -17,8 +17,8 @@ import 'package:syphon/global/colours.dart'; import 'package:syphon/global/dimensions.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/rooms/actions.dart'; -import 'package:syphon/store/rooms/events/model.dart'; -import 'package:syphon/store/rooms/events/selectors.dart'; +import 'package:syphon/store/events/model.dart'; +import 'package:syphon/store/events/selectors.dart'; import 'package:syphon/store/rooms/room/model.dart'; import 'package:syphon/store/rooms/selectors.dart' as roomSelectors; import 'package:syphon/store/settings/chat-settings/actions.dart'; diff --git a/lib/views/home/chat/details-message.dart b/lib/views/home/chat/details-message.dart index 6a94e4d7a..afe10ff8f 100644 --- a/lib/views/home/chat/details-message.dart +++ b/lib/views/home/chat/details-message.dart @@ -13,8 +13,8 @@ import 'package:redux/redux.dart'; import 'package:syphon/global/dimensions.dart'; import 'package:syphon/global/themes.dart'; import 'package:syphon/store/index.dart'; -import 'package:syphon/store/rooms/events/ephemeral/m.read/model.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/ephemeral/m.read/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/views/widgets/messages/message.dart'; final String debug = DotEnv().env['DEBUG']; diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index 06b3979b3..bba0d81a5 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -25,9 +25,9 @@ import 'package:syphon/global/themes.dart'; import 'package:syphon/store/crypto/actions.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/rooms/actions.dart'; -import 'package:syphon/store/rooms/events/actions.dart'; -import 'package:syphon/store/rooms/events/model.dart'; -import 'package:syphon/store/rooms/events/selectors.dart'; +import 'package:syphon/store/events/actions.dart'; +import 'package:syphon/store/events/model.dart'; +import 'package:syphon/store/events/selectors.dart'; import 'package:syphon/store/rooms/room/model.dart'; import 'package:syphon/store/rooms/selectors.dart' as roomSelectors; import 'package:syphon/store/user/model.dart'; diff --git a/lib/views/home/index.dart b/lib/views/home/index.dart index a02ec7f29..a0628fd66 100644 --- a/lib/views/home/index.dart +++ b/lib/views/home/index.dart @@ -11,7 +11,7 @@ import 'package:flutter_svg/flutter_svg.dart'; import 'package:redux/redux.dart'; import 'package:syphon/global/colours.dart'; import 'package:syphon/global/themes.dart'; -import 'package:syphon/store/rooms/events/selectors.dart'; +import 'package:syphon/store/events/selectors.dart'; import 'package:url_launcher/url_launcher.dart'; // Project imports: @@ -22,7 +22,7 @@ import 'package:syphon/global/strings.dart'; import 'package:syphon/global/values.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/rooms/actions.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/store/rooms/room/model.dart'; import 'package:syphon/store/rooms/room/selectors.dart'; import 'package:syphon/store/rooms/selectors.dart'; diff --git a/lib/views/widgets/appbars/appbar-options-message.dart b/lib/views/widgets/appbars/appbar-options-message.dart index 044f39a32..a11607841 100644 --- a/lib/views/widgets/appbars/appbar-options-message.dart +++ b/lib/views/widgets/appbars/appbar-options-message.dart @@ -4,8 +4,8 @@ import 'dart:async'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; -import 'package:syphon/store/rooms/events/model.dart'; -import 'package:syphon/store/rooms/events/selectors.dart'; +import 'package:syphon/store/events/model.dart'; +import 'package:syphon/store/events/selectors.dart'; import 'package:syphon/store/rooms/room/model.dart'; import 'package:syphon/views/home/chat/details-message.dart'; diff --git a/lib/views/widgets/messages/message.dart b/lib/views/widgets/messages/message.dart index a6e7ba0a1..d5f0220e3 100644 --- a/lib/views/widgets/messages/message.dart +++ b/lib/views/widgets/messages/message.dart @@ -9,7 +9,7 @@ import 'package:syphon/global/dimensions.dart'; import 'package:syphon/global/formatters.dart'; import 'package:syphon/global/strings.dart'; import 'package:syphon/global/themes.dart'; -import 'package:syphon/store/rooms/events/model.dart'; +import 'package:syphon/store/events/model.dart'; import 'package:syphon/views/widgets/avatars/avatar.dart'; class MessageWidget extends StatelessWidget { From e680b2e30f8012eef7d78683a4a00fb2b5b4ac8a Mon Sep 17 00:00:00 2001 From: ereio Date: Tue, 8 Dec 2020 23:30:25 -0500 Subject: [PATCH 18/25] cold storage for messages and other events is mostly finished, needs work on load/fetch differentiation --- lib/global/cache/index.dart | 38 +++++++-------- lib/global/cache/serializer.dart | 27 +++++++---- lib/global/cache/storage.dart | 17 +++---- lib/global/cache/threadables.dart | 4 +- lib/global/storage/codec.dart | 1 + lib/global/storage/index.dart | 64 ++++++++++++++++++-------- lib/main.dart | 37 ++++++++++----- lib/store/events/actions.dart | 19 +++++--- lib/store/events/selectors.dart | 14 +++++- lib/store/events/state.dart | 4 ++ lib/store/events/storage.dart | 52 ++++++++------------- lib/store/index.dart | 14 ++++-- lib/store/rooms/actions.dart | 6 +-- lib/store/rooms/room/model.dart | 18 ++++++-- lib/store/rooms/storage.dart | 12 +++-- lib/store/sync/background/service.dart | 28 +++++------ lib/store/user/storage.dart | 23 ++++++--- lib/views/home/chat/index.dart | 12 ++++- 18 files changed, 248 insertions(+), 142 deletions(-) diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index 12f794645..62f221fb4 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -12,7 +12,7 @@ import 'package:syphon/global/print.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; -class CacheSecure { +class Cache { // encryption references (in memory only) static String ivKey; static String ivKeyNext; @@ -46,20 +46,20 @@ class CacheSecure { * * (needs cold storage extracted as it's own entity) */ -Future initCache() async { +Future initCache() async { // Configure cache encryption/decryption instance - CacheSecure.ivKey = await unlockIVKey(); - CacheSecure.ivKeyNext = await unlockIVKeyNext(); - CacheSecure.cryptKey = await unlockCryptKey(); + Cache.ivKey = await unlockIVKey(); + Cache.ivKeyNext = await unlockIVKeyNext(); + Cache.cryptKey = await unlockCryptKey(); try { - var cachePath = '${CacheSecure.cacheKeyMain}.db'; + var cachePath = '${Cache.cacheKeyMain}.db'; var cacheFactory; if (Platform.isAndroid || Platform.isIOS) { var directory = await getApplicationDocumentsDirectory(); await directory.create(); - cachePath = join(directory.path, '${CacheSecure.cacheKeyMain}.db'); + cachePath = join(directory.path, '${Cache.cacheKeyMain}.db'); cacheFactory = databaseFactoryIo; } @@ -77,19 +77,21 @@ Future initCache() async { ); } - CacheSecure.cacheMain = await cacheFactory.openDatabase( + Cache.cacheMain = await cacheFactory.openDatabase( cachePath, ); - return Future.sync(() => null); + + return Cache.cacheMain; } catch (error) { printDebug('[initCache] ${error}'); + return null; } } // // Closes and saves storage -void closeCache() async { - if (CacheSecure.cacheMain != null) { - CacheSecure.cacheMain.close(); +void closeCache(Database cache) async { + if (cache != null) { + cache.close(); } } @@ -100,7 +102,7 @@ String createIVKey() { Future saveIVKey(String ivKey) async { // Check if storage has been created before return await FlutterSecureStorage().write( - key: CacheSecure.ivKeyLocation, + key: Cache.ivKeyLocation, value: ivKey, ); } @@ -108,7 +110,7 @@ Future saveIVKey(String ivKey) async { Future saveIVKeyNext(String ivKey) async { // Check if storage has been created before return await FlutterSecureStorage().write( - key: CacheSecure.ivKeyNextLocation, + key: Cache.ivKeyNextLocation, value: ivKey, ); } @@ -118,7 +120,7 @@ Future unlockIVKey() async { final storageEngine = FlutterSecureStorage(); final ivKeyStored = await storageEngine.read( - key: CacheSecure.ivKeyLocation, + key: Cache.ivKeyLocation, ); // Create a encryptionKey if a serialized one is not found @@ -130,7 +132,7 @@ Future unlockIVKeyNext() async { final storageEngine = FlutterSecureStorage(); final ivKeyStored = await storageEngine.read( - key: CacheSecure.ivKeyNextLocation, + key: Cache.ivKeyNextLocation, ); // Create a encryptionKey if a serialized one is not found @@ -145,7 +147,7 @@ Future unlockCryptKey() async { try { // Check if crypt key already exists cryptKey = await storageEngine.read( - key: CacheSecure.cryptKeyLocation, + key: Cache.cryptKeyLocation, ); } catch (error) { printDebug('[unlockCryptKey] ${error}'); @@ -156,7 +158,7 @@ Future unlockCryptKey() async { cryptKey = Key.fromSecureRandom(32).base64; await storageEngine.write( - key: CacheSecure.cryptKeyLocation, + key: Cache.cryptKeyLocation, value: cryptKey, ); } diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index 779d54a08..fc0376982 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -15,6 +15,8 @@ import 'package:syphon/global/storage/index.dart'; // Project imports: import 'package:syphon/store/crypto/state.dart'; +import 'package:syphon/store/events/model.dart'; +import 'package:syphon/store/events/state.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/sync/state.dart'; import 'package:syphon/store/user/state.dart'; @@ -29,6 +31,11 @@ import 'package:syphon/store/settings/state.dart'; * Handles serialization, encryption, and storage for caching redux stores */ class CacheSerializer implements StateSerializer { + final Database cache; + final Map> preloaded; + + CacheSerializer({this.cache, this.preloaded}); + @override Uint8List encode(AppState state) { final List stores = [ @@ -43,10 +50,10 @@ class CacheSerializer implements StateSerializer { // if the previously schedule task has not finished Future.microtask(() async { // // create a new IV for the encrypted cache - CacheSecure.ivKey = createIVKey(); + Cache.ivKey = createIVKey(); // // backup the IV in case the app is force closed before caching finishes - await saveIVKeyNext(CacheSecure.ivKey); + await saveIVKeyNext(Cache.ivKey); // run through all redux stores for encryption and encoding await Future.wait(stores.map((store) async { @@ -81,8 +88,8 @@ class CacheSerializer implements StateSerializer { jsonEncrypted = await compute( encryptJsonBackground, { - 'ivKey': CacheSecure.ivKey, - 'cryptKey': CacheSecure.cryptKey, + 'ivKey': Cache.ivKey, + 'cryptKey': Cache.cryptKey, 'type': type, 'json': jsonEncoded, }, @@ -95,7 +102,6 @@ class CacheSerializer implements StateSerializer { try { // Stopwatch stopwatchSave = new Stopwatch()..start(); - final cache = CacheSecure.cacheMain; final storeRef = StoreRef.main(); await storeRef.record(type).put(cache, jsonEncrypted); @@ -113,7 +119,7 @@ class CacheSerializer implements StateSerializer { })); // Rotate encryption for the next save - await saveIVKey(CacheSecure.ivKey); + await saveIVKey(Cache.ivKey); return Future.value(null); }); @@ -131,7 +137,7 @@ class CacheSerializer implements StateSerializer { // Load stores previously fetched from cache, // mutable global due to redux_presist not extendable beyond Uint8List - final stores = CacheSecure.cacheStores; + final stores = Cache.cacheStores; // decode each store cache synchronously stores.forEach((type, store) { @@ -175,10 +181,13 @@ class CacheSerializer implements StateSerializer { mediaStore: mediaStore ?? MediaStore(), settingsStore: settingsStore ?? SettingsStore(), roomStore: RoomStore().copyWith( - rooms: StorageSecure.storageData['rooms'] ?? {}, + rooms: preloaded['rooms'] ?? {}, ), userStore: UserStore().copyWith( - users: StorageSecure.storageData['users'] ?? {}, + users: preloaded['users'] ?? {}, + ), + eventStore: EventStore().copyWith( + messages: preloaded['messages'] ?? Map>(), ), ); } diff --git a/lib/global/cache/storage.dart b/lib/global/cache/storage.dart index 9504aa043..a8a4167fc 100644 --- a/lib/global/cache/storage.dart +++ b/lib/global/cache/storage.dart @@ -23,10 +23,12 @@ final List stores = [ ]; class CacheStorage implements StorageEngine { + final Database cache; + + CacheStorage({this.cache}); + @override Future load() async { - final cache = CacheSecure.cacheMain; - await Future.wait(stores.map((store) async { final type = store.runtimeType.toString(); try { @@ -43,9 +45,9 @@ class CacheStorage implements StorageEngine { final jsonDecoded = await compute( decryptJsonBackground, { - 'ivKey': CacheSecure.ivKey, - 'ivKeyNext': CacheSecure.ivKeyNext, - 'cryptKey': CacheSecure.cryptKey, + 'ivKey': Cache.ivKey, + 'ivKeyNext': Cache.ivKeyNext, + 'cryptKey': Cache.cryptKey, 'type': type, 'json': jsonEncrypted, }, @@ -55,7 +57,7 @@ class CacheStorage implements StorageEngine { // printDebug('[CacheStorage] decrypt ${stopwatchStore.elapsed}'); // Load for CacheSerializer to use later - CacheSecure.cacheStores[type] = jsonDecoded; + Cache.cacheStores[type] = jsonDecoded; // printDebug('[CacheStorage] total time ${stopwatchTotal.elapsed} '); } catch (error) { printError(error.toString(), title: 'CacheStorage|$type'); @@ -71,8 +73,7 @@ class CacheStorage implements StorageEngine { return null; } - static Future saveOffload(String jsonEncrypted, {String type}) async { - final cache = CacheSecure.cacheMain; + Future saveOffload(String jsonEncrypted, {String type}) async { final table = StoreRef.main(); final record = table.record(type); await record.put(cache, jsonEncrypted); diff --git a/lib/global/cache/threadables.dart b/lib/global/cache/threadables.dart index 2c3bb8d36..dac22bc35 100644 --- a/lib/global/cache/threadables.dart +++ b/lib/global/cache/threadables.dart @@ -82,10 +82,10 @@ Future serializeJsonBackground(Object store) async { final storageEngine = FlutterSecureStorage(); final ivKey = await storageEngine.read( - key: CacheSecure.ivKeyLocation, + key: Cache.ivKeyLocation, ); final cryptKey = await storageEngine.read( - key: CacheSecure.cryptKeyLocation, + key: Cache.cryptKeyLocation, ); final iv = IV.fromBase64(ivKey); diff --git a/lib/global/storage/codec.dart b/lib/global/storage/codec.dart index baee043db..40f0e4448 100644 --- a/lib/global/storage/codec.dart +++ b/lib/global/storage/codec.dart @@ -6,6 +6,7 @@ import 'package:crypto/crypto.dart'; import 'package:encrypt/encrypt.dart'; import 'package:meta/meta.dart'; import 'package:sembast/src/api/v2/sembast.dart'; +import 'package:syphon/global/print.dart'; var _random = Random.secure(); diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index 9c252953c..abd66ff48 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -9,21 +9,24 @@ import 'package:syphon/global/storage/codec.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite/sqflite.dart' as sqflite; import 'package:sqflite_common_ffi/sqflite_ffi.dart' as sqflite_ffi; +import 'package:syphon/store/events/model.dart'; +import 'package:syphon/store/events/storage.dart'; +import 'package:syphon/store/rooms/room/model.dart'; import 'package:syphon/store/rooms/storage.dart'; import 'package:syphon/store/user/storage.dart'; -class StorageSecure { +class Storage { // cold storage references - static Database storageMain; + static Database main; // preloaded cold storage data static Map storageData = {}; // storage identifiers - static const storageKeyMain = '${Values.appNameLabel}-main-storage.db'; + static const mainKey = '${Values.appNameLabel}-main-storage.db'; } -Future initStorage() async { +Future initStorage() async { try { DatabaseFactory storageFactory; @@ -47,31 +50,52 @@ Future initStorage() async { ); } - final codec = getEncryptSembastCodec(password: CacheSecure.cryptKey); + final codec = getEncryptSembastCodec(password: Cache.cryptKey); - StorageSecure.storageMain = await storageFactory.openDatabase( - StorageSecure.storageKeyMain, + // TODO: make actions have reference to the storage/cache through state + Storage.main = await storageFactory.openDatabase( + Storage.mainKey, codec: codec, ); + + return Storage.main; } catch (error) { debugPrint('[initStorage] $error'); + return null; } } -Future loadStorage() async { - StorageSecure.storageData = { - 'users': await loadUsers( - storage: StorageSecure.storageMain, - ), - 'rooms': await loadRooms( - storage: StorageSecure.storageMain, - ) - }; -} - // // Closes and saves storage void closeStorage() async { - if (StorageSecure.storageMain != null) { - StorageSecure.storageMain.close(); + if (Storage.main != null) { + Storage.main.close(); + } +} + +Future>> loadStorage(Database storage) async { + // load all rooms from cold storages + final rooms = await loadRooms( + storage: storage, + ); + + final users = await loadUsers( + storage: storage, + ); + + // load message using rooms loaded from cold storage + Map> messages = new Map(); + for (Room room in rooms.values) { + messages[room.id] = await loadMessages( + room.messageIds, + storage: storage, + encrypted: room.encryptionEnabled, + limit: 20, + ); } + + return { + 'users': users, + 'rooms': rooms, + 'messages': messages.isNotEmpty ? messages : null, + }; } diff --git a/lib/main.dart b/lib/main.dart index 8303093fb..35cb4744e 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -22,6 +22,7 @@ import 'package:syphon/global/storage/index.dart'; import 'package:syphon/global/themes.dart'; import 'package:syphon/store/alerts/actions.dart'; import 'package:syphon/store/auth/actions.dart'; +import 'package:syphon/store/events/actions.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/settings/state.dart'; import 'package:syphon/store/sync/actions.dart'; @@ -60,16 +61,16 @@ void main() async { } // init hot cache and cold storage - await initCache(); + final cache = await initCache(); // init cold storage and load to data - await initStorage(); + final storage = await initStorage(); - // actually load storage to memory (to rehydrate cache for now) - await loadStorage(); + // init redux store + final store = await initStore(cache, storage); // init hot cache and start - runApp(Syphon(store: await initStore())); + runApp(Syphon(store: store, cache: cache, storage: storage)); } class Syphon extends StatefulWidget { @@ -77,22 +78,36 @@ class Syphon extends StatefulWidget { final Database storage; final Store store; - const Syphon({Key key, this.store, this.cache, this.storage}) - : super(key: key); + const Syphon({ + Key key, + this.store, + this.cache, + this.storage, + }) : super(key: key); @override - SyphonState createState() => SyphonState(store: store); + SyphonState createState() => SyphonState( + store: store, + cache: cache, + storage: storage, + ); } class SyphonState extends State with WidgetsBindingObserver { - SyphonState({this.store}); - + final Database cache; + final Database storage; final Store store; final GlobalKey globalScaffold = GlobalKey(); Widget defaultHome = Home(); StreamSubscription alertsListener; + SyphonState({ + this.store, + this.cache, + this.storage, + }); + @override void initState() { WidgetsBinding.instance.addObserver(this); @@ -200,7 +215,7 @@ class SyphonState extends State with WidgetsBindingObserver { @override void deactivate() { - closeCache(); + closeCache(cache); WidgetsBinding.instance.removeObserver(this); store.dispatch(stopAuthObserver()); store.dispatch(stopAlertsObserver()); diff --git a/lib/store/events/actions.dart b/lib/store/events/actions.dart index 0c6781eba..0c37d0755 100644 --- a/lib/store/events/actions.dart +++ b/lib/store/events/actions.dart @@ -40,10 +40,12 @@ class SetState { } /** - * Init Messages + * Load Message Events * * Pulls initial messages from storage or paginates through - * those existing in cold storage + * those existing in cold storage depending on requests from client + * + * Make sure these have been exhausted before calling fetchMessageEvents */ ThunkAction loadMessageEvents({ Room room, @@ -54,13 +56,18 @@ ThunkAction loadMessageEvents({ try { store.dispatch(UpdateRoom(id: room.id, syncing: true)); - final messages = await loadMessages( - storage: StorageSecure.storageMain, - offset: offset, + printDebug('[loadMessageEvents]'); + final messagesStored = await loadMessages( + room.messageIds, + storage: Storage.main, + offset: offset, // offset from the most recent eventId found limit: !room.encryptionEnabled ? limit : null, ); - store.dispatch(SetMessages(roomId: room.id, messages: messages.values)); + store.dispatch(SetMessages( + roomId: room.id, + messages: room.messages + messagesStored, + )); } catch (error) { printDebug('[fetchMessageEvents] $error'); } finally { diff --git a/lib/store/events/selectors.dart b/lib/store/events/selectors.dart index 3e58bcb15..8ee86e0ed 100644 --- a/lib/store/events/selectors.dart +++ b/lib/store/events/selectors.dart @@ -1,7 +1,15 @@ // Project imports: import 'package:syphon/store/events/model.dart'; +import 'package:syphon/store/index.dart'; import 'package:syphon/store/rooms/room/model.dart'; +// TODO: replaces latestMessages() selectors with this +List latestRoomMessages(AppState state, String roomId) { + final messagesAll = state.eventStore.messages; + + return messagesAll[roomId] ?? []; +} + List latestMessages(List messages) { final sortedList = messages ?? []; @@ -24,8 +32,10 @@ List latestMessages(List messages) { return sortedList; } -List wrapOutboxMessages( - {List messages, List outbox}) { +List wrapOutboxMessages({ + List messages, + List outbox, +}) { return [outbox, messages].expand((x) => x).toList(); } diff --git a/lib/store/events/state.dart b/lib/store/events/state.dart index 016d29a1a..2a537d4ac 100644 --- a/lib/store/events/state.dart +++ b/lib/store/events/state.dart @@ -13,16 +13,19 @@ part 'state.g.dart'; class EventStore extends Equatable { final Map> states; // indexed by roomId final Map> messages; // indexed by roomId + final Map> receipts; const EventStore({ this.states = const {}, this.messages = const {}, + this.receipts = const {}, }); @override List get props => [ states, messages, + receipts, ]; EventStore copyWith({ @@ -32,6 +35,7 @@ class EventStore extends Equatable { EventStore( states: states ?? this.states, messages: messages ?? this.messages, + receipts: receipts ?? this.receipts, ); Map toJson() => _$EventStoreToJson(this); diff --git a/lib/store/events/storage.dart b/lib/store/events/storage.dart index 4fea36885..4771fddc8 100644 --- a/lib/store/events/storage.dart +++ b/lib/store/events/storage.dart @@ -8,7 +8,6 @@ const String MESSAGES = 'messages'; Future saveMessages( List messages, { - Database cache, Database storage, }) async { final store = StoreRef(MESSAGES); @@ -21,48 +20,37 @@ Future saveMessages( }); } -Future> loadMessages({ - List eventIds, - bool encrypted, - Database cache, +/** + * Load Messages (Cold Storage) + * + * In storage, messages are indexed by eventId + * In redux, they're indexed by RoomID and placed in a list + */ +Future> loadMessages( + List eventIds, { Database storage, + bool encrypted, int offset = 0, - int page = 20, // default amount loaded - int limit, + int limit = 20, // default amount loaded }) async { - final Map messages = {}; + final List messages = []; try { final store = StoreRef(MESSAGES); - final count = limit ?? await store.count(storage); - final finder = Finder( - limit: page, - offset: offset, - ); + final eventIdsPaginated = eventIds.skip(offset).take(limit).toList(); - final messagesPaginated = await store.find( - storage, - finder: finder, - ); + final messagesPaginated = + await store.records(eventIdsPaginated).get(storage); - if (messagesPaginated.isEmpty) { - return messages; + for (String message in messagesPaginated) { + messages.add(Message.fromJson(json.decode(message))); } - for (RecordSnapshot record in messagesPaginated) { - messages[record.key] = Message.fromJson(json.decode(record.value)); - } - - if (offset < count) { - messages.addAll(await loadMessages( - offset: offset + limit, - storage: storage, - )); - } + printDebug('[messages] loaded ${messages.length.toString()}'); + return messages; } catch (error) { - printDebug(error.toString()); + printDebug(error.toString(), title: 'loadMessages'); + return null; } - printDebug('[messages] loaded ${messages.length.toString()}'); - return messages; } diff --git a/lib/store/index.dart b/lib/store/index.dart index f50414bc8..5dced8344 100644 --- a/lib/store/index.dart +++ b/lib/store/index.dart @@ -7,7 +7,10 @@ import 'package:equatable/equatable.dart'; import 'package:redux/redux.dart'; import 'package:redux_persist/redux_persist.dart'; import 'package:redux_thunk/redux_thunk.dart'; +import 'package:sembast/sembast.dart'; +import 'package:syphon/global/algos.dart'; import 'package:syphon/global/cache/storage.dart'; +import 'package:syphon/global/storage/index.dart'; // Project imports: import 'package:syphon/store/alerts/model.dart'; @@ -97,11 +100,16 @@ AppState appReducer(AppState state, action) => AppState( * Initialize Store * - Hot redux state cache for top level data */ -Future initStore() async { +Future initStore(Database cache, Database storage) async { + // partially load storage to memory to rehydrate cache + final data = await loadStorage(storage); + + printJson(data); + // Configure redux persist instance final persistor = Persistor( - storage: CacheStorage(), - serializer: CacheSerializer(), + storage: CacheStorage(cache: cache), + serializer: CacheSerializer(cache: cache, preloaded: data), // TODO: can remove once cold storage is in place throttleDuration: Duration(milliseconds: 4500), shouldSave: (Store store, dynamic action) { diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index 1a26c895b..a45b19550 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -165,9 +165,9 @@ ThunkAction syncRooms(Map roomData) { ); // save cold storage objects - saveUsers(room.users, storage: StorageSecure.storageMain); - saveRooms({room.id: room}, storage: StorageSecure.storageMain); - saveEvents(room.messages, storage: StorageSecure.storageMain); + saveUsers(room.users, storage: Storage.main); + saveRooms({room.id: room}, storage: Storage.main); + saveMessages(room.messages, storage: Storage.main); // fetch avatar if a uri was found if (room.avatarUri != null) { diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index 2282fb06a..2ce824a02 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -52,11 +52,15 @@ class Room { // Associated user ids final List userIds; + final List messageIds; + final List outbox; // TODO: removed until state timeline work can be done - // final List state; + @JsonKey(ignore: true) + final List state; + + @JsonKey(ignore: true) final List messages; - final List outbox; // TODO: offload messageReads, for large rooms these are ridiculously large @JsonKey(ignore: true) @@ -109,6 +113,7 @@ class Room { this.userIds = const [], this.outbox = const [], this.messages = const [], + this.messageIds = const [], this.lastRead = 0, this.lastUpdate = 0, this.namePriority = 4, @@ -123,6 +128,7 @@ class Room { this.nextHash, this.prevHash, this.messageReads, + this.state, }); Room copyWith({ @@ -153,11 +159,12 @@ class Room { events, outbox, messages, + messageIds, messageReads, lastHash, prevHash, nextHash, - // state, + state, }) => Room( id: id ?? this.id, @@ -183,6 +190,7 @@ class Room { usersTyping: usersTyping ?? this.usersTyping, isDraftRoom: isDraftRoom ?? this.isDraftRoom, outbox: outbox ?? this.outbox, + messageIds: messageIds ?? this.messageIds, messages: messages ?? this.messages, users: users ?? this.users, userIds: userIds ?? this.userIds, @@ -190,7 +198,7 @@ class Room { lastHash: lastHash ?? this.lastHash, prevHash: prevHash ?? this.prevHash, nextHash: nextHash ?? this.nextHash, - // state: state ?? this.state, + state: state ?? this.state, ); Map toJson() => _$RoomToJson(this); @@ -550,12 +558,14 @@ class Room { ); // Filter to find startTime and endTime + final messageIds = List.from(messagesMap.keys); final messagesAll = List.from(messagesMap.values); // Save values to room return this.copyWith( outbox: outbox, messages: messagesAll, + messageIds: messageIds, limited: limited ?? this.limited, encryptionEnabled: this.encryptionEnabled || hasEncrypted != null, lastUpdate: lastUpdate ?? this.lastUpdate, diff --git a/lib/store/rooms/storage.dart b/lib/store/rooms/storage.dart index 2ed6209c9..a29d3236d 100644 --- a/lib/store/rooms/storage.dart +++ b/lib/store/rooms/storage.dart @@ -55,9 +55,15 @@ Future> loadRooms({ storage: storage, )); } + + if (rooms.isEmpty) { + return null; + } + + printDebug('[rooms] loaded ${rooms.length.toString()}'); + return rooms; } catch (error) { - printDebug(error.toString()); + printDebug(error.toString(), title: 'loadRooms'); + return null; } - printDebug('[rooms] loaded ${rooms.length.toString()}'); - return rooms; } diff --git a/lib/store/sync/background/service.dart b/lib/store/sync/background/service.dart index 07519bcfd..28d09fd1a 100644 --- a/lib/store/sync/background/service.dart +++ b/lib/store/sync/background/service.dart @@ -60,27 +60,27 @@ class BackgroundSync { await Future.wait([ secureStorage.write( - key: CacheSecure.protocolKey, + key: Cache.protocolKey, value: protocol, ), secureStorage.write( - key: CacheSecure.homeserverKey, + key: Cache.homeserverKey, value: homeserver, ), secureStorage.write( - key: CacheSecure.accessTokenKey, + key: Cache.accessTokenKey, value: accessToken, ), secureStorage.write( - key: CacheSecure.lastSinceKey, + key: Cache.lastSinceKey, value: lastSince, ), secureStorage.write( - key: CacheSecure.userIdKey, + key: Cache.userIdKey, value: currentUser, ), secureStorage.write( - key: CacheSecure.roomNamesKey, + key: Cache.roomNamesKey, value: jsonEncode(roomNames), ) ]); @@ -124,14 +124,14 @@ void notificationSyncIsolate() async { try { final secureStorage = FlutterSecureStorage(); - protocol = await secureStorage.read(key: CacheSecure.protocolKey); - homeserver = await secureStorage.read(key: CacheSecure.homeserverKey); - accessToken = await secureStorage.read(key: CacheSecure.accessTokenKey); - lastSince = await secureStorage.read(key: CacheSecure.lastSinceKey); - userId = await secureStorage.read(key: CacheSecure.userIdKey); + protocol = await secureStorage.read(key: Cache.protocolKey); + homeserver = await secureStorage.read(key: Cache.homeserverKey); + accessToken = await secureStorage.read(key: Cache.accessTokenKey); + lastSince = await secureStorage.read(key: Cache.lastSinceKey); + userId = await secureStorage.read(key: Cache.userIdKey); roomNames = jsonDecode( - await secureStorage.read(key: CacheSecure.roomNamesKey), + await secureStorage.read(key: Cache.roomNamesKey), ); } catch (error) { print('[notificationSyncIsolate] $error'); @@ -197,7 +197,7 @@ FutureOr syncLoop({ final secureStorage = FlutterSecureStorage(); lastSinceNew = await secureStorage.read( - key: CacheSecure.lastSinceKey, + key: Cache.lastSinceKey, ); } catch (error) { print('[syncLoop] $error'); @@ -225,7 +225,7 @@ FutureOr syncLoop({ final secureStorage = FlutterSecureStorage(); await secureStorage.write( - key: CacheSecure.lastSinceKey, + key: Cache.lastSinceKey, value: lastSinceNew, ); } catch (error) { diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index 4304071b0..fa80edf92 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -19,11 +19,16 @@ Future saveUsers( }); } +/** + * Load Users (Cold Storage) + * + * Example of useful recursion + */ Future> loadUsers({ Database cache, Database storage, int offset = 0, - int limit = 5000, + int page = 5000, }) async { final Map users = {}; @@ -32,7 +37,7 @@ Future> loadUsers({ final count = await store.count(storage); final finder = Finder( - limit: limit, + limit: page, offset: offset, ); @@ -51,13 +56,19 @@ Future> loadUsers({ if (offset < count) { users.addAll(await loadUsers( - offset: offset + limit, + offset: offset + page, storage: storage, )); } + + if (users.isEmpty) { + return null; + } + + printDebug('[users] loaded ${users.length}'); + return users; } catch (error) { - printDebug(error.toString()); + printDebug(error.toString(), title: 'loadUsers'); + return null; } - printDebug('[users] loaded ${users.length.toString()}'); - return users; } diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index bba0d81a5..f08209e70 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -815,8 +815,18 @@ class _Props extends Equatable { onLoadMoreMessages: () { final room = store.state.roomStore.rooms[roomId] ?? Room(); + // load message from cold storage + if (room.messages.length < room.messageIds.length) { + return store.dispatch( + loadMessageEvents( + room: room, + offset: room.messages.length, + ), + ); + } + // fetch messages beyond the oldest known message - lastHash - store.dispatch(fetchMessageEvents( + return store.dispatch(fetchMessageEvents( room: room, from: room.lastHash, oldest: true, From 807aa2d663f317e5ce32018bf2b5a388b1bd193c Mon Sep 17 00:00:00 2001 From: ereio Date: Tue, 8 Dec 2020 23:41:08 -0500 Subject: [PATCH 19/25] message previews appearing in home again with on init preloaded messages --- lib/store/events/selectors.dart | 8 +++----- lib/store/index.dart | 2 -- lib/views/home/chat/details-chat.dart | 4 +--- lib/views/home/index.dart | 7 +++++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/store/events/selectors.dart b/lib/store/events/selectors.dart index 8ee86e0ed..53a1325c7 100644 --- a/lib/store/events/selectors.dart +++ b/lib/store/events/selectors.dart @@ -5,16 +5,14 @@ import 'package:syphon/store/rooms/room/model.dart'; // TODO: replaces latestMessages() selectors with this List latestRoomMessages(AppState state, String roomId) { - final messagesAll = state.eventStore.messages; - - return messagesAll[roomId] ?? []; + return state.eventStore.messages[roomId] ?? []; } List latestMessages(List messages) { - final sortedList = messages ?? []; + final sortedList = List.from(messages ?? []); // sort descending - messages.sort((a, b) { + sortedList.sort((a, b) { if (a.pending && !b.pending) { return -1; } diff --git a/lib/store/index.dart b/lib/store/index.dart index 5dced8344..fd71dc500 100644 --- a/lib/store/index.dart +++ b/lib/store/index.dart @@ -104,8 +104,6 @@ Future initStore(Database cache, Database storage) async { // partially load storage to memory to rehydrate cache final data = await loadStorage(storage); - printJson(data); - // Configure redux persist instance final persistor = Persistor( storage: CacheStorage(cache: cache), diff --git a/lib/views/home/chat/details-chat.dart b/lib/views/home/chat/details-chat.dart index f9ccae0c3..7d215be4d 100644 --- a/lib/views/home/chat/details-chat.dart +++ b/lib/views/home/chat/details-chat.dart @@ -533,9 +533,7 @@ class _Props extends Equatable { userList: List.from( roomSelectors.room(id: roomId, state: store.state).users.values, ), - messages: latestMessages( - roomSelectors.room(id: roomId, state: store.state).messages, - ), + messages: latestRoomMessages(store.state, roomId), onLeaveChat: () async { await store.dispatch(removeRoom(room: Room(id: roomId))); }, diff --git a/lib/views/home/index.dart b/lib/views/home/index.dart index a0628fd66..0963b7905 100644 --- a/lib/views/home/index.dart +++ b/lib/views/home/index.dart @@ -282,8 +282,8 @@ class HomeViewState extends State { itemCount: rooms.length, itemBuilder: (BuildContext context, int index) { final room = rooms[index]; - final messages = room.messages ?? const []; - final messagesLatest = latestMessages(room.messages); + final messages = props.messages[room.id] ?? const []; + final messagesLatest = latestMessages(messages); final messagePreview = formatPreview( room: room, prefetched: messagesLatest, @@ -594,6 +594,7 @@ class _Props extends Equatable { final User currentUser; final ThemeType theme; final Map chatSettings; + final Map> messages; final Function onDebug; final Function onLeaveChat; @@ -609,6 +610,7 @@ class _Props extends Equatable { @required this.offline, @required this.syncing, @required this.unauthed, + @required this.messages, @required this.currentUser, @required this.chatSettings, @required this.roomTypeBadgesEnabled, @@ -638,6 +640,7 @@ class _Props extends Equatable { sortedPrioritizedRooms(store.state.roomStore.rooms), hidden: store.state.roomStore.roomsHidden, ), + messages: store.state.eventStore.messages, unauthed: store.state.syncStore.unauthed, offline: store.state.syncStore.offline, syncing: () { From 91785923c1f48917acbea2171365d684ba0796d4 Mon Sep 17 00:00:00 2001 From: ereio Date: Wed, 9 Dec 2020 22:28:06 -0500 Subject: [PATCH 20/25] bug fixes regarding cold storage save/load and handling message parsing --- lib/global/storage/index.dart | 28 +++++++++++++++ lib/store/auth/actions.dart | 6 ++++ lib/store/crypto/actions.dart | 5 +-- lib/store/events/actions.dart | 18 ++++++++-- lib/store/events/reducer.dart | 19 +++++++++- lib/store/events/selectors.dart | 4 +-- lib/store/events/storage.dart | 9 ++--- lib/store/rooms/actions.dart | 40 +++++++++++++++++----- lib/store/rooms/room/model.dart | 11 ++++-- lib/store/rooms/room/selectors.dart | 8 +---- lib/store/rooms/selectors.dart | 8 ++--- lib/store/rooms/storage.dart | 2 +- lib/store/user/actions.dart | 11 ++++++ lib/store/user/reducer.dart | 4 +++ lib/store/user/selectors.dart | 12 ++++++- lib/store/user/storage.dart | 2 +- lib/views/home/chat/details-all-users.dart | 5 ++- lib/views/home/chat/details-chat.dart | 7 ++-- lib/views/home/chat/index.dart | 11 +++--- lib/views/home/index.dart | 3 +- lib/views/home/settings/devices.dart | 19 +--------- 21 files changed, 163 insertions(+), 69 deletions(-) diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index abd66ff48..a0b134d73 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -5,6 +5,7 @@ import 'package:flutter/material.dart'; import 'package:sembast/sembast.dart'; import 'package:sembast_sqflite/sembast_sqflite.dart'; import 'package:syphon/global/cache/index.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/global/storage/codec.dart'; import 'package:syphon/global/values.dart'; import 'package:sqflite/sqflite.dart' as sqflite; @@ -72,6 +73,30 @@ void closeStorage() async { } } +Future deleteStorage() async { + try { + DatabaseFactory storageFactory; + + if (Platform.isAndroid || Platform.isIOS) { + storageFactory = getDatabaseFactorySqflite( + sqflite.databaseFactory, + ); + } + + if (Platform.isLinux || Platform.isWindows || Platform.isMacOS) { + storageFactory = getDatabaseFactorySqflite( + sqflite_ffi.databaseFactoryFfi, + ); + } + + Storage.main = await storageFactory.deleteDatabase( + Storage.mainKey, + ); + } catch (error) { + printDebug(error.toString()); + } +} + Future>> loadStorage(Database storage) async { // load all rooms from cold storages final rooms = await loadRooms( @@ -91,6 +116,9 @@ Future>> loadStorage(Database storage) async { encrypted: room.encryptionEnabled, limit: 20, ); + printDebug( + '[loadMessages] ${messages[room.id].length.toString()} ${room.name} loaded', + ); } return { diff --git a/lib/store/auth/actions.dart b/lib/store/auth/actions.dart index 603e3130e..96610f464 100644 --- a/lib/store/auth/actions.dart +++ b/lib/store/auth/actions.dart @@ -18,6 +18,7 @@ import 'package:syphon/global/libs/matrix/auth.dart'; import 'package:syphon/global/libs/matrix/errors.dart'; import 'package:syphon/global/libs/matrix/index.dart'; import 'package:syphon/global/notifications.dart'; +import 'package:syphon/global/storage/index.dart'; import 'package:syphon/global/strings.dart'; import 'package:syphon/global/values.dart'; import 'package:syphon/store/alerts/actions.dart'; @@ -366,6 +367,7 @@ ThunkAction logoutUser() { } final temp = '${store.state.authStore.user.accessToken}'; store.state.authStore.authObserver.add(null); + final data = await MatrixApi.logoutUser( protocol: protocol, homeserver: store.state.authStore.user.homeserver, @@ -380,6 +382,10 @@ ThunkAction logoutUser() { } } + // wipe cold storage + await deleteStorage(); + + // tell authObserver to wipe auth user store.state.authStore.authObserver.add(null); } catch (error) { store.dispatch(addAlert( diff --git a/lib/store/crypto/actions.dart b/lib/store/crypto/actions.dart index ab9fc4534..c5394bd61 100644 --- a/lib/store/crypto/actions.dart +++ b/lib/store/crypto/actions.dart @@ -1012,8 +1012,9 @@ ThunkAction exportMessageSession({String roomId}) { * fetches the keys uploaded to the matrix homeserver * by other users */ -ThunkAction fetchDeviceKeys( - {Map users, List userIds}) { +ThunkAction fetchDeviceKeys({ + List userIds, +}) { return (Store store) async { try { final Map userIdMap = Map.fromIterable( diff --git a/lib/store/events/actions.dart b/lib/store/events/actions.dart index 0c37d0755..b2def330a 100644 --- a/lib/store/events/actions.dart +++ b/lib/store/events/actions.dart @@ -9,6 +9,7 @@ import 'package:flutter/material.dart'; import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:redux/redux.dart'; import 'package:redux_thunk/redux_thunk.dart'; +import 'package:syphon/global/algos.dart'; // Project imports: import 'package:syphon/global/libs/matrix/index.dart'; @@ -39,6 +40,16 @@ class SetState { SetState({this.roomId, this.states}); } +ThunkAction setMessageEvents({ + Room room, + List messages, + int offset = 0, + int limit = 20, +}) => + (Store store) { + return store.dispatch(SetMessages(roomId: room.id, messages: messages)); + }; + /** * Load Message Events * @@ -56,17 +67,16 @@ ThunkAction loadMessageEvents({ try { store.dispatch(UpdateRoom(id: room.id, syncing: true)); - printDebug('[loadMessageEvents]'); final messagesStored = await loadMessages( room.messageIds, storage: Storage.main, offset: offset, // offset from the most recent eventId found - limit: !room.encryptionEnabled ? limit : null, + limit: !room.encryptionEnabled ? limit : room.messageIds.length, ); store.dispatch(SetMessages( roomId: room.id, - messages: room.messages + messagesStored, + messages: messagesStored, )); } catch (error) { printDebug('[fetchMessageEvents] $error'); @@ -114,6 +124,8 @@ ThunkAction fetchMessageEvents({ // The messages themselves final List messages = messagesJson['chunk'] ?? []; + messages.forEach((m) => printJson(m['content'])); + // reuse the logic for syncing await store.dispatch( syncRooms({ diff --git a/lib/store/events/reducer.dart b/lib/store/events/reducer.dart index bcb39326c..c805b1c00 100644 --- a/lib/store/events/reducer.dart +++ b/lib/store/events/reducer.dart @@ -9,7 +9,24 @@ EventStore eventReducer( case SetMessages: final roomId = action.roomId; final messages = Map>.from(state.messages); - messages[roomId] = action.messages; + + final messagesOld = Map.fromIterable( + state.messages[roomId], + key: (message) => message.id, + value: (message) => message, + ); + + final messagesNew = Map.fromIterable( + action.messages, + key: (message) => message.id, + value: (message) => message, + ); + + // combine new and old messages on event id to prevent duplicates + final messagesAll = messagesOld..addAll(messagesNew); + + messages[roomId] = messagesAll.values.toList(); + return state.copyWith(messages: messages); case SetState: diff --git a/lib/store/events/selectors.dart b/lib/store/events/selectors.dart index 53a1325c7..6a6eacfa0 100644 --- a/lib/store/events/selectors.dart +++ b/lib/store/events/selectors.dart @@ -1,10 +1,8 @@ // Project imports: import 'package:syphon/store/events/model.dart'; import 'package:syphon/store/index.dart'; -import 'package:syphon/store/rooms/room/model.dart'; -// TODO: replaces latestMessages() selectors with this -List latestRoomMessages(AppState state, String roomId) { +List roomMessages(AppState state, String roomId) { return state.eventStore.messages[roomId] ?? []; } diff --git a/lib/store/events/storage.dart b/lib/store/events/storage.dart index 4771fddc8..00ef53732 100644 --- a/lib/store/events/storage.dart +++ b/lib/store/events/storage.dart @@ -6,13 +6,11 @@ import 'package:syphon/store/events/model.dart'; const String MESSAGES = 'messages'; -Future saveMessages( - List messages, { - Database storage, -}) async { +Future saveMessages(List messages, {Database storage}) async { + printDebug('[saveMessages] saving ${messages.length}'); final store = StoreRef(MESSAGES); - await storage.transaction((txn) async { + return await storage.transaction((txn) async { for (Message message in messages) { final record = store.record(message.id); await record.put(txn, json.encode(message)); @@ -47,7 +45,6 @@ Future> loadMessages( messages.add(Message.fromJson(json.decode(message))); } - printDebug('[messages] loaded ${messages.length.toString()}'); return messages; } catch (error) { printDebug(error.toString(), title: 'loadMessages'); diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index a45b19550..1667f9f35 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -10,6 +10,7 @@ import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:redux/redux.dart'; import 'package:redux_thunk/redux_thunk.dart'; import 'package:syphon/global/cache/index.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/global/storage/index.dart'; import 'package:syphon/store/events/storage.dart'; @@ -26,6 +27,7 @@ import 'package:syphon/store/events/actions.dart'; import 'package:syphon/store/events/selectors.dart'; import 'package:syphon/store/rooms/storage.dart'; import 'package:syphon/store/sync/actions.dart'; +import 'package:syphon/store/user/actions.dart'; import 'package:syphon/store/user/storage.dart'; import 'package:syphon/store/user/model.dart'; import '../events/model.dart'; @@ -157,17 +159,40 @@ ThunkAction syncRooms(Map roomData) { json['timeline']['events'] = decryptedTimelineEvents; } - // filter through parsers + // TODO: eventually remove the need for this with modular parsers room = room.fromSync( json: json, currentUser: user, lastSince: lastSince, ); - // save cold storage objects - saveUsers(room.users, storage: Storage.main); - saveRooms({room.id: room}, storage: Storage.main); - saveMessages(room.messages, storage: Storage.main); + printDebug( + '[fromSync] ${room.name} after sync msg count ${room.messages.length}', + ); + printDebug( + '[fromSync] ${room.name} msg id count ${room.messageIds.length}', + ); + + // update store + await store.dispatch( + setUsers(room.users), + ); + await store.dispatch( + setMessageEvents(room: room, messages: room.messages), + ); + + // update cold storage + await Future.wait([ + saveUsers(room.users, storage: Storage.main), + saveRooms({room.id: room}, storage: Storage.main), + saveMessages(room.messages, storage: Storage.main), + ]); + + // TODO: remove with parsers - clear users from parsed room objects + room = room.copyWith( + users: Map(), + messages: List(), + ); // fetch avatar if a uri was found if (room.avatarUri != null) { @@ -521,9 +546,8 @@ ThunkAction markRoomRead({String roomId}) { // send read receipt remotely to mark locally on /sync if (store.state.settingsStore.readReceipts) { - final messagesSorted = latestMessages( - roomSelectors.room(id: roomId, state: store.state).messages, - ); + final messagesSorted = + latestMessages(roomMessages(store.state, roomId)); if (messagesSorted.isNotEmpty) { store.dispatch(sendReadReceipts( diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index 2ce824a02..37c15eaf0 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -4,6 +4,7 @@ import 'dart:collection'; // Package imports: import 'package:flutter/material.dart'; import 'package:json_annotation/json_annotation.dart'; +import 'package:syphon/global/print.dart'; // Project imports: import 'package:syphon/global/strings.dart'; @@ -504,6 +505,9 @@ class Room { String nextHash, }) { try { + printDebug( + '[fromMessageEvents] ${this.name} ${events.length.toString()}', + ); bool limited; int lastUpdate = this.lastUpdate; List messagesNew = events ?? []; @@ -546,6 +550,7 @@ class Room { // Combine current and existing messages on unique ids messagesExisting.addAll(messagesNew); + final messagesMap = HashMap.fromIterable( messagesExisting, key: (message) => message.id, @@ -557,15 +562,17 @@ class Room { (message) => messagesMap.containsKey(message.id), ); - // Filter to find startTime and endTime + // save message and message id updates final messageIds = List.from(messagesMap.keys); final messagesAll = List.from(messagesMap.values); + final messageIdsAll = List.from(this.messageIds) + ..addAll(messageIds); // Save values to room return this.copyWith( outbox: outbox, messages: messagesAll, - messageIds: messageIds, + messageIds: messageIdsAll, limited: limited ?? this.limited, encryptionEnabled: this.encryptionEnabled || hasEncrypted != null, lastUpdate: lastUpdate ?? this.lastUpdate, diff --git a/lib/store/rooms/room/selectors.dart b/lib/store/rooms/room/selectors.dart index 4ba8a75cf..41c53f147 100644 --- a/lib/store/rooms/room/selectors.dart +++ b/lib/store/rooms/room/selectors.dart @@ -23,9 +23,7 @@ String formatTotalUsers(int totalUsers) { return totalUsers.toString(); } -String formatPreview({Room room, List prefetched}) { - var messages = prefetched ?? room.messages; - +String formatPreview({Room room, List messages}) { // Prioritize drafts for any room, regardless of state if (room.draft != null && room.draft.body != null) { return 'Draft: ${formatPreviewMessage(room.draft.body)}'; @@ -46,10 +44,6 @@ String formatPreview({Room room, List prefetched}) { } // sort messages found - if (prefetched == null) { - messages = latestMessages(messages); - } - final recentMessage = messages[0]; var body = formatPreviewMessage(recentMessage.body); diff --git a/lib/store/rooms/selectors.dart b/lib/store/rooms/selectors.dart index b2e81b21b..6475dfb0a 100644 --- a/lib/store/rooms/selectors.dart +++ b/lib/store/rooms/selectors.dart @@ -2,15 +2,15 @@ import 'package:syphon/store/index.dart'; import './room/model.dart'; -List rooms(AppState state) { - return state.roomStore.roomList; -} - Room room({AppState state, String id}) { if (state.roomStore.rooms == null) return Room(); return state.roomStore.rooms[id] ?? Room(); } +List rooms(AppState state) { + return state.roomStore.roomList; +} + List sortedPrioritizedRooms(Map rooms) { final List sortedList = rooms != null ? List.from(rooms.values) : []; diff --git a/lib/store/rooms/storage.dart b/lib/store/rooms/storage.dart index a29d3236d..73c85cfe2 100644 --- a/lib/store/rooms/storage.dart +++ b/lib/store/rooms/storage.dart @@ -11,7 +11,7 @@ Future saveRooms( }) async { final store = StoreRef('rooms'); - await storage.transaction((txn) async { + return await storage.transaction((txn) async { for (Room room in rooms.values) { final record = store.record(room.id); await record.put(txn, jsonEncode(room)); diff --git a/lib/store/user/actions.dart b/lib/store/user/actions.dart index 6ebc17e64..a08a3468e 100644 --- a/lib/store/user/actions.dart +++ b/lib/store/user/actions.dart @@ -25,6 +25,11 @@ class SaveUser { SaveUser({this.user}); } +class SetUsers { + final Map users; + SetUsers({this.users}); +} + class SetUserInvites { final List users; SetUserInvites({this.users}); @@ -32,6 +37,12 @@ class SetUserInvites { class ClearUserInvites {} +ThunkAction setUsers(Map users) { + return (Store store) async { + store.dispatch(SetUsers(users: users)); + }; +} + ThunkAction setUserInvites({List users}) { return (Store store) async { store.dispatch(SetUserInvites(users: users)); diff --git a/lib/store/user/reducer.dart b/lib/store/user/reducer.dart index 4281f14f4..76eb9dde6 100644 --- a/lib/store/user/reducer.dart +++ b/lib/store/user/reducer.dart @@ -11,6 +11,10 @@ UserStore userReducer([UserStore state = const UserStore(), dynamic action]) { return state.copyWith(invites: action.users); case ClearUserInvites: return state.copyWith(invites: const []); + case SetUsers: + final users = Map.from(state.users); + users.addAll(action.users); + return state.copyWith(users: users); case SaveUser: final user = action.user as User; final users = Map.from(state.users); diff --git a/lib/store/user/selectors.dart b/lib/store/user/selectors.dart index 3b01480fd..48c2e3a01 100644 --- a/lib/store/user/selectors.dart +++ b/lib/store/user/selectors.dart @@ -61,7 +61,17 @@ String formatInitials(String fullword) { return initials.toUpperCase(); } -List searchUsersLocal(List users, {String searchText = ''}) { +List roomUsers(AppState state, String roomId) { + final room = state.roomStore.rooms[roomId] ?? Room(id: roomId); + return room.userIds.map((userId) => state.userStore.users[userId]).toList(); +} + +List searchUsersLocal( + AppState state, { + String roomId, + String searchText = '', +}) { + var users = roomUsers(state, roomId); if (searchText == null || searchText.isEmpty) { return users; } diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index fa80edf92..a71d74cc2 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -11,7 +11,7 @@ Future saveUsers( }) async { final store = StoreRef('users'); - await storage.transaction((txn) async { + return await storage.transaction((txn) async { for (User user in users.values) { final record = store.record(user.userId); await record.put(txn, jsonEncode(user)); diff --git a/lib/views/home/chat/details-all-users.dart b/lib/views/home/chat/details-all-users.dart index b19370e88..a8d078118 100644 --- a/lib/views/home/chat/details-all-users.dart +++ b/lib/views/home/chat/details-all-users.dart @@ -217,9 +217,8 @@ class _Props extends Equatable { searchText: store.state.searchStore.searchText, room: store.state.roomStore.rooms[roomId] ?? Room(), usersFiltered: searchUsersLocal( - List.from( - (store.state.roomStore.rooms[roomId] ?? Room()).users.values, - ), + store.state, + roomId: roomId, searchText: store.state.searchStore.searchText, ), onSearch: (text) { diff --git a/lib/views/home/chat/details-chat.dart b/lib/views/home/chat/details-chat.dart index 7d215be4d..22b221495 100644 --- a/lib/views/home/chat/details-chat.dart +++ b/lib/views/home/chat/details-chat.dart @@ -7,6 +7,7 @@ import 'package:equatable/equatable.dart'; import 'package:flutter_redux/flutter_redux.dart'; import 'package:redux/redux.dart'; import 'package:syphon/global/strings.dart'; +import 'package:syphon/store/user/selectors.dart'; import 'package:syphon/views/home/chat/details-all-users.dart'; import 'package:syphon/views/widgets/containers/card-section.dart'; import 'package:syphon/views/widgets/lists/list-user-bubbles.dart'; @@ -530,10 +531,8 @@ class _Props extends Equatable { static _Props mapStateToProps(Store store, String roomId) => _Props( room: roomSelectors.room(id: roomId, state: store.state), loading: store.state.roomStore.loading, - userList: List.from( - roomSelectors.room(id: roomId, state: store.state).users.values, - ), - messages: latestRoomMessages(store.state, roomId), + userList: roomUsers(store.state, roomId), + messages: roomMessages(store.state, roomId), onLeaveChat: () async { await store.dispatch(removeRoom(room: Room(id: roomId))); }, diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index f08209e70..415cece14 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -132,7 +132,7 @@ class ChatViewState extends State { }); } - if (props.room.messages.length < 10) { + if (props.messages.length < 10) { props.onLoadFirstBatch(); } @@ -721,7 +721,7 @@ class _Props extends Equatable { users: store.state.userStore.users, messages: latestMessages( wrapOutboxMessages( - messages: roomSelectors.room(id: roomId, state: store.state).messages, + messages: roomMessages(store.state, roomId), outbox: roomSelectors.room(id: roomId, state: store.state).outbox, ), ), @@ -814,18 +814,21 @@ class _Props extends Equatable { }, onLoadMoreMessages: () { final room = store.state.roomStore.rooms[roomId] ?? Room(); + final messages = roomMessages(store.state, roomId); // load message from cold storage - if (room.messages.length < room.messageIds.length) { + if (messages.length < room.messageIds.length) { + printDebug('[onLoadMoreMessages] loading from cold storage'); return store.dispatch( loadMessageEvents( room: room, - offset: room.messages.length, + offset: messages.length, ), ); } // fetch messages beyond the oldest known message - lastHash + printDebug('[onLoadMoreMessages] loading from remote server'); return store.dispatch(fetchMessageEvents( room: room, from: room.lastHash, diff --git a/lib/views/home/index.dart b/lib/views/home/index.dart index 0963b7905..f1d844ae6 100644 --- a/lib/views/home/index.dart +++ b/lib/views/home/index.dart @@ -286,7 +286,7 @@ class HomeViewState extends State { final messagesLatest = latestMessages(messages); final messagePreview = formatPreview( room: room, - prefetched: messagesLatest, + messages: messagesLatest, ); final roomSettings = props.chatSettings[room.id] ?? null; @@ -331,6 +331,7 @@ class HomeViewState extends State { textStyle = TextStyle(fontStyle: FontStyle.italic); } + // display message as being 'unread' if (messages != null && messages.isNotEmpty) { final messageRecent = messagesLatest[0]; diff --git a/lib/views/home/settings/devices.dart b/lib/views/home/settings/devices.dart index 2bb4db0f6..d6a750970 100644 --- a/lib/views/home/settings/devices.dart +++ b/lib/views/home/settings/devices.dart @@ -118,15 +118,6 @@ class DeviceViewState extends State { ], ), actions: [ - IconButton( - icon: Icon(Icons.text_fields), - iconSize: Dimensions.buttonAppBarSize, - tooltip: 'TEST KEY FUNCTION GENERIC', - color: Colors.white, - onPressed: () { - props.onTest(); - }, - ), IconButton( icon: Icon(Icons.edit), iconSize: Dimensions.buttonAppBarSize, @@ -330,7 +321,6 @@ class Props extends Equatable { final Function onFetchDevices; final Function onDeleteDevices; - final Function onTest; Props({ @required this.loading, @@ -339,7 +329,6 @@ class Props extends Equatable { @required this.currentDeviceId, @required this.onFetchDevices, @required this.onDeleteDevices, - @required this.onTest, }); @override @@ -348,17 +337,11 @@ class Props extends Equatable { devices, ]; - static Props mapStateToProps( - Store store, - ) => - Props( + static Props mapStateToProps(Store store) => Props( loading: store.state.settingsStore.loading, devices: store.state.settingsStore.devices ?? const [], session: store.state.authStore.session, currentDeviceId: store.state.authStore.user.deviceId, - onTest: () { - store.dispatch(fetchDeviceKeys()); - }, onDeleteDevices: ( BuildContext context, List devices, { From 3c54432f0843bcc900a0cc97c0eb77986d4ff898 Mon Sep 17 00:00:00 2001 From: ereio Date: Wed, 9 Dec 2020 23:19:48 -0500 Subject: [PATCH 21/25] print cleanup, more bugfixes, just needs a lot of fixes now --- lib/global/storage/index.dart | 2 +- lib/store/events/actions.dart | 3 +-- lib/store/events/reducer.dart | 15 ++++++--------- lib/store/events/storage.dart | 1 - lib/store/user/actions.dart | 2 +- lib/views/home/chat/index.dart | 3 ++- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index a0b134d73..55ca9c019 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -93,7 +93,7 @@ Future deleteStorage() async { Storage.mainKey, ); } catch (error) { - printDebug(error.toString()); + printDebug('[deleteStorage] ${error.toString()}'); } } diff --git a/lib/store/events/actions.dart b/lib/store/events/actions.dart index b2def330a..635e209c1 100644 --- a/lib/store/events/actions.dart +++ b/lib/store/events/actions.dart @@ -74,6 +74,7 @@ ThunkAction loadMessageEvents({ limit: !room.encryptionEnabled ? limit : room.messageIds.length, ); + // load cold storage messages to state store.dispatch(SetMessages( roomId: room.id, messages: messagesStored, @@ -124,8 +125,6 @@ ThunkAction fetchMessageEvents({ // The messages themselves final List messages = messagesJson['chunk'] ?? []; - messages.forEach((m) => printJson(m['content'])); - // reuse the logic for syncing await store.dispatch( syncRooms({ diff --git a/lib/store/events/reducer.dart b/lib/store/events/reducer.dart index c805b1c00..836bc3f42 100644 --- a/lib/store/events/reducer.dart +++ b/lib/store/events/reducer.dart @@ -9,20 +9,17 @@ EventStore eventReducer( case SetMessages: final roomId = action.roomId; final messages = Map>.from(state.messages); - final messagesOld = Map.fromIterable( - state.messages[roomId], - key: (message) => message.id, - value: (message) => message, + messages[roomId] ?? [], + key: (msg) => msg.id, + value: (msg) => msg, ); - final messagesNew = Map.fromIterable( - action.messages, - key: (message) => message.id, - value: (message) => message, + action.messages ?? [], + key: (msg) => msg.id, + value: (msg) => msg, ); - // combine new and old messages on event id to prevent duplicates final messagesAll = messagesOld..addAll(messagesNew); messages[roomId] = messagesAll.values.toList(); diff --git a/lib/store/events/storage.dart b/lib/store/events/storage.dart index 00ef53732..63dcd8ecb 100644 --- a/lib/store/events/storage.dart +++ b/lib/store/events/storage.dart @@ -7,7 +7,6 @@ import 'package:syphon/store/events/model.dart'; const String MESSAGES = 'messages'; Future saveMessages(List messages, {Database storage}) async { - printDebug('[saveMessages] saving ${messages.length}'); final store = StoreRef(MESSAGES); return await storage.transaction((txn) async { diff --git a/lib/store/user/actions.dart b/lib/store/user/actions.dart index a08a3468e..013038799 100644 --- a/lib/store/user/actions.dart +++ b/lib/store/user/actions.dart @@ -38,7 +38,7 @@ class SetUserInvites { class ClearUserInvites {} ThunkAction setUsers(Map users) { - return (Store store) async { + return (Store store) { store.dispatch(SetUsers(users: users)); }; } diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index 415cece14..818d0a82f 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -818,7 +818,8 @@ class _Props extends Equatable { // load message from cold storage if (messages.length < room.messageIds.length) { - printDebug('[onLoadMoreMessages] loading from cold storage'); + printDebug( + '[onLoadMoreMessages] loading from cold storage ${messages.length} ${room.messageIds.length}'); return store.dispatch( loadMessageEvents( room: room, From b7340519826b3d0613191cf8084909ed65360915 Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 13 Dec 2020 20:18:43 -0500 Subject: [PATCH 22/25] fixing several bugs regarding message loading, initialization, and recent users list --- lib/global/cache/index.dart | 4 +- lib/global/cache/serializer.dart | 4 +- lib/global/cache/storage.dart | 7 -- lib/global/cache/threadables.dart | 2 +- lib/global/print.dart | 11 +- lib/global/storage/index.dart | 5 +- lib/main.dart | 2 +- lib/store/events/actions.dart | 49 +++++++- lib/store/events/parsers.dart | 112 ++++++++++++++++-- lib/store/events/storage.dart | 26 +++- lib/store/media/state.dart | 2 +- lib/store/rooms/actions.dart | 75 ++++-------- lib/store/rooms/reducer.dart | 2 +- lib/store/rooms/room/model.dart | 101 ++++++++-------- lib/store/rooms/storage.dart | 4 +- lib/store/sync/actions.dart | 35 ++---- lib/store/sync/background/service.dart | 5 +- lib/store/user/selectors.dart | 4 +- lib/store/user/storage.dart | 4 +- lib/views/home/chat/details-chat.dart | 8 +- lib/views/home/chat/index.dart | 26 ++-- .../widgets/messages/message-typing.dart | 2 + pubspec.lock | 7 ++ pubspec.yaml | 3 + 24 files changed, 323 insertions(+), 177 deletions(-) diff --git a/lib/global/cache/index.dart b/lib/global/cache/index.dart index 62f221fb4..6fd1ce51b 100644 --- a/lib/global/cache/index.dart +++ b/lib/global/cache/index.dart @@ -83,7 +83,7 @@ Future initCache() async { return Cache.cacheMain; } catch (error) { - printDebug('[initCache] ${error}'); + printError('[initCache] ${error}'); return null; } } @@ -150,7 +150,7 @@ Future unlockCryptKey() async { key: Cache.cryptKeyLocation, ); } catch (error) { - printDebug('[unlockCryptKey] ${error}'); + printError('[unlockCryptKey] ${error}'); } // Create a encryptionKey if a serialized one is not found diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index fc0376982..21b163053 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -11,7 +11,7 @@ import 'package:redux_persist/redux_persist.dart'; import 'package:sembast/sembast.dart'; import 'package:syphon/global/cache/index.dart'; import 'package:syphon/global/cache/threadables.dart'; -import 'package:syphon/global/storage/index.dart'; +import 'package:syphon/global/print.dart'; // Project imports: import 'package:syphon/store/crypto/state.dart'; @@ -169,7 +169,7 @@ class CacheSerializer implements StateSerializer { break; } } catch (error) { - debugPrint('[CacheSerializer.decode] $error'); + printError('[CacheSerializer.decode] $error'); } }); diff --git a/lib/global/cache/storage.dart b/lib/global/cache/storage.dart index a8a4167fc..288d79e92 100644 --- a/lib/global/cache/storage.dart +++ b/lib/global/cache/storage.dart @@ -32,15 +32,11 @@ class CacheStorage implements StorageEngine { await Future.wait(stores.map((store) async { final type = store.runtimeType.toString(); try { - // Stopwatch stopwatchTotal = new Stopwatch()..start(); - // Stopwatch stopwatchStore = new Stopwatch()..start(); // Fetch from database final table = StoreRef.main(); final record = table.record(store.runtimeType.toString()); final jsonEncrypted = await record.get(cache); - // printDebug('[CacheStorage] load ${stopwatchStore.elapsed}'); - // Decrypt from database final jsonDecoded = await compute( decryptJsonBackground, @@ -54,11 +50,8 @@ class CacheStorage implements StorageEngine { debugLabel: 'decryptJsonBackground', ); - // printDebug('[CacheStorage] decrypt ${stopwatchStore.elapsed}'); - // Load for CacheSerializer to use later Cache.cacheStores[type] = jsonDecoded; - // printDebug('[CacheStorage] total time ${stopwatchTotal.elapsed} '); } catch (error) { printError(error.toString(), title: 'CacheStorage|$type'); } diff --git a/lib/global/cache/threadables.dart b/lib/global/cache/threadables.dart index dac22bc35..d8f39ce41 100644 --- a/lib/global/cache/threadables.dart +++ b/lib/global/cache/threadables.dart @@ -98,7 +98,7 @@ Future serializeJsonBackground(Object store) async { return encrypted.base64; } catch (error) { - printDebug('[serializeJsonBackground] $error'); + printError('[serializeJsonBackground] $error'); return null; } } diff --git a/lib/global/print.dart b/lib/global/print.dart index 0c94b4965..ace2d5c5c 100644 --- a/lib/global/print.dart +++ b/lib/global/print.dart @@ -1,21 +1,25 @@ +import 'package:ansicolor/ansicolor.dart'; import 'package:flutter/material.dart'; typedef PrintDebug = void Function(String message, {String title}); typedef PrintError = void Function(String message, {String title}); void _printInfo(String content, {String title}) { + final pen = AnsiPen()..white(bold: true); final body = title != null ? '[$title] $content' : content; - print('\u001b[32m$body\u001b[0m'); + print(pen(body)); } void _printWarning(String content, {String title}) { + final pen = AnsiPen()..yellow(bold: true); final body = title != null ? '[$title] $content' : content; - print('\u001b[34m$body\u001b[0m'); + print(pen(body)); } void _printError(String content, {String title}) { + final pen = AnsiPen()..red(bold: true); final body = title != null ? '[$title] $content' : content; - debugPrint('\u001b[31m$body\u001b[0m'); + print(pen(body)); } void _printDebug(String content, {String title}) { @@ -25,6 +29,5 @@ void _printDebug(String content, {String title}) { PrintDebug printInfo = _printInfo; PrintDebug printDebug = _printDebug; - PrintError printError = _printError; PrintError printWarning = _printWarning; diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index 55ca9c019..ba1106699 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -93,7 +93,7 @@ Future deleteStorage() async { Storage.mainKey, ); } catch (error) { - printDebug('[deleteStorage] ${error.toString()}'); + printError('[deleteStorage] ${error.toString()}'); } } @@ -114,9 +114,8 @@ Future>> loadStorage(Database storage) async { room.messageIds, storage: storage, encrypted: room.encryptionEnabled, - limit: 20, ); - printDebug( + printError( '[loadMessages] ${messages[room.id].length.toString()} ${room.name} loaded', ); } diff --git a/lib/main.dart b/lib/main.dart index 35cb4744e..6839f22af 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -57,7 +57,7 @@ void main() async { // init background sync for Android only if (Platform.isAndroid) { final backgroundSyncStatus = await BackgroundSync.init(); - debugPrint('[main] background service started $backgroundSyncStatus'); + printDebug('[main] background service started $backgroundSyncStatus'); } // init hot cache and cold storage diff --git a/lib/store/events/actions.dart b/lib/store/events/actions.dart index 635e209c1..392bc2887 100644 --- a/lib/store/events/actions.dart +++ b/lib/store/events/actions.dart @@ -57,6 +57,8 @@ ThunkAction setMessageEvents({ * those existing in cold storage depending on requests from client * * Make sure these have been exhausted before calling fetchMessageEvents + * + * TODO: still needs work */ ThunkAction loadMessageEvents({ Room room, @@ -80,7 +82,7 @@ ThunkAction loadMessageEvents({ messages: messagesStored, )); } catch (error) { - printDebug('[fetchMessageEvents] $error'); + printError('[fetchMessageEvents] $error'); } finally { store.dispatch(UpdateRoom(id: room.id, syncing: false)); } @@ -125,6 +127,11 @@ ThunkAction fetchMessageEvents({ // The messages themselves final List messages = messagesJson['chunk'] ?? []; + messages.forEach((msg) { + printDebug("[fetchMessageEvents] *** PRINT MESSAGES *** ${room.name}"); + printJson(msg); + }); + // reuse the logic for syncing await store.dispatch( syncRooms({ @@ -145,6 +152,46 @@ ThunkAction fetchMessageEvents({ }; } +/** + * Decrypt Events + * + * Reattribute decrypted events to the timeline + */ +ThunkAction decryptEvents(Room room, Map json) { + return (Store store) async { + try { + // First past to decrypt encrypted events + final List timelineEvents = json['timeline']['events']; + + // map through each event and decrypt if possible + final decryptTimelineActions = timelineEvents.map((event) async { + final eventType = event['type']; + switch (eventType) { + case EventTypes.encrypted: + return await store.dispatch( + decryptMessageEvent(roomId: room.id, event: event), + ); + default: + return event; + } + }); + + // add the decrypted events back to the + final decryptedTimelineEvents = await Future.wait( + decryptTimelineActions, + ); + + return decryptedTimelineEvents; + } catch (error) { + debugPrint( + '[decryptEvents] ${room.name ?? 'Unknown Room Name'} ${error.toString()}', + ); + } finally { + store.dispatch(UpdateRoom(id: room.id, syncing: false)); + } + }; +} + /** * * Fetch State Events diff --git a/lib/store/events/parsers.dart b/lib/store/events/parsers.dart index 738720c1f..32b428b08 100644 --- a/lib/store/events/parsers.dart +++ b/lib/store/events/parsers.dart @@ -1,12 +1,100 @@ +import 'dart:collection'; + +/** + * + * Event Parsers + * + * It's going to be difficult to parse external to room context + * because so much of the rooms context is gather through the DAG of + * events. You'd need to pass back both an updated room AND a list of messages + * to save seperately, or sacrific iterating through the message list again + * + */ import 'package:sembast/sembast.dart'; import 'package:syphon/store/events/model.dart'; -import 'package:syphon/store/user/model.dart'; +import 'package:syphon/store/rooms/room/model.dart'; + +Map parseMessages({ + Room room, + List messages = const [], + List existing = const [], +}) { + bool limited; + int lastUpdate = room.lastUpdate; + List outbox = List.from(room.outbox ?? []); + List messagesAll = List.from(existing ?? []); + + // Converting only message events + final hasEncrypted = messages.firstWhere( + (msg) => msg.type == EventTypes.encrypted, + orElse: () => null, + ); + + // See if the newest message has a greater timestamp + if (messages.isNotEmpty && lastUpdate < messages[0].timestamp) { + lastUpdate = messages[0].timestamp; + } + + // limited indicates need to fetch additional data for room timelines + if (room.limited) { + // Check to see if the new messages contain those existing in cache + if (messages.isNotEmpty && room.messageIds.isNotEmpty) { + final messageLatest = room.messageIds.firstWhere( + (id) => id == messages[0].id, + orElse: () => null, + ); + // Set limited to false if they now exist + limited = messageLatest != null; + } -Future> parseStateEvents( + // Set limited to false false if + // - the oldest hash (lastHash) is non-existant + // - the previous hash (most recent) is non-existant + // - the oldest hash equals the previously fetched hash + if (room.lastHash == null || + room.prevHash == null || + room.lastHash == room.prevHash) { + limited = false; + } + } + + // Combine current and existing messages on unique ids + messagesAll.addAll(messages); + + // Map messages to ids to filter out ids and outbox + final messagesAllMap = HashMap.fromIterable( + messagesAll, + key: (message) => message.id, + value: (message) => message, + ); + + // Remove outboxed messages + outbox.removeWhere( + (message) => messagesAllMap.containsKey(message.id), + ); + + // save messages and unique message id updates + final messageIdsAll = Set.from(room.messageIds) + ..addAll(messagesAllMap.keys); + + return { + 'messages': messages, + 'room': room.copyWith( + outbox: outbox, + messageIds: messageIdsAll.toList(), + limited: limited ?? room.limited, + lastUpdate: lastUpdate ?? room.lastUpdate, + encryptionEnabled: room.encryptionEnabled || hasEncrypted != null, + ), + }; +} + +Map parseEvents( Map json, { Database database, }) { List stateEvents = []; + List messageEvents = []; if (json['state'] != null) { final List stateEventsRaw = json['state']['events']; @@ -25,13 +113,21 @@ Future> parseStateEvents( if (json['timeline'] != null) { final List timelineEventsRaw = json['timeline']['events']; - for (dynamic event in timelineEventsRaw) { - if (!(event['type'] == EventTypes.message || - event['type'] == EventTypes.encrypted)) { - stateEvents.add(Event.fromMatrix(event)); + final List timelineEvents = List.from( + timelineEventsRaw.map((event) => Event.fromMatrix(event)), + ); + + for (dynamic event in timelineEvents) { + if (event.type == EventTypes.message || + event.type == EventTypes.encrypted) { + messageEvents.add(Message.fromEvent(event)); + } else { + stateEvents.add(event); } } } - - return Future.value(); + return { + 'states': stateEvents, + "messages": messageEvents, + }; } diff --git a/lib/store/events/storage.dart b/lib/store/events/storage.dart index 63dcd8ecb..f0c60cef6 100644 --- a/lib/store/events/storage.dart +++ b/lib/store/events/storage.dart @@ -2,11 +2,16 @@ import 'dart:convert'; import 'package:sembast/sembast.dart'; import 'package:syphon/global/print.dart'; +import 'package:syphon/store/events/ephemeral/m.read/model.dart'; import 'package:syphon/store/events/model.dart'; const String MESSAGES = 'messages'; +const String RECEIPTS = 'receipts'; -Future saveMessages(List messages, {Database storage}) async { +Future saveMessages( + List messages, { + Database storage, +}) async { final store = StoreRef(MESSAGES); return await storage.transaction((txn) async { @@ -35,7 +40,8 @@ Future> loadMessages( try { final store = StoreRef(MESSAGES); - final eventIdsPaginated = eventIds.skip(offset).take(limit).toList(); + // TODO: properly paginate through cold storage messages instead of loading all + final eventIdsPaginated = eventIds; //.skip(offset).take(limit).toList(); final messagesPaginated = await store.records(eventIdsPaginated).get(storage); @@ -46,7 +52,21 @@ Future> loadMessages( return messages; } catch (error) { - printDebug(error.toString(), title: 'loadMessages'); + printError(error.toString(), title: 'loadMessages'); return null; } } + +Future saveReceipts( + Map receipts, { + Database storage, +}) async { + final store = StoreRef(RECEIPTS); + + return await storage.transaction((txn) async { + for (String roomId in receipts.keys) { + final record = store.record(roomId); + await record.put(txn, json.encode(receipts[roomId])); + } + }); +} diff --git a/lib/store/media/state.dart b/lib/store/media/state.dart index 23ac8d892..efa947b7c 100644 --- a/lib/store/media/state.dart +++ b/lib/store/media/state.dart @@ -34,7 +34,7 @@ class MediaStore extends Equatable { mediaChecks: mediaChecks ?? this.mediaChecks, ); - // NOTE: custom json converter to allow Uint8List when in cache + // custom json converter to allow Uint8List when in cache // TODO: figure out how to make image-matrix.dart play nice with in component coonversions // Would repeatedly update even if a locally cached version matched factory MediaStore.fromJson(Map json) { diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index 1667f9f35..1391b1674 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -9,13 +9,12 @@ import 'package:flutter/foundation.dart'; import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:redux/redux.dart'; import 'package:redux_thunk/redux_thunk.dart'; -import 'package:syphon/global/cache/index.dart'; +import 'package:syphon/global/algos.dart'; import 'package:syphon/global/print.dart'; import 'package:syphon/global/storage/index.dart'; import 'package:syphon/store/events/storage.dart'; // Project imports: -import 'package:syphon/store/rooms/selectors.dart' as roomSelectors; import 'package:syphon/global/libs/matrix/encryption.dart'; import 'package:syphon/global/libs/matrix/errors.dart'; import 'package:syphon/global/libs/matrix/index.dart'; @@ -70,12 +69,13 @@ class UpdateRoom { } class RemoveRoom { - final Room room; - RemoveRoom({this.room}); + final String roomId; + RemoveRoom({this.roomId}); } -class ResetRooms { - ResetRooms(); +class AddArchive { + final String roomId; + AddArchive({this.roomId}); } /** @@ -96,17 +96,11 @@ class SaveOutboxMessage { class DeleteOutboxMessage { final Message message; // room id - DeleteOutboxMessage({ - this.message, - }); + DeleteOutboxMessage({this.message}); } -class AddArchive { - final String roomId; - - AddArchive({ - this.roomId, - }); +class ResetRooms { + ResetRooms(); } /** @@ -135,28 +129,10 @@ ThunkAction syncRooms(Map roomData) { // First past to decrypt encrypted events if (room.encryptionEnabled) { - final List timelineEvents = json['timeline']['events']; - - // map through each event and decrypt if possible - final decryptTimelineActions = timelineEvents.map((event) async { - final eventType = event['type']; - switch (eventType) { - case EventTypes.encrypted: - return await store.dispatch( - decryptMessageEvent(roomId: room.id, event: event), - ); - default: - return event; - } - }); - - // add the decrypted events back to the - final decryptedTimelineEvents = await Future.wait( - decryptTimelineActions, - ); - // reassign the mapped decrypted evets to the json timeline - json['timeline']['events'] = decryptedTimelineEvents; + json['timeline']['events'] = await store.dispatch( + decryptEvents(room, json), + ); } // TODO: eventually remove the need for this with modular parsers @@ -167,31 +143,31 @@ ThunkAction syncRooms(Map roomData) { ); printDebug( - '[fromSync] ${room.name} after sync msg count ${room.messages.length}', + '[syncRooms] ${room.name} new msg count ${room.messagesNew.length}', ); printDebug( - '[fromSync] ${room.name} msg id count ${room.messageIds.length}', + '[syncRooms] ${room.name} ids msg count ${room.messageIds.length}', ); // update store await store.dispatch( - setUsers(room.users), + setUsers(room.usersNew), ); await store.dispatch( - setMessageEvents(room: room, messages: room.messages), + setMessageEvents(room: room, messages: room.messagesNew), ); // update cold storage await Future.wait([ - saveUsers(room.users, storage: Storage.main), + saveUsers(room.usersNew, storage: Storage.main), saveRooms({room.id: room}, storage: Storage.main), - saveMessages(room.messages, storage: Storage.main), + saveMessages(room.messagesNew, storage: Storage.main), ]); // TODO: remove with parsers - clear users from parsed room objects room = room.copyWith( users: Map(), - messages: List(), + messagesNew: List(), ); // fetch avatar if a uri was found @@ -272,6 +248,7 @@ ThunkAction fetchRooms() { '${room.id}': { 'state': { 'events': stateEvents, + 'prev_batch': messageEvents['from'], }, 'timeline': { 'events': messageEvents['chunk'], @@ -897,9 +874,9 @@ ThunkAction removeRoom({Room room}) { // Remove the room locally if it's already been removed remotely if (leaveData['errcode'] != null) { if (leaveData['errcode'] == MatrixErrors.room_unknown) { - await store.dispatch(RemoveRoom(room: Room(id: room.id))); + await store.dispatch(RemoveRoom(roomId: room.id)); } else if (leaveData['errcode'] == MatrixErrors.room_not_found) { - await store.dispatch(RemoveRoom(room: Room(id: room.id))); + await store.dispatch(RemoveRoom(roomId: room.id)); } if (room.direct) { @@ -917,7 +894,7 @@ ThunkAction removeRoom({Room room}) { if (forgetData['errcode'] != null) { if (leaveData['errcode'] == MatrixErrors.room_not_found) { - await store.dispatch(RemoveRoom(room: Room(id: room.id))); + await store.dispatch(RemoveRoom(roomId: room.id)); } if (room.direct) { await store.dispatch(toggleDirectRoom(room: room, enabled: false)); @@ -932,7 +909,7 @@ ThunkAction removeRoom({Room room}) { if (room.direct) { await store.dispatch(toggleDirectRoom(room: room, enabled: false)); } - await store.dispatch(RemoveRoom(room: Room(id: room.id))); + await store.dispatch(RemoveRoom(roomId: room.id)); store.dispatch(SetLoading(loading: false)); } catch (error) { debugPrint('[removeRoom] $error'); @@ -972,11 +949,11 @@ ThunkAction leaveRoom({Room room}) { if (deleteData['errcode'] != null) { if (deleteData['errcode'] == MatrixErrors.room_unknown) { - store.dispatch(RemoveRoom(room: Room(id: room.id))); + store.dispatch(RemoveRoom(roomId: room.id)); } throw deleteData['error']; } - store.dispatch(RemoveRoom(room: Room(id: room.id))); + store.dispatch(RemoveRoom(roomId: room.id)); } catch (error) { debugPrint('[leaveRoom] $error'); } finally { diff --git a/lib/store/rooms/reducer.dart b/lib/store/rooms/reducer.dart index 2a246e39c..71740d97c 100644 --- a/lib/store/rooms/reducer.dart +++ b/lib/store/rooms/reducer.dart @@ -42,7 +42,7 @@ RoomStore roomReducer([RoomStore state = const RoomStore(), dynamic action]) { case RemoveRoom: final rooms = Map.from(state.rooms); - rooms.remove(action.room.id); + rooms.remove(action.roomId); return state.copyWith(rooms: rooms); case SaveOutboxMessage: diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index 37c15eaf0..dea0fba9c 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -61,14 +61,14 @@ class Room { final List state; @JsonKey(ignore: true) - final List messages; + final List messagesNew; // TODO: offload messageReads, for large rooms these are ridiculously large @JsonKey(ignore: true) final Map messageReads; @JsonKey(ignore: true) - final Map users; + final Map usersNew; @JsonKey(ignore: true) final bool userTyping; @@ -110,10 +110,10 @@ class Room { this.sending = false, this.limited = false, this.draft, - this.users, this.userIds = const [], this.outbox = const [], - this.messages = const [], + this.usersNew = const {}, + this.messagesNew = const [], this.messageIds = const [], this.lastRead = 0, this.lastUpdate = 0, @@ -133,9 +133,9 @@ class Room { }); Room copyWith({ - id, - name, - homeserver, + String id, + String name, + String homeserver, avatar, avatarUri, topic, @@ -158,9 +158,9 @@ class Room { users, userIds, events, - outbox, - messages, - messageIds, + List outbox, + List messagesNew, + List messageIds, messageReads, lastHash, prevHash, @@ -192,8 +192,8 @@ class Room { isDraftRoom: isDraftRoom ?? this.isDraftRoom, outbox: outbox ?? this.outbox, messageIds: messageIds ?? this.messageIds, - messages: messages ?? this.messages, - users: users ?? this.users, + messagesNew: messagesNew ?? this.messagesNew, + usersNew: users ?? this.usersNew, userIds: userIds ?? this.userIds, messageReads: messageReads ?? this.messageReads, lastHash: lastHash ?? this.lastHash, @@ -311,7 +311,7 @@ class Room { currentUser: currentUser, ) .fromMessageEvents( - events: messageEvents, + messages: messageEvents, lastHash: lastHash, prevHash: prevHash, nextHash: lastSince, @@ -368,7 +368,10 @@ class Room { bool direct = this.direct ?? false; int lastUpdate = this.lastUpdate; int namePriority = this.namePriority != 4 ? this.namePriority : 4; - Map users = this.users ?? Map(); + + var usersAdd = Map.from(this.usersNew); + var userIdsRemove = List(); + Set userIds = Set.from(this.userIds ?? []); events.forEach((event) { @@ -413,18 +416,23 @@ class Room { case 'm.room.member': final displayName = event.content['displayname']; final memberAvatarUri = event.content['avatar_url']; + final membership = event.content['membership']; direct = !direct ? event.content['is_direct'] ?? false : direct; // Cache user to rooms user cache if not present - if (!users.containsKey(event.sender)) { - users[event.sender] = User( - userId: event.sender, + if (!usersAdd.containsKey(event.stateKey)) { + usersAdd[event.stateKey] = User( + userId: event.stateKey, displayName: displayName, avatarUri: memberAvatarUri, ); } + if (membership == 'leave') { + userIdsRemove.add(event.stateKey); + } + break; case 'm.room.encryption': encryptionEnabled = true; @@ -439,15 +447,18 @@ class Room { } }); + userIds = userIds..addAll(usersAdd.keys ?? []); + userIds = userIds..removeWhere((id) => userIdsRemove.contains(id)); + try { // checks to make sure someone didn't name the room after the authed user final badRoomName = name == currentUser.displayName || name == currentUser.userId; // no name room check - if ((namePriority > 3 && users.isNotEmpty && direct) || badRoomName) { + if ((namePriority > 3 && usersAdd.isNotEmpty && direct) || badRoomName) { // Filter out number of non current users to show preview of total - final otherUsers = users.values.where( + final otherUsers = usersAdd.values.where( (user) => user.userId != currentUser.userId && user.displayName != currentUser.displayName, @@ -460,7 +471,7 @@ class Room { // set name and avi to first non user or that + total others name = hasMultipleUsers - ? '${shownUser.displayName} and ${users.values.length - 1}' + ? '${shownUser.displayName} and ${usersAdd.values.length - 1}' : shownUser.displayName; // set avatar if one has not been assigned @@ -473,16 +484,14 @@ class Room { } } catch (error) {} - userIds = userIds..addAll(users.keys ?? []); - return this.copyWith( name: name ?? this.name ?? Strings.labelRoomNameDefault, topic: topic ?? this.topic, - users: users ?? this.users, + users: usersAdd ?? this.usersNew, direct: direct ?? this.direct, invite: invite ?? this.invite, limited: limited ?? this.limited, - userIds: userIds.toList() ?? this.userIds, + userIds: userIds != null ? userIds.toList() : this.userIds ?? [], avatarUri: avatarUri ?? this.avatarUri, joinRule: joinRule ?? this.joinRule, lastUpdate: lastUpdate > 0 ? lastUpdate : this.lastUpdate, @@ -499,42 +508,42 @@ class Room { * outside displaying messages */ Room fromMessageEvents({ - List events, + List messages, String lastHash, String prevHash, // previously fetched hash String nextHash, }) { try { printDebug( - '[fromMessageEvents] ${this.name} ${events.length.toString()}', + '[fromMessageEvents] ${this.name} ${messages.length.toString()}', ); + bool limited; int lastUpdate = this.lastUpdate; - List messagesNew = events ?? []; List outbox = List.from(this.outbox ?? []); - List messagesExisting = List.from(this.messages ?? []); // Converting only message events - final hasEncrypted = messagesNew.firstWhere( + final hasEncrypted = messages.firstWhere( (msg) => msg.type == EventTypes.encrypted, orElse: () => null, ); // See if the newest message has a greater timestamp - if (messagesNew.isNotEmpty && lastUpdate < messagesNew[0].timestamp) { - lastUpdate = messagesNew[0].timestamp; + if (messages.isNotEmpty && lastUpdate < messages[0].timestamp) { + lastUpdate = messages[0].timestamp; } // limited indicates need to fetch additional data for room timelines if (this.limited) { // Check to see if the new messages contain those existing in cache - if (messagesNew.isNotEmpty && messagesExisting.isNotEmpty) { - final messageLatest = messagesExisting.firstWhere( - (msg) => msg.id == messagesNew[0].id, - orElse: () => null, - ); + if (messages.isNotEmpty && this.messageIds.isNotEmpty) { + final messageKnown = this.messageIds.firstWhere( + (id) => id == messages[0].id, + orElse: () => null, + ); + // Set limited to false if they now exist - limited = messageLatest != null; + limited = messageKnown != null; } // Set limited to false false if @@ -549,10 +558,8 @@ class Room { } // Combine current and existing messages on unique ids - messagesExisting.addAll(messagesNew); - final messagesMap = HashMap.fromIterable( - messagesExisting, + messages, key: (message) => message.id, value: (message) => message, ); @@ -562,17 +569,17 @@ class Room { (message) => messagesMap.containsKey(message.id), ); - // save message and message id updates - final messageIds = List.from(messagesMap.keys); - final messagesAll = List.from(messagesMap.values); - final messageIdsAll = List.from(this.messageIds) - ..addAll(messageIds); + // save messages and unique message id updates + final messageIdsNew = Set.from(messagesMap.keys); + final messagesNew = List.from(messagesMap.values); + final messageIdsAll = Set.from(this.messageIds) + ..addAll(messageIdsNew); // Save values to room return this.copyWith( outbox: outbox, - messages: messagesAll, - messageIds: messageIdsAll, + messagesNew: messagesNew, + messageIds: messageIdsAll.toList(), limited: limited ?? this.limited, encryptionEnabled: this.encryptionEnabled || hasEncrypted != null, lastUpdate: lastUpdate ?? this.lastUpdate, diff --git a/lib/store/rooms/storage.dart b/lib/store/rooms/storage.dart index 73c85cfe2..b1bb093c8 100644 --- a/lib/store/rooms/storage.dart +++ b/lib/store/rooms/storage.dart @@ -60,10 +60,10 @@ Future> loadRooms({ return null; } - printDebug('[rooms] loaded ${rooms.length.toString()}'); + printInfo('[rooms] loaded ${rooms.length.toString()}'); return rooms; } catch (error) { - printDebug(error.toString(), title: 'loadRooms'); + printError(error.toString(), title: 'loadRooms'); return null; } } diff --git a/lib/store/sync/actions.dart b/lib/store/sync/actions.dart index 0e3b60cc8..674938ecd 100644 --- a/lib/store/sync/actions.dart +++ b/lib/store/sync/actions.dart @@ -16,6 +16,7 @@ import 'package:redux_thunk/redux_thunk.dart'; import 'package:syphon/global/algos.dart'; import 'package:syphon/global/libs/matrix/errors.dart'; import 'package:syphon/global/libs/matrix/index.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/store/crypto/actions.dart'; import 'package:syphon/store/crypto/events/actions.dart'; import 'package:syphon/store/index.dart'; @@ -159,11 +160,13 @@ ThunkAction stopSyncObserver() { ThunkAction initialSync() { return (Store store) async { // Start initial sync in background - store.dispatch(fetchSync()); + await store.dispatch(fetchSync()); - // Fetch All Room Ids - await store.dispatch(fetchRooms()); + // Fetch All Room Ids - continue showing a sync + await store.dispatch(SetSyncing(syncing: true)); await store.dispatch(fetchDirectRooms()); + await store.dispatch(fetchRooms()); + await store.dispatch(SetSyncing(syncing: false)); }; } @@ -219,13 +222,15 @@ ThunkAction fetchSync({String since, bool forceFull = false}) { throw data['error']; } + // TODO: Unfiltered + // final Map rawLeft = data['rooms']['leave']; + // final Map presence = data['presence']; + final nextBatch = data['next_batch']; final oneTimeKeyCount = data['device_one_time_keys_count']; final Map rawJoined = data['rooms']['join']; final Map rawInvites = data['rooms']['invite']; - final Map rawLeft = data['rooms']['leave']; final Map rawToDevice = data['to_device']; - // TODO: final Map presence = data['presence']; // Updates for rooms await store.dispatch(syncRooms(rawJoined)); @@ -237,11 +242,6 @@ ThunkAction fetchSync({String since, bool forceFull = false}) { // Update encryption one time key count store.dispatch(updateOneTimeKeyCounts(oneTimeKeyCount)); - // TODO: cold storage cache the full sync in encrypted file - // if (isFullSync) { - // store.dispatch(saveSync(data)); - // } - // Update synced to indicate init sync and next batch id (lastSince) store.dispatch(SetSynced( synced: true, @@ -253,19 +253,8 @@ ThunkAction fetchSync({String since, bool forceFull = false}) { debugPrint('[fetchSync] full sync completed'); } } catch (error) { - String message = ''; - - try { - // try to understand the error message - message = (error.message as String); - } catch (error) { - debugPrint('[fetchSync] $error'); - } - - if (message.contains('SocketException')) { - debugPrint('[fetchSync] $error'); - store.dispatch(SetOffline(offline: true)); - } + store.dispatch(SetOffline(offline: true)); + printError('[fetchSync] ${error.toString()}'); final backoff = store.state.syncStore.backoff; final nextBackoff = backoff != 0 ? backoff + 1 : 5; diff --git a/lib/store/sync/background/service.dart b/lib/store/sync/background/service.dart index 28d09fd1a..ae459283e 100644 --- a/lib/store/sync/background/service.dart +++ b/lib/store/sync/background/service.dart @@ -235,9 +235,10 @@ FutureOr syncLoop({ // Filter each room through the parser rawRooms.forEach((roomId, json) { final room = Room().fromSync(json: json, lastSince: lastSinceNew); + final messagesNew = room.messagesNew; - if (room.messages.length == 1) { - final String messageSender = room.messages[0].sender; + if (messagesNew.length == 1) { + final String messageSender = messagesNew[0].sender; final String formattedSender = trimAlias(messageSender); if (!formattedSender.contains(userId)) { diff --git a/lib/store/user/selectors.dart b/lib/store/user/selectors.dart index 48c2e3a01..7e039060c 100644 --- a/lib/store/user/selectors.dart +++ b/lib/store/user/selectors.dart @@ -14,9 +14,11 @@ dynamic homeserver(AppState state) { List friendlyUsers(AppState state) { final rooms = state.roomStore.rooms.values; final users = state.userStore.users; + final userCurrent = state.authStore.user.userId; final roomsDirect = rooms.where((room) => room.direct); final roomUserIdsList = roomsDirect.map((room) => room.userIds); - final roomDirectUserIds = roomUserIdsList.expand((pair) => pair).toList(); + final roomDirectUserIdsAll = roomUserIdsList.expand((pair) => pair).toList(); + final roomDirectUserIds = roomDirectUserIdsAll..remove(userCurrent); final roomsDirectUsers = roomDirectUserIds.map((userId) => users[userId]); return List.from(roomsDirectUsers); diff --git a/lib/store/user/storage.dart b/lib/store/user/storage.dart index a71d74cc2..0bca3c151 100644 --- a/lib/store/user/storage.dart +++ b/lib/store/user/storage.dart @@ -65,10 +65,10 @@ Future> loadUsers({ return null; } - printDebug('[users] loaded ${users.length}'); + printInfo('[users] loaded ${users.length}'); return users; } catch (error) { - printDebug(error.toString(), title: 'loadUsers'); + printError(error.toString(), title: 'loadUsers'); return null; } } diff --git a/lib/views/home/chat/details-chat.dart b/lib/views/home/chat/details-chat.dart index 22b221495..13afc73e1 100644 --- a/lib/views/home/chat/details-chat.dart +++ b/lib/views/home/chat/details-chat.dart @@ -264,7 +264,7 @@ class ChatDetailsState extends State { maxHeight: Dimensions.avatarSizeLarge, ), child: ListUserBubbles( - users: props.userList, + users: props.users, roomId: props.room.id, ), ) @@ -501,7 +501,7 @@ class _Props extends Equatable { final bool loading; final Color roomPrimaryColor; final List messages; - final List userList; + final List users; final Function onLeaveChat; final Function onSelectPrimaryColor; @@ -510,7 +510,7 @@ class _Props extends Equatable { _Props({ @required this.room, - @required this.userList, + @required this.users, @required this.loading, @required this.messages, @required this.onLeaveChat, @@ -531,7 +531,7 @@ class _Props extends Equatable { static _Props mapStateToProps(Store store, String roomId) => _Props( room: roomSelectors.room(id: roomId, state: store.state), loading: store.state.roomStore.loading, - userList: roomUsers(store.state, roomId), + users: roomUsers(store.state, roomId), messages: roomMessages(store.state, roomId), onLeaveChat: () async { await store.dispatch(removeRoom(room: Room(id: roomId))); diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index 818d0a82f..8e2a40742 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -421,9 +421,9 @@ class ChatViewState extends State { controller: messagesController, children: [ MessageTypingWidget( + roomUsers: props.users, typing: props.room.userTyping, usersTyping: props.room.usersTyping, - roomUsers: props.users, selectedMessageId: this.selectedMessage != null ? this.selectedMessage.id : null, @@ -814,22 +814,22 @@ class _Props extends Equatable { }, onLoadMoreMessages: () { final room = store.state.roomStore.rooms[roomId] ?? Room(); - final messages = roomMessages(store.state, roomId); // load message from cold storage - if (messages.length < room.messageIds.length) { - printDebug( - '[onLoadMoreMessages] loading from cold storage ${messages.length} ${room.messageIds.length}'); - return store.dispatch( - loadMessageEvents( - room: room, - offset: messages.length, - ), - ); - } + // TODO: everything is loaded when opening the app for now + // final messages = roomMessages(store.state, roomId); + // if (messages.length < room.messageIds.length) { + // printDebug( + // '[onLoadMoreMessages] loading from cold storage ${messages.length} ${room.messageIds.length}'); + // return store.dispatch( + // loadMessageEvents( + // room: room, + // offset: messages.length, + // ), + // ); + // } // fetch messages beyond the oldest known message - lastHash - printDebug('[onLoadMoreMessages] loading from remote server'); return store.dispatch(fetchMessageEvents( room: room, from: room.lastHash, diff --git a/lib/views/widgets/messages/message-typing.dart b/lib/views/widgets/messages/message-typing.dart index 2550f6892..01084efe8 100644 --- a/lib/views/widgets/messages/message-typing.dart +++ b/lib/views/widgets/messages/message-typing.dart @@ -3,6 +3,7 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:syphon/global/colours.dart'; +import 'package:syphon/global/print.dart'; // Project imports: import 'package:syphon/store/user/model.dart'; @@ -85,6 +86,7 @@ class MessageTypingState extends State } if (widget.usersTyping.length > 0) { + printDebug('[MessageTypingWidget] ${widget.usersTyping.length}'); final usernamesTyping = widget.usersTyping; userTyping = widget.roomUsers[usernamesTyping[0]]; } diff --git a/pubspec.lock b/pubspec.lock index f5c6e8435..afefa26b7 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -22,6 +22,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.4.5+15" + ansicolor: + dependency: "direct main" + description: + name: ansicolor + url: "https://pub.dartlang.org" + source: hosted + version: "1.1.1" archive: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index efe1a1cd8..d07cc6cac 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -113,6 +113,9 @@ dependencies: flutter_material_color_picker: ^1.0.5 palette_generator: 0.2.3 + # fun + ansicolor: 1.1.1 + dev_dependencies: json_serializable: ^3.5.0 flutter_launcher_icons: "^0.7.5" From f8b7e89e0ff0ae8b97a7f9872860cc9c5260eb37 Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 13 Dec 2020 20:59:26 -0500 Subject: [PATCH 23/25] always initially load 10+ messages if they are present in the room --- lib/global/libs/matrix/events.dart | 7 ++++++- lib/global/libs/matrix/user.dart | 2 +- lib/store/events/actions.dart | 10 ++++------ lib/store/events/model.dart | 4 ++-- lib/store/user/actions.dart | 17 +++++++++-------- lib/views/home/chat/index.dart | 2 ++ 6 files changed, 24 insertions(+), 18 deletions(-) diff --git a/lib/global/libs/matrix/events.dart b/lib/global/libs/matrix/events.dart index 78bb8f0b5..3a6f44041 100644 --- a/lib/global/libs/matrix/events.dart +++ b/lib/global/libs/matrix/events.dart @@ -8,6 +8,7 @@ import 'package:syphon/global/algos.dart'; // Project imports: import 'package:syphon/global/libs/matrix/encryption.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/store/events/model.dart'; abstract class Events { @@ -50,16 +51,20 @@ abstract class Events { String from, String to, int limit = 10, // default limit by matrix - bool desc = true, // Direction of events + bool desc = true, // direction of events }) async { String url = '$protocol$homeserver/_matrix/client/r0/rooms/$roomId/messages'; + printDebug('[Matrix.fetchMessageEvents] ${limit}'); + // Params url += '?limit=$limit'; url += from != null ? '&from=${from}' : ''; url += to != null ? '&to=${to}' : ''; url += desc ? '&dir=b' : '&dir=f'; + // TODO: remove after implementing reactions + url += '&filter={"not_types":["${EventTypes.member}", "m.reaction"]}'; Map headers = { 'Authorization': 'Bearer $accessToken', diff --git a/lib/global/libs/matrix/user.dart b/lib/global/libs/matrix/user.dart index ed645ee60..de1ae344c 100644 --- a/lib/global/libs/matrix/user.dart +++ b/lib/global/libs/matrix/user.dart @@ -94,7 +94,7 @@ abstract class Users { List roomIds = const [], }) async { String url = - '$protocol$homeserver/_matrix/client/r0/user/$userId/account_data/${EventTypes.ignoredUserList}'; + '$protocol$homeserver/_matrix/client/r0/user/$userId/account_data/${AccountDataTypes.ignoredUserList}'; Map headers = { 'Authorization': 'Bearer $accessToken', diff --git a/lib/store/events/actions.dart b/lib/store/events/actions.dart index 392bc2887..20852b9c7 100644 --- a/lib/store/events/actions.dart +++ b/lib/store/events/actions.dart @@ -118,6 +118,9 @@ ThunkAction fetchMessageEvents({ "limit": limit, }); + printDebug("[fetchMessageEvents] CALLED FOR ${room.name} ${to} ${from}"); + printJson(messagesJson); + // The token the pagination ends at. If dir=b this token should be used again to request even earlier events. final String end = messagesJson['end']; @@ -127,11 +130,6 @@ ThunkAction fetchMessageEvents({ // The messages themselves final List messages = messagesJson['chunk'] ?? []; - messages.forEach((msg) { - printDebug("[fetchMessageEvents] *** PRINT MESSAGES *** ${room.name}"); - printJson(msg); - }); - // reuse the logic for syncing await store.dispatch( syncRooms({ @@ -145,7 +143,7 @@ ThunkAction fetchMessageEvents({ }), ); } catch (error) { - debugPrint('[fetchMessageEvents] $error'); + debugPrint('[fetchMessageEvents] error $error'); } finally { store.dispatch(UpdateRoom(id: room.id, syncing: false)); } diff --git a/lib/store/events/model.dart b/lib/store/events/model.dart index 9f8e94100..f46a11bcc 100644 --- a/lib/store/events/model.dart +++ b/lib/store/events/model.dart @@ -22,6 +22,7 @@ class EventTypes { static const message = 'm.room.message'; static const encrypted = 'm.room.encrypted'; static const member = 'm.room.member'; + static const reaction = 'm.reaction'; static const guestAccess = 'm.room.guest_access'; static const joinRules = 'm.room.join_rules'; @@ -29,8 +30,6 @@ class EventTypes { static const powerLevels = 'm.room.power_levels'; static const encryption = 'm.room.encryption'; static const roomKey = 'm.room_key'; - - static const ignoredUserList = 'm.ignored_user_list'; } class MessageTypes { @@ -42,6 +41,7 @@ class MessageTypes { static const AUDIO = 'm.text'; static const LOCATION = 'm.location'; static const VIDEO = 'm.video'; + static const ANNOTATIONO = 'm.annotation'; } class MediumType { diff --git a/lib/store/user/actions.dart b/lib/store/user/actions.dart index 013038799..39f839148 100644 --- a/lib/store/user/actions.dart +++ b/lib/store/user/actions.dart @@ -86,12 +86,12 @@ ThunkAction fetchUserProfile({User user}) { } /** - * Toggle Direct Room + * Toggle Block User * * NOTE: https://github.com/matrix-org/matrix-doc/issues/1519 * - * Fetch the direct rooms list and recalculate it without the - * given alias + * Fetch the blocked user list and recalculate + * events without the given user id */ ThunkAction toggleBlockUser({User user, Room room, bool block}) { return (Store store) async { @@ -100,11 +100,12 @@ ThunkAction toggleBlockUser({User user, Room room, bool block}) { // Pull remote direct room data final data = await MatrixApi.fetchAccountData( - protocol: protocol, - homeserver: store.state.authStore.user.homeserver, - accessToken: store.state.authStore.user.accessToken, - userId: store.state.authStore.user.userId, - type: AccountDataTypes.ignoredUserList); + protocol: protocol, + homeserver: store.state.authStore.user.homeserver, + accessToken: store.state.authStore.user.accessToken, + userId: store.state.authStore.user.userId, + type: AccountDataTypes.ignoredUserList, + ); if (data['errcode'] != null) { throw data['error']; diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index 8e2a40742..88929f302 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -799,10 +799,12 @@ class _Props extends Equatable { }, onLoadFirstBatch: () { final room = store.state.roomStore.rooms[roomId] ?? Room(); + printDebug('[onLoadFirstBatch]'); store.dispatch( fetchMessageEvents( room: room, from: room.nextHash, + limit: 25, ), ); }, From 3209407febab3c9b86fff8dbedec87771d6da36c Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 13 Dec 2020 22:48:41 -0500 Subject: [PATCH 24/25] users are being blocked as one would expect, both hidding messages and direct rooms --- lib/global/cache/serializer.dart | 12 +- lib/global/cache/storage.dart | 2 + lib/global/libs/matrix/errors.dart | 2 +- lib/global/libs/matrix/events.dart | 3 - lib/global/libs/matrix/index.dart | 2 +- lib/global/libs/matrix/user.dart | 14 +- lib/store/events/selectors.dart | 8 + lib/store/index.dart | 2 +- lib/store/rooms/actions.dart | 4 +- lib/store/rooms/selectors.dart | 15 +- lib/store/user/actions.dart | 74 ++++---- lib/store/user/reducer.dart | 6 +- lib/store/user/state.dart | 18 +- lib/views/home/chat/details-all-users.dart | 1 - lib/views/home/chat/index.dart | 13 +- lib/views/home/index.dart | 7 +- lib/views/home/settings/blocked.dart | 168 ++++++++++++++++++ lib/views/home/settings/privacy.dart | 13 ++ lib/views/navigation.dart | 2 + .../widgets/modals/modal-user-details.dart | 72 ++++---- 20 files changed, 323 insertions(+), 115 deletions(-) create mode 100644 lib/views/home/settings/blocked.dart diff --git a/lib/global/cache/serializer.dart b/lib/global/cache/serializer.dart index 21b163053..4e3347640 100644 --- a/lib/global/cache/serializer.dart +++ b/lib/global/cache/serializer.dart @@ -44,6 +44,7 @@ class CacheSerializer implements StateSerializer { state.cryptoStore, state.mediaStore, state.settingsStore, + state.userStore, ]; // Queue up a cache saving will wait @@ -66,8 +67,8 @@ class CacheSerializer implements StateSerializer { // Stopwatch stopwatchSerialize = new Stopwatch()..start(); try { // HACK: unable to pass certain stores directly to an isolate - final sensitiveStorage = [AuthStore, SyncStore, CryptoStore]; - if (!sensitiveStorage.contains(store.runtimeType)) { + final sensitiveStorage = [MediaStore]; + if (sensitiveStorage.contains(store.runtimeType)) { jsonEncoded = await compute(jsonEncode, store); } else { jsonEncoded = json.encode(store); @@ -131,6 +132,7 @@ class CacheSerializer implements StateSerializer { AppState decode(Uint8List data) { AuthStore authStore = AuthStore(); SyncStore syncStore = SyncStore(); + UserStore userStore = UserStore(); CryptoStore cryptoStore = CryptoStore(); MediaStore mediaStore = MediaStore(); SettingsStore settingsStore = SettingsStore(); @@ -162,8 +164,10 @@ class CacheSerializer implements StateSerializer { case 'SettingsStore': settingsStore = SettingsStore.fromJson(store); break; - case 'RoomStore': case 'UserStore': + userStore = UserStore.fromJson(store); + break; + case 'RoomStore': // --- cold storage only --- default: break; @@ -183,7 +187,7 @@ class CacheSerializer implements StateSerializer { roomStore: RoomStore().copyWith( rooms: preloaded['rooms'] ?? {}, ), - userStore: UserStore().copyWith( + userStore: userStore.copyWith( users: preloaded['users'] ?? {}, ), eventStore: EventStore().copyWith( diff --git a/lib/global/cache/storage.dart b/lib/global/cache/storage.dart index 288d79e92..387f3d840 100644 --- a/lib/global/cache/storage.dart +++ b/lib/global/cache/storage.dart @@ -13,10 +13,12 @@ import 'package:syphon/store/media/state.dart'; import 'package:syphon/store/rooms/state.dart'; import 'package:syphon/store/settings/state.dart'; import 'package:syphon/store/sync/state.dart'; +import 'package:syphon/store/user/state.dart'; final List stores = [ AuthStore(), SyncStore(), + UserStore(), MediaStore(), CryptoStore(), SettingsStore(), diff --git a/lib/global/libs/matrix/errors.dart b/lib/global/libs/matrix/errors.dart index cdf763406..5cf2341f1 100644 --- a/lib/global/libs/matrix/errors.dart +++ b/lib/global/libs/matrix/errors.dart @@ -1,7 +1,7 @@ class MatrixErrors { static const String room_unknown = 'M_UNKNOWN'; static const String not_authorized = 'M_UNAUTHORIZED'; - static const String room_not_found = 'M_NOT_FOUND'; + static const String not_found = "M_NOT_FOUND"; static const String user_in_use = 'M_USER_IN_USE'; static const String unknown_token = 'M_UNKNOWN_TOKEN'; static const String email_in_use = 'M_THREEPID_IN_USE'; diff --git a/lib/global/libs/matrix/events.dart b/lib/global/libs/matrix/events.dart index 3a6f44041..a57f04695 100644 --- a/lib/global/libs/matrix/events.dart +++ b/lib/global/libs/matrix/events.dart @@ -56,9 +56,6 @@ abstract class Events { String url = '$protocol$homeserver/_matrix/client/r0/rooms/$roomId/messages'; - printDebug('[Matrix.fetchMessageEvents] ${limit}'); - - // Params url += '?limit=$limit'; url += from != null ? '&from=${from}' : ''; url += to != null ? '&to=${to}' : ''; diff --git a/lib/global/libs/matrix/index.dart b/lib/global/libs/matrix/index.dart index beb9ec2f8..4f57be845 100644 --- a/lib/global/libs/matrix/index.dart +++ b/lib/global/libs/matrix/index.dart @@ -56,9 +56,9 @@ abstract class MatrixApi { static final updateAvatarUri = Users.updateAvatarUri; // Users - static final blockUser = Users.blockUser; static final inviteUser = Users.inviteUser; static final fetchUserProfile = Users.fetchUserProfile; + static final updateBlockedUsers = Users.updateBlockedUsers; // Media static final fetchThumbnail = Media.fetchThumbnail; diff --git a/lib/global/libs/matrix/user.dart b/lib/global/libs/matrix/user.dart index de1ae344c..8c64cfce4 100644 --- a/lib/global/libs/matrix/user.dart +++ b/lib/global/libs/matrix/user.dart @@ -86,12 +86,12 @@ abstract class Users { * to the user that set the account_data. The config will be synced * to clients in the top-level account_data. */ - static Future blockUser({ + static Future updateBlockedUsers({ String protocol = 'https://', String homeserver = 'matrix.org', String accessToken, String userId, - List roomIds = const [], + Map blockUserList = const {"ignored_users": {}}, }) async { String url = '$protocol$homeserver/_matrix/client/r0/user/$userId/account_data/${AccountDataTypes.ignoredUserList}'; @@ -100,16 +100,14 @@ abstract class Users { 'Authorization': 'Bearer $accessToken', }; - final accountData = {userId: []}; - - if (roomIds.isNotEmpty) { - accountData[userId] = roomIds; - } + final body = { + 'ignored_users': blockUserList ?? {}, + }; final saveResponse = await http.put( url, headers: headers, - body: json.encode(accountData), + body: json.encode(body), ); return await json.decode( diff --git a/lib/store/events/selectors.dart b/lib/store/events/selectors.dart index 6a6eacfa0..e308a27ad 100644 --- a/lib/store/events/selectors.dart +++ b/lib/store/events/selectors.dart @@ -6,6 +6,14 @@ List roomMessages(AppState state, String roomId) { return state.eventStore.messages[roomId] ?? []; } +// remove messages from blocked users +List filterMessages(List messages, List blocked) { + return messages + ..removeWhere( + (message) => blocked.contains(message.sender), + ); +} + List latestMessages(List messages) { final sortedList = List.from(messages ?? []); diff --git a/lib/store/index.dart b/lib/store/index.dart index fd71dc500..3a95c6b3c 100644 --- a/lib/store/index.dart +++ b/lib/store/index.dart @@ -109,7 +109,7 @@ Future initStore(Database cache, Database storage) async { storage: CacheStorage(cache: cache), serializer: CacheSerializer(cache: cache, preloaded: data), // TODO: can remove once cold storage is in place - throttleDuration: Duration(milliseconds: 4500), + throttleDuration: Duration(milliseconds: 4000), shouldSave: (Store store, dynamic action) { // TODO: can remove once cold storage is in place switch (action.runtimeType) { diff --git a/lib/store/rooms/actions.dart b/lib/store/rooms/actions.dart index 1391b1674..639d88b5a 100644 --- a/lib/store/rooms/actions.dart +++ b/lib/store/rooms/actions.dart @@ -875,7 +875,7 @@ ThunkAction removeRoom({Room room}) { if (leaveData['errcode'] != null) { if (leaveData['errcode'] == MatrixErrors.room_unknown) { await store.dispatch(RemoveRoom(roomId: room.id)); - } else if (leaveData['errcode'] == MatrixErrors.room_not_found) { + } else if (leaveData['errcode'] == MatrixErrors.not_found) { await store.dispatch(RemoveRoom(roomId: room.id)); } @@ -893,7 +893,7 @@ ThunkAction removeRoom({Room room}) { ); if (forgetData['errcode'] != null) { - if (leaveData['errcode'] == MatrixErrors.room_not_found) { + if (leaveData['errcode'] == MatrixErrors.not_found) { await store.dispatch(RemoveRoom(roomId: room.id)); } if (room.direct) { diff --git a/lib/store/rooms/selectors.dart b/lib/store/rooms/selectors.dart index 6475dfb0a..c1a0be282 100644 --- a/lib/store/rooms/selectors.dart +++ b/lib/store/rooms/selectors.dart @@ -11,9 +11,18 @@ List rooms(AppState state) { return state.roomStore.roomList; } -List sortedPrioritizedRooms(Map rooms) { - final List sortedList = - rooms != null ? List.from(rooms.values) : []; +List filterBlockedRooms(List rooms, List blocked) { + final List roomList = rooms != null ? rooms : []; + + return roomList + ..removeWhere((room) => + room.userIds.length == 2 && + room.userIds.any((userId) => blocked.contains(userId))) + ..toList(); +} + +List sortedPrioritizedRooms(List rooms) { + final sortedList = rooms != null ? rooms : []; // sort descending sortedList.sort((a, b) { diff --git a/lib/store/user/actions.dart b/lib/store/user/actions.dart index 39f839148..c1f805c27 100644 --- a/lib/store/user/actions.dart +++ b/lib/store/user/actions.dart @@ -2,7 +2,10 @@ import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:redux/redux.dart'; import 'package:redux_thunk/redux_thunk.dart'; +import 'package:syphon/global/algos.dart'; +import 'package:syphon/global/libs/matrix/errors.dart'; import 'package:syphon/global/libs/matrix/index.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/store/alerts/actions.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/events/model.dart'; @@ -30,6 +33,11 @@ class SetUsers { SetUsers({this.users}); } +class SetUsersBlocked { + final List userIds; + SetUsersBlocked({this.userIds}); +} + class SetUserInvites { final List users; SetUserInvites({this.users}); @@ -43,14 +51,20 @@ ThunkAction setUsers(Map users) { }; } +ThunkAction setUsersBlocked(List userIds) { + return (Store store) { + store.dispatch(SetUsersBlocked(userIds: userIds)); + }; +} + ThunkAction setUserInvites({List users}) { - return (Store store) async { + return (Store store) { store.dispatch(SetUserInvites(users: users)); }; } ThunkAction clearUserInvites() { - return (Store store) async { + return (Store store) { store.dispatch(ClearUserInvites()); }; } @@ -88,12 +102,10 @@ ThunkAction fetchUserProfile({User user}) { /** * Toggle Block User * - * NOTE: https://github.com/matrix-org/matrix-doc/issues/1519 - * * Fetch the blocked user list and recalculate * events without the given user id */ -ThunkAction toggleBlockUser({User user, Room room, bool block}) { +ThunkAction toggleBlockUser({User user}) { return (Store store) async { try { store.dispatch(SetLoading(loading: true)); @@ -107,50 +119,34 @@ ThunkAction toggleBlockUser({User user, Room room, bool block}) { type: AccountDataTypes.ignoredUserList, ); + // skip error if there's no blocked users list yet if (data['errcode'] != null) { - throw data['error']; + if (data['errcode'] != MatrixErrors.not_found) { + throw data['error']; + } } - return false; - // Pull the direct room for that specific user - Map directRoomUsers = data as Map; - final usersDirectRooms = directRoomUsers[user] ?? []; + Map usersBlocked = data['ignored_users'] ?? {}; - if (usersDirectRooms.isEmpty && block) { - directRoomUsers[user.userId] = [room.id]; + // toggle based on if the id is already present + if (!usersBlocked.containsKey(user.userId)) { + usersBlocked[user.userId] = {}; + } else { + usersBlocked.remove(user.userId); } - // Toggle the direct room data based on user actions - directRoomUsers = directRoomUsers.map((userId, rooms) { - List updatedRooms = List.from(rooms ?? []); - - if (userId != user.userId) { - return MapEntry(userId, updatedRooms); - } - - if (block) { - updatedRooms.add(room.id); - } else { - updatedRooms.removeWhere((roomId) => roomId == room.id); - } + // locally track the blocked users list + final usersBlockedList = usersBlocked.keys.toList(); + await store.dispatch(setUsersBlocked(usersBlockedList)); - return MapEntry(userId, updatedRooms); - }); - - // Filter out empty list entries for a user - directRoomUsers.removeWhere((key, value) { - final roomIds = value ?? []; - return roomIds.isEmpty; - }); - - final saveData = await MatrixApi.saveAccountData( + // save blocked users list back to account_data remotely + final saveData = await MatrixApi.updateBlockedUsers( protocol: protocol, accessToken: store.state.authStore.user.accessToken, homeserver: store.state.authStore.user.homeserver, userId: store.state.authStore.user.userId, - type: AccountDataTypes.ignoredUserList, - accountData: directRoomUsers, + blockUserList: usersBlocked, ); if (saveData['errcode'] != null) { @@ -159,8 +155,8 @@ ThunkAction toggleBlockUser({User user, Room room, bool block}) { } catch (error) { store.dispatch(addAlert( error: error, - message: "Failed to load users profile", - origin: 'fetchUserProfile', + message: "Failed to toggle user on blocklist", + origin: 'toggleBlockUser', )); } finally { store.dispatch(SetLoading(loading: false)); diff --git a/lib/store/user/reducer.dart b/lib/store/user/reducer.dart index 76eb9dde6..d9673c1d0 100644 --- a/lib/store/user/reducer.dart +++ b/lib/store/user/reducer.dart @@ -9,8 +9,8 @@ UserStore userReducer([UserStore state = const UserStore(), dynamic action]) { return state.copyWith(loading: action.loading); case SetUserInvites: return state.copyWith(invites: action.users); - case ClearUserInvites: - return state.copyWith(invites: const []); + case SetUsersBlocked: + return state.copyWith(blocked: action.userIds); case SetUsers: final users = Map.from(state.users); users.addAll(action.users); @@ -30,6 +30,8 @@ UserStore userReducer([UserStore state = const UserStore(), dynamic action]) { } return state.copyWith(users: action.users); + case ClearUserInvites: + return state.copyWith(invites: const []); default: return state; } diff --git a/lib/store/user/state.dart b/lib/store/user/state.dart index 7023ad150..61bc0096b 100644 --- a/lib/store/user/state.dart +++ b/lib/store/user/state.dart @@ -7,8 +7,12 @@ import 'package:syphon/store/user/model.dart'; part 'state.g.dart'; -@JsonSerializable(nullable: true, includeIfNull: true) +@JsonSerializable() class UserStore extends Equatable { + // user.id's + final List blocked; + + @JsonKey(ignore: true) final Map users; @JsonKey(ignore: true) @@ -20,6 +24,7 @@ class UserStore extends Equatable { const UserStore({ this.users = const {}, this.invites = const [], + this.blocked = const [], this.loading = false, }); @@ -30,19 +35,20 @@ class UserStore extends Equatable { loading, ]; - Map toJson() => _$UserStoreToJson(this); - - factory UserStore.fromJson(Map json) => - _$UserStoreFromJson(json); - UserStore copyWith({ bool loading, List invites, Map users, + List blocked, }) => UserStore( users: users ?? this.users, invites: invites ?? this.invites, loading: loading ?? this.loading, + blocked: blocked ?? this.blocked, ); + Map toJson() => _$UserStoreToJson(this); + + factory UserStore.fromJson(Map json) => + _$UserStoreFromJson(json); } diff --git a/lib/views/home/chat/details-all-users.dart b/lib/views/home/chat/details-all-users.dart index a8d078118..75509ce5a 100644 --- a/lib/views/home/chat/details-all-users.dart +++ b/lib/views/home/chat/details-all-users.dart @@ -81,7 +81,6 @@ class ChatUsersDetailState extends State { isScrollControlled: true, backgroundColor: Colors.transparent, builder: (context) => ModalUserDetails( - roomId: roomId, userId: userId, ), ); diff --git a/lib/views/home/chat/index.dart b/lib/views/home/chat/index.dart index 88929f302..36894b042 100644 --- a/lib/views/home/chat/index.dart +++ b/lib/views/home/chat/index.dart @@ -293,15 +293,11 @@ class ChatViewState extends State { @protected onViewUserDetails({Message message, String userId}) { - final arguements = - ModalRoute.of(context).settings.arguments as ChatViewArguements; - showModalBottomSheet( context: context, isScrollControlled: true, backgroundColor: Colors.transparent, builder: (context) => ModalUserDetails( - roomId: arguements.roomId, userId: userId ?? message.sender, ), ); @@ -720,9 +716,12 @@ class _Props extends Equatable { room: roomSelectors.room(id: roomId, state: store.state), users: store.state.userStore.users, messages: latestMessages( - wrapOutboxMessages( - messages: roomMessages(store.state, roomId), - outbox: roomSelectors.room(id: roomId, state: store.state).outbox, + filterMessages( + wrapOutboxMessages( + messages: roomMessages(store.state, roomId).toList(), + outbox: roomSelectors.room(id: roomId, state: store.state).outbox, + ), + store.state.userStore.blocked, ), ), roomPrimaryColor: () { diff --git a/lib/views/home/index.dart b/lib/views/home/index.dart index f1d844ae6..a55e03b21 100644 --- a/lib/views/home/index.dart +++ b/lib/views/home/index.dart @@ -638,7 +638,12 @@ class _Props extends Equatable { static _Props mapStateToProps(Store store) => _Props( theme: store.state.settingsStore.theme, rooms: availableRooms( - sortedPrioritizedRooms(store.state.roomStore.rooms), + sortedPrioritizedRooms( + filterBlockedRooms( + store.state.roomStore.rooms.values.toList(), + store.state.userStore.blocked, + ), + ), hidden: store.state.roomStore.roomsHidden, ), messages: store.state.eventStore.messages, diff --git a/lib/views/home/settings/blocked.dart b/lib/views/home/settings/blocked.dart new file mode 100644 index 000000000..72523c951 --- /dev/null +++ b/lib/views/home/settings/blocked.dart @@ -0,0 +1,168 @@ +// Dart imports: +import 'dart:async'; + +// Flutter imports: +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; + +// Package imports: +import 'package:equatable/equatable.dart'; +import 'package:flutter_redux/flutter_redux.dart'; +import 'package:redux/redux.dart'; +import 'package:syphon/views/widgets/containers/card-section.dart'; +import 'package:syphon/views/widgets/modals/modal-user-details.dart'; + +// Project imports: +import 'package:syphon/global/colours.dart'; +import 'package:syphon/global/dimensions.dart'; +import 'package:syphon/store/index.dart'; +import 'package:syphon/store/user/model.dart'; +import 'package:syphon/store/user/selectors.dart'; +import 'package:syphon/views/widgets/avatars/avatar.dart'; + +class BlockedUsersView extends StatefulWidget { + const BlockedUsersView({Key key}) : super(key: key); + + @override + BlockedUsersState createState() => BlockedUsersState(); +} + +class BlockedUsersState extends State { + bool loading = false; + + BlockedUsersState({Key key}); + + // componentDidMount(){} + @override + void didChangeDependencies() { + super.didChangeDependencies(); + } + + @protected + onShowUserDetails({ + BuildContext context, + String userId, + }) async { + await showModalBottomSheet( + context: context, + isScrollControlled: true, + backgroundColor: Colors.transparent, + builder: (context) => ModalUserDetails( + userId: userId, + ), + ); + } + + @protected + Widget buildUserList(BuildContext context, _Props props) => ListView.builder( + shrinkWrap: true, + scrollDirection: Axis.vertical, + itemCount: props.usersBlocked.length, + itemBuilder: (BuildContext context, int index) { + final user = props.usersBlocked[index]; + + return GestureDetector( + onTap: () => this.onShowUserDetails( + context: context, + userId: user.userId, + ), + child: CardSection( + padding: EdgeInsets.zero, + elevation: 0, + child: Container( + child: ListTile( + leading: Avatar( + uri: user.avatarUri, + alt: user.displayName ?? user.userId, + size: Dimensions.avatarSizeMin, + background: Colours.hashedColor( + user.displayName ?? user.userId, + ), + ), + title: Text( + formatUsername(user), + overflow: TextOverflow.ellipsis, + style: Theme.of(context).textTheme.bodyText1, + ), + subtitle: Text( + user.userId, + style: Theme.of(context).textTheme.caption.merge( + TextStyle( + color: props.loading + ? Color(Colours.greyDisabled) + : null, + ), + ), + ), + ), + ), + ), + ); + }, + ); + + @override + Widget build(BuildContext context) { + return StoreConnector( + distinct: true, + converter: (Store store) => _Props.mapStateToProps(store), + builder: (context, props) => Scaffold( + appBar: AppBar( + title: Text('Blocked users'), + leading: IconButton( + icon: Icon(Icons.arrow_back, color: Colors.white), + onPressed: () => Navigator.pop(context, false), + ), + ), + body: Stack( + children: [ + buildUserList(context, props), + Positioned( + child: Visibility( + visible: this.loading, + child: Container( + margin: EdgeInsets.only(top: 16), + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + RefreshProgressIndicator( + strokeWidth: Dimensions.defaultStrokeWidth, + valueColor: new AlwaysStoppedAnimation( + Theme.of(context).primaryColor, + ), + value: null, + ), + ], + ), + ), + ), + ), + ], + ), + ), + ); + } +} + +class _Props extends Equatable { + final bool loading; + final List usersBlocked; + + _Props({ + @required this.loading, + @required this.usersBlocked, + }); + + @override + List get props => [ + loading, + usersBlocked, + ]; + + static _Props mapStateToProps(Store store) => _Props( + loading: store.state.roomStore.loading, + usersBlocked: store.state.userStore.blocked + .map((id) => store.state.userStore.users[id]) + .toList(), + ); +} diff --git a/lib/views/home/settings/privacy.dart b/lib/views/home/settings/privacy.dart index f2d96c5fe..266f32f74 100644 --- a/lib/views/home/settings/privacy.dart +++ b/lib/views/home/settings/privacy.dart @@ -112,6 +112,19 @@ class PrivacyPreferences extends StatelessWidget { style: Theme.of(context).textTheme.caption, ), ), + ListTile( + onTap: () { + Navigator.pushNamed(context, '/blocked'); + }, + contentPadding: Dimensions.listPadding, + title: Text( + 'Blocked Users', + ), + subtitle: Text( + 'View and manage blocked users', + style: Theme.of(context).textTheme.caption, + ), + ), ], ), ), diff --git a/lib/views/navigation.dart b/lib/views/navigation.dart index 93c3a4dc2..a5c9dfde9 100644 --- a/lib/views/navigation.dart +++ b/lib/views/navigation.dart @@ -17,6 +17,7 @@ import 'package:syphon/views/home/search/search-groups.dart'; import 'package:syphon/views/home/search/search-rooms.dart'; import 'package:syphon/views/home/search/search-users.dart'; import 'package:syphon/views/home/settings/advanced.dart'; +import 'package:syphon/views/home/settings/blocked.dart'; import 'package:syphon/views/home/settings/chats.dart'; import 'package:syphon/views/home/settings/devices.dart'; import 'package:syphon/views/home/settings/index.dart'; @@ -86,6 +87,7 @@ class NavigationProvider { '/theming': (BuildContext context) => Theming(), '/devices': (BuildContext context) => DevicesView(), '/settings': (BuildContext context) => SettingsScreen(), + '/blocked': (BuildContext context) => BlockedUsersView(), '/loading': (BuildContext context) => Loading(), }; } diff --git a/lib/views/widgets/modals/modal-user-details.dart b/lib/views/widgets/modals/modal-user-details.dart index 6d5ee84ad..a19f96262 100644 --- a/lib/views/widgets/modals/modal-user-details.dart +++ b/lib/views/widgets/modals/modal-user-details.dart @@ -1,5 +1,4 @@ // Flutter imports: -import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; @@ -13,10 +12,11 @@ import 'package:syphon/global/colours.dart'; // Project imports: import 'package:syphon/global/dimensions.dart'; +import 'package:syphon/global/print.dart'; import 'package:syphon/global/strings.dart'; -import 'package:syphon/store/alerts/actions.dart'; import 'package:syphon/store/index.dart'; import 'package:syphon/store/rooms/actions.dart'; +import 'package:syphon/store/user/actions.dart'; import 'package:syphon/store/user/model.dart'; import 'package:syphon/store/user/selectors.dart'; import 'package:syphon/views/home/chat/index.dart'; @@ -29,13 +29,11 @@ class ModalUserDetails extends StatelessWidget { ModalUserDetails({ Key key, this.user, - this.roomId, this.userId, }) : super(key: key); final User user; final String userId; - final String roomId; @protected void onNavigateToProfile({BuildContext context, _Props props}) async { @@ -93,7 +91,6 @@ class ModalUserDetails extends StatelessWidget { converter: (Store store) => _Props.mapStateToProps( store, user: user, - roomId: roomId, userId: userId, ), builder: (context, props) => Container( @@ -173,6 +170,10 @@ class ModalUserDetails extends StatelessWidget { child: Column( children: [ ListTile( + onTap: () => this.onMessageUser( + context: context, + props: props, + ), title: Text( 'Send A Message', style: Theme.of(context).textTheme.subtitle1, @@ -189,12 +190,12 @@ class ModalUserDetails extends StatelessWidget { color: Theme.of(context).iconTheme.color, ), ), - onTap: () => this.onMessageUser( + ), + ListTile( + onTap: () => this.onNavigateToInvite( context: context, props: props, ), - ), - ListTile( title: Text( 'Invite To Room', style: Theme.of(context).textTheme.subtitle1, @@ -206,12 +207,12 @@ class ModalUserDetails extends StatelessWidget { size: Dimensions.iconSize, ), ), - onTap: () => this.onNavigateToInvite( + ), + ListTile( + onTap: () => this.onNavigateToProfile( context: context, props: props, ), - ), - ListTile( title: Text( 'View Profile', style: Theme.of(context).textTheme.subtitle1, @@ -223,28 +224,21 @@ class ModalUserDetails extends StatelessWidget { size: Dimensions.iconSize, ), ), - onTap: () => this.onNavigateToProfile( - context: context, - props: props, - ), ), - GestureDetector( - onTap: () => props.onDisabled(), - child: ListTile( - enabled: false, - title: Text( - 'Block', - ), - leading: Container( - padding: EdgeInsets.all(4), - child: Icon( - Icons.block, - size: Dimensions.iconSize, - ), + ListTile( + onTap: () async { + await props.onBlockUser(props.user); + Navigator.pop(context); + }, + title: Text( + props.blocked ? 'Unblock' : 'Block', + ), + leading: Container( + padding: EdgeInsets.all(4), + child: Icon( + Icons.block, + size: Dimensions.iconSize, ), - onTap: () async { - Navigator.pop(context); - }, ), ), ], @@ -258,14 +252,17 @@ class ModalUserDetails extends StatelessWidget { class _Props extends Equatable { final User user; - - final Function onDisabled; + final bool blocked; + final bool loading; + final Function onBlockUser; final Function onCreateChatDirect; _Props({ @required this.user, + @required this.loading, + @required this.blocked, @required this.onCreateChatDirect, - @required this.onDisabled, + @required this.onBlockUser, }); @override @@ -277,7 +274,6 @@ class _Props extends Equatable { Store store, { User user, String userId, - String roomId, }) => _Props( user: () { @@ -291,7 +287,11 @@ class _Props extends Equatable { return store.state.userStore.users[userId]; }(), - onDisabled: () => store.dispatch(addInProgress()), + loading: store.state.userStore.loading, + blocked: store.state.userStore.blocked.contains(userId ?? user.userId), + onBlockUser: (User user) async { + await store.dispatch(toggleBlockUser(user: user)); + }, onCreateChatDirect: ({User user}) async => store.dispatch( createRoom( isDirect: true, From e3e9b84a9dd12e16a71a820eea52e7fd5c628ce8 Mon Sep 17 00:00:00 2001 From: ereio Date: Sun, 13 Dec 2020 23:28:16 -0500 Subject: [PATCH 25/25] bugfixes and touchups --- lib/global/storage/index.dart | 2 +- lib/store/events/actions.dart | 3 --- lib/store/events/storage.dart | 3 ++- lib/store/rooms/room/model.dart | 19 ++++++++++--------- lib/store/sync/actions.dart | 2 +- lib/views/home/index.dart | 8 +++++++- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/global/storage/index.dart b/lib/global/storage/index.dart index ba1106699..1ff7490d7 100644 --- a/lib/global/storage/index.dart +++ b/lib/global/storage/index.dart @@ -116,7 +116,7 @@ Future>> loadStorage(Database storage) async { encrypted: room.encryptionEnabled, ); printError( - '[loadMessages] ${messages[room.id].length.toString()} ${room.name} loaded', + '[loadMessages] ${messages[room.id]?.length} ${room.name} loaded', ); } diff --git a/lib/store/events/actions.dart b/lib/store/events/actions.dart index 20852b9c7..47dfb54cb 100644 --- a/lib/store/events/actions.dart +++ b/lib/store/events/actions.dart @@ -118,9 +118,6 @@ ThunkAction fetchMessageEvents({ "limit": limit, }); - printDebug("[fetchMessageEvents] CALLED FOR ${room.name} ${to} ${from}"); - printJson(messagesJson); - // The token the pagination ends at. If dir=b this token should be used again to request even earlier events. final String end = messagesJson['end']; diff --git a/lib/store/events/storage.dart b/lib/store/events/storage.dart index f0c60cef6..d7941c256 100644 --- a/lib/store/events/storage.dart +++ b/lib/store/events/storage.dart @@ -41,7 +41,8 @@ Future> loadMessages( final store = StoreRef(MESSAGES); // TODO: properly paginate through cold storage messages instead of loading all - final eventIdsPaginated = eventIds; //.skip(offset).take(limit).toList(); + final eventIdsPaginated = + eventIds ?? []; //.skip(offset).take(limit).toList(); final messagesPaginated = await store.records(eventIdsPaginated).get(storage); diff --git a/lib/store/rooms/room/model.dart b/lib/store/rooms/room/model.dart index dea0fba9c..c278fc7cf 100644 --- a/lib/store/rooms/room/model.dart +++ b/lib/store/rooms/room/model.dart @@ -369,7 +369,7 @@ class Room { int lastUpdate = this.lastUpdate; int namePriority = this.namePriority != 4 ? this.namePriority : 4; - var usersAdd = Map.from(this.usersNew); + var usersAdd = Map.from(this.usersNew ?? {}); var userIdsRemove = List(); Set userIds = Set.from(this.userIds ?? []); @@ -508,19 +508,20 @@ class Room { * outside displaying messages */ Room fromMessageEvents({ - List messages, + List messages = const [], String lastHash, String prevHash, // previously fetched hash String nextHash, }) { try { printDebug( - '[fromMessageEvents] ${this.name} ${messages.length.toString()}', + '[fromMessageEvents] ${this.name} ${messages.length}', ); bool limited; int lastUpdate = this.lastUpdate; List outbox = List.from(this.outbox ?? []); + final messageIds = this.messageIds ?? []; // Converting only message events final hasEncrypted = messages.firstWhere( @@ -536,11 +537,11 @@ class Room { // limited indicates need to fetch additional data for room timelines if (this.limited) { // Check to see if the new messages contain those existing in cache - if (messages.isNotEmpty && this.messageIds.isNotEmpty) { - final messageKnown = this.messageIds.firstWhere( - (id) => id == messages[0].id, - orElse: () => null, - ); + if (messages.isNotEmpty && messageIds.isNotEmpty) { + final messageKnown = messageIds.firstWhere( + (id) => id == messages[0].id, + orElse: () => null, + ); // Set limited to false if they now exist limited = messageKnown != null; @@ -572,7 +573,7 @@ class Room { // save messages and unique message id updates final messageIdsNew = Set.from(messagesMap.keys); final messagesNew = List.from(messagesMap.values); - final messageIdsAll = Set.from(this.messageIds) + final messageIdsAll = Set.from(this.messageIds ?? []) ..addAll(messageIdsNew); // Save values to room diff --git a/lib/store/sync/actions.dart b/lib/store/sync/actions.dart index 674938ecd..50805b577 100644 --- a/lib/store/sync/actions.dart +++ b/lib/store/sync/actions.dart @@ -160,10 +160,10 @@ ThunkAction stopSyncObserver() { ThunkAction initialSync() { return (Store store) async { // Start initial sync in background + await store.dispatch(SetSyncing(syncing: true)); await store.dispatch(fetchSync()); // Fetch All Room Ids - continue showing a sync - await store.dispatch(SetSyncing(syncing: true)); await store.dispatch(fetchDirectRooms()); await store.dispatch(fetchRooms()); await store.dispatch(SetSyncing(syncing: false)); diff --git a/lib/views/home/index.dart b/lib/views/home/index.dart index a55e03b21..2eb79ae67 100644 --- a/lib/views/home/index.dart +++ b/lib/views/home/index.dart @@ -650,6 +650,7 @@ class _Props extends Equatable { unauthed: store.state.syncStore.unauthed, offline: store.state.syncStore.offline, syncing: () { + final synced = store.state.syncStore.synced; final syncing = store.state.syncStore.syncing; final offline = store.state.syncStore.offline; final backgrounded = store.state.syncStore.backgrounded; @@ -658,11 +659,16 @@ class _Props extends Equatable { final lastAttempt = DateTime.fromMillisecondsSinceEpoch( store.state.syncStore.lastAttempt ?? 0); - // See if the last attempted sync is older than 60 seconds + // See if the last attempted sy nc is older than 60 seconds final isLastAttemptOld = DateTime.now() .difference(lastAttempt) .compareTo(Duration(seconds: 90)); + // syncing for the first time + if (syncing && !synced) { + return true; + } + // syncing for the first time since going offline if (syncing && offline) { return true;