Fixed:

- Journal Sync will not hang up during big replication, especially the initial one. - All changes which have been replicated while rebuilding will not be postponed (Previous behaviour). Improved: - Now Journal Sync works efficiently in download and parse, or pack and upload. - Less server storage and faster packing/unpacking usage by the new chunk format.
vrtmrz · Apr 30, 2024 · daa3fee · daa3fee
1 parent 7b5f7d0
commit daa3fee
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 24 deletions.
diff --git a/src/CmdSetupLiveSync.ts b/src/CmdSetupLiveSync.ts
@@ -1,4 +1,4 @@
-import { type EntryDoc, type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, REMOTE_COUCHDB } from "./lib/src/types";
+import { type EntryDoc, type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, REMOTE_COUCHDB, REMOTE_MINIO } from "./lib/src/types";
 import { configURIBase } from "./types";
 import { Logger } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
@@ -312,15 +312,15 @@ Of course, we are able to disable these features.`
     }
     async suspendReflectingDatabase() {
         if (this.plugin.settings.doNotSuspendOnFetching) return;
-        // if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
+        if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
         Logger(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL_NOTICE);
         this.plugin.settings.suspendParseReplicationResult = true;
         this.plugin.settings.suspendFileWatching = true;
         await this.plugin.saveSettings();
     }
     async resumeReflectingDatabase() {
         if (this.plugin.settings.doNotSuspendOnFetching) return;
-        // if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
+        if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
         Logger(`Database and storage reflection has been resumed!`, LOG_LEVEL_NOTICE);
         this.plugin.settings.suspendParseReplicationResult = false;
         this.plugin.settings.suspendFileWatching = false;

diff --git a/src/KeyValueDB.ts b/src/KeyValueDB.ts
@@ -1,8 +1,8 @@
 import { deleteDB, type IDBPDatabase, openDB } from "idb";
 export interface KeyValueDatabase {
-    get<T>(key: string): Promise<T>;
-    set<T>(key: string, value: T): Promise<IDBValidKey>;
-    del(key: string): Promise<void>;
+    get<T>(key: IDBValidKey): Promise<T>;
+    set<T>(key: IDBValidKey, value: T): Promise<IDBValidKey>;
+    del(key: IDBValidKey): Promise<void>;
     clear(): Promise<void>;
     keys(query?: IDBValidKey | IDBKeyRange, count?: number): Promise<IDBValidKey[]>;
     close(): void;
@@ -23,20 +23,20 @@ export const OpenKeyValueDatabase = async (dbKey: string): Promise<KeyValueDatab
     const db = await dbPromise;
     databaseCache[dbKey] = db;
     return {
-        get<T>(key: string): Promise<T> {
-            return db.get(storeKey, key);
+        async get<T>(key: IDBValidKey): Promise<T> {
+            return await db.get(storeKey, key);
         },
-        set<T>(key: string, value: T) {
-            return db.put(storeKey, value, key);
+        async set<T>(key: IDBValidKey, value: T) {
+            return await db.put(storeKey, value, key);
         },
-        del(key: string) {
-            return db.delete(storeKey, key);
+        async del(key: IDBValidKey) {
+            return await db.delete(storeKey, key);
         },
-        clear() {
-            return db.clear(storeKey);
+        async clear() {
+            return await db.clear(storeKey);
         },
-        keys(query?: IDBValidKey | IDBKeyRange, count?: number) {
-            return db.getAllKeys(storeKey, query, count);
+        async keys(query?: IDBValidKey | IDBKeyRange, count?: number) {
+            return await db.getAllKeys(storeKey, query, count);
         },
         close() {
             delete databaseCache[dbKey];

diff --git a/src/ObsidianLiveSyncSettingTab.ts b/src/ObsidianLiveSyncSettingTab.ts
@@ -2331,7 +2331,7 @@ ${stringifyYaml(pluginConfig)}`;
                         .setWarning()
                         .setDisabled(false)
                         .onClick(async () => {
-                            await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, receivedFiles: [], knownIDs: [] }));
+                            await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, receivedFiles: new Set(), knownIDs: new Set() }));
                             Logger(`Journal received history has been cleared.`, LOG_LEVEL_NOTICE);
                         })
                 )
@@ -2344,7 +2344,7 @@ ${stringifyYaml(pluginConfig)}`;
                         .setWarning()
                         .setDisabled(false)
                         .onClick(async () => {
-                            await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, lastLocalSeq: 0, sentIDs: [], sentFiles: [] }));
+                            await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, lastLocalSeq: 0, sentIDs: new Set(), sentFiles: new Set() }));
                             Logger(`Journal sent history has been cleared.`, LOG_LEVEL_NOTICE);
                         })
                 )
@@ -2357,10 +2357,24 @@ ${stringifyYaml(pluginConfig)}`;
                         .setWarning()
                         .setDisabled(false)
                         .onClick(async () => {
-                            await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, receivedFiles: [], knownIDs: [], lastLocalSeq: 0, sentIDs: [], sentFiles: [] }));
+                            await this.plugin.getMinioJournalSyncClient().resetCheckpointInfo();
                             Logger(`Journal exchange history has been cleared.`, LOG_LEVEL_NOTICE);
                         })
                 )
+            new Setting(containerMaintenanceEl)
+                .setName("Purge all journal counter")
+                .setDesc("Purge all sending and downloading cache.")
+                .addButton((button) =>
+                    button
+                        .setButtonText("Reset all")
+                        .setWarning()
+                        .setDisabled(false)
+                        .onClick(async () => {
+                            await this.plugin.getMinioJournalSyncClient().resetAllCaches();
+                            Logger(`Journal sending and downloading cache has been cleared.`, LOG_LEVEL_NOTICE);
+                        })
+                )
+
             new Setting(containerMaintenanceEl)
                 .setName("Make empty the bucket")
                 .setDesc("Delete all data on the remote.")
@@ -2370,7 +2384,7 @@ ${stringifyYaml(pluginConfig)}`;
                         .setWarning()
                         .setDisabled(false)
                         .onClick(async () => {
-                            await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, receivedFiles: [], knownIDs: [], lastLocalSeq: 0, sentIDs: [], sentFiles: [] }));
+                            await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, receivedFiles: new Set(), knownIDs: new Set(), lastLocalSeq: 0, sentIDs: new Set(), sentFiles: new Set() }));
                             await this.plugin.resetRemoteBucket();
                             Logger(`the bucket has been cleared.`, LOG_LEVEL_NOTICE);
                         })

diff --git a/src/lib b/src/lib
diff --git a/src/main.ts b/src/main.ts
@@ -4,7 +4,7 @@ import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stri
 import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
 import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, REMOTE_MINIO, REMOTE_COUCHDB, type BucketSyncSetting, } from "./lib/src/types";
 import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
-import { arrayToChunkedArray, createBlob, delay, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle } from "./lib/src/utils";
+import { arrayToChunkedArray, createBlob, delay, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle, type SimpleStore } from "./lib/src/utils";
 import { Logger, setGlobalLogFunction } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
 import { ConflictResolveModal } from "./ConflictResolveModal";
@@ -37,7 +37,7 @@ import { initializeStores } from "./stores.js";
 import { JournalSyncMinio } from "./lib/src/JournalSyncMinio.js";
 import { LiveSyncJournalReplicator, type LiveSyncJournalReplicatorEnv } from "./lib/src/LiveSyncJournalReplicator.js";
 import { LiveSyncCouchDBReplicator, type LiveSyncCouchDBReplicatorEnv } from "./lib/src/LiveSyncReplicator.js";
-import type { CheckPointInfo, SimpleStore } from "./lib/src/JournalSyncTypes.js";
+import type { CheckPointInfo } from "./lib/src/JournalSyncTypes.js";
 import { ObsHttpHandler } from "./ObsHttpHandler.js";
 
 setNoticeClass(Notice);
@@ -477,7 +477,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         },
         keys: async (from: string | undefined, to: string | undefined, count?: number | undefined): Promise<string[]> => {
             const ret = this.kvDB.keys(IDBKeyRange.bound(`os-${from || ""}`, `os-${to || ""}`), count);
-            return (await ret).map(e => e.toString());
+            return (await ret).map(e => e.toString()).filter(e => e.startsWith("os-")).map(e => e.substring(3));
         }
     }
     getMinioJournalSyncClient() {
+369 −254		src/JournalSyncAbstract.ts
+8 −14		src/JournalSyncTypes.ts
+6 −2		src/LiveSyncJournalReplicator.ts
+161 −0		src/memutil.ts
+1 −1		src/processor.ts
+46 −0		src/utils.ts