New feature: Automatic data compression to reduce amount of traffic a…

…nd the usage of remote database.
vrtmrz · Apr 18, 2024 · 84adec4 · 84adec4
1 parent f0b202b
commit 84adec4
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 3 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -55,6 +55,7 @@
     },
     "dependencies": {
         "diff-match-patch": "^1.0.5",
+        "fflate": "^0.8.2",
         "idb": "^8.0.0",
         "minimatch": "^9.0.3",
         "xxhash-wasm": "0.4.2",

diff --git a/src/ObsidianLiveSyncSettingTab.ts b/src/ObsidianLiveSyncSettingTab.ts
@@ -544,6 +544,19 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
             text: "",
         });
 
+
+        containerRemoteDatabaseEl.createEl("h4", { text: "Effective Storage Using" });
+        new Setting(containerRemoteDatabaseEl)
+            .setName("Data Compression (Experimental)")
+            .setDesc("Compresses data during transfer, saving space in the remote database. Note: Please ensure that all devices have v0.22.18 and connected tools are also supported compression.")
+            .addToggle((toggle) =>
+                toggle.setValue(this.plugin.settings.enableCompression).onChange(async (value) => {
+                    this.plugin.settings.enableCompression = value;
+                    await this.plugin.saveSettings();
+                    this.display();
+                })
+            );
+
         containerRemoteDatabaseEl.createEl("h4", { text: "Confidentiality" });
 
         const e2e = new Setting(containerRemoteDatabaseEl)

diff --git a/src/lib b/src/lib
diff --git a/src/main.ts b/src/main.ts
@@ -12,7 +12,7 @@ import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab";
 import { DocumentHistoryModal } from "./DocumentHistoryModal";
 import { applyPatch, cancelAllPeriodicTask, cancelAllTasks, cancelTask, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, flattenObject, path2id, scheduleTask, tryParseJSON, isValidPath, isInternalMetadata, isPluginMetadata, stripInternalMetadataPrefix, isChunk, askSelectString, askYesNo, askString, PeriodicProcessor, getPath, getPathWithoutPrefix, getPathFromTFile, performRebuildDB, memoIfNotExist, memoObject, retrieveMemoObject, disposeMemoObject, isCustomisationSyncMetadata, compareFileFreshness, BASE_IS_NEW, TARGET_IS_NEW, EVEN, compareMTime, markChangesAreSame } from "./utils";
 import { encrypt, tryDecrypt } from "./lib/src/e2ee_v2";
-import { balanceChunkPurgedDBs, enableEncryption, isCloudantURI, isErrorOfMissingDoc, isValidRemoteCouchDBURI, purgeUnreferencedChunks } from "./lib/src/utils_couchdb";
+import { balanceChunkPurgedDBs, enableCompression, enableEncryption, isCloudantURI, isErrorOfMissingDoc, isValidRemoteCouchDBURI, purgeUnreferencedChunks } from "./lib/src/utils_couchdb";
 import { logStore, type LogEntry, collectingChunks, pluginScanningCount, hiddenFilesProcessingCount, hiddenFilesEventCount, logMessages } from "./lib/src/stores";
 import { setNoticeClass } from "./lib/src/wrapper";
 import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/strbin";
@@ -119,7 +119,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
     requestCount = reactiveSource(0);
     responseCount = reactiveSource(0);
     processReplication = (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => this.parseReplicationResult(e);
-    async connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean, performSetup: boolean, skipInfo: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
+    async connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean, performSetup: boolean, skipInfo: boolean, compression: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
         if (!isValidRemoteCouchDBURI(uri)) return "Remote URI is not valid";
         if (uri.toLowerCase() != uri) return "Remote URI and database name could not contain capital letters.";
         if (uri.indexOf(" ") !== -1) return "Remote URI and database name could not contain spaces.";
@@ -237,6 +237,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         };
 
         const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf);
+        enableCompression(db, compression);
         if (passphrase !== "false" && typeof passphrase === "string") {
             enableEncryption(db, passphrase, useDynamicIterationCount, false);
         }
+3 −1		src/LiveSyncReplicator.ts
+13 −0		src/strbin.ts
+2 −0		src/types.ts
+72 −2		src/utils_couchdb.ts