v0.24.2

Rewritten

-   Hidden File Sync is now respects the file changes on the storage. Not simply comparing modified times.
    -   This makes hidden file sync more robust and reliable.

Fixed

-   `Scan hidden files before replication` is now configurable again.
-   Some unexpected errors are now handled more gracefully.
-   Meaningless event passing during boot sequence is now prevented.
-   Error handling for non-existing files has been fixed.
-   Hidden files will not be batched to avoid the potential error.
    -   This behaviour had been causing the error in the previous versions in specific situations.
-   The log which checking automatic conflict resolution is now in verbose level.
-   Replication log (skipping non-targetting files) shows the correct information.
-   The dialogue that asking enabling optional feature during `Rebuild Everything` now prevents to show the `overwrite` option.
    -   The rebuilding device is the first, meaningless.
-   Files with different modified time but identical content are no longer processed repeatedly.
-   Some unexpected errors which caused after terminating plug-in are now avoided.
-

Improved

-   JSON files are now more transferred efficiently.
    -   Now the JSON files are transferred in more fine chunks, which makes the transfer more efficient.

src/common/utils.ts

@@ -11,6 +11,8 @@ import {
 
 import { Logger } from "../lib/src/common/logger.ts";
 import {
+    LOG_LEVEL_INFO,
+    LOG_LEVEL_NOTICE,
     LOG_LEVEL_VERBOSE,
     type AnyEntry,
     type DocumentID,
@@ -25,15 +27,11 @@ import type ObsidianLiveSyncPlugin from "../main.ts";
 import { writeString } from "../lib/src/string_and_binary/convert.ts";
 import { fireAndForget } from "../lib/src/common/utils.ts";
 import { sameChangePairs } from "./stores.ts";
+import type { KeyValueDatabase } from "./KeyValueDB.ts";
+import { scheduleTask } from "octagonal-wheels/concurrency/task";
+import { EVENT_PLUGIN_UNLOADED, eventHub } from "./events.ts";
 
-export {
-    scheduleTask,
-    setPeriodicTask,
-    cancelTask,
-    cancelAllTasks,
-    cancelPeriodicTask,
-    cancelAllPeriodicTask,
-} from "../lib/src/concurrency/task.ts";
+export { scheduleTask, cancelTask, cancelAllTasks } from "../lib/src/concurrency/task.ts";
 
 // For backward compatibility, using the path for determining id.
 // Only CouchDB unacceptable ID (that starts with an underscore) has been prefixed with "/".
@@ -72,10 +70,25 @@ export function getPathWithoutPrefix(entry: AnyEntry) {
 export function getPathFromTFile(file: TAbstractFile) {
     return file.path as FilePath;
 }
+
+export function isInternalFile(file: UXFileInfoStub | string | FilePathWithPrefix) {
+    if (typeof file == "string") return file.startsWith(ICHeader);
+    if (file.isInternal) return true;
+    return false;
+}
 export function getPathFromUXFileInfo(file: UXFileInfoStub | string | FilePathWithPrefix) {
     if (typeof file == "string") return file as FilePathWithPrefix;
     return file.path;
 }
+export function getStoragePathFromUXFileInfo(file: UXFileInfoStub | string | FilePathWithPrefix) {
+    if (typeof file == "string") return stripAllPrefixes(file as FilePathWithPrefix);
+    return stripAllPrefixes(file.path);
+}
+export function getDatabasePathFromUXFileInfo(file: UXFileInfoStub | string | FilePathWithPrefix) {
+    const prefix = isInternalFile(file) ? ICHeader : "";
+    if (typeof file == "string") return (prefix + stripAllPrefixes(file as FilePathWithPrefix)) as FilePathWithPrefix;
+    return (prefix + stripAllPrefixes(file.path)) as FilePathWithPrefix;
+}
 
 const memos: { [key: string]: any } = {};
 export function memoObject<T>(key: string, obj: T): T {
@@ -148,11 +161,14 @@ export function isCustomisationSyncMetadata(str: string): boolean {
 
 export class PeriodicProcessor {
     _process: () => Promise<any>;
-    _timer?: number;
+    _timer?: number = undefined;
     _plugin: ObsidianLiveSyncPlugin;
     constructor(plugin: ObsidianLiveSyncPlugin, process: () => Promise<any>) {
         this._plugin = plugin;
         this._process = process;
+        eventHub.onceEvent(EVENT_PLUGIN_UNLOADED, () => {
+            this.disable();
+        });
     }
     async process() {
         try {
@@ -265,18 +281,28 @@ export function compareMTime(
     throw new Error("Unexpected error");
 }
 
+function getKey(file: AnyEntry | string | UXFileInfoStub) {
+    const key = typeof file == "string" ? file : stripAllPrefixes(file.path);
+    return key;
+}
+
 export function markChangesAreSame(file: AnyEntry | string | UXFileInfoStub, mtime1: number, mtime2: number) {
     if (mtime1 === mtime2) return true;
-    const key = typeof file == "string" ? file : "_id" in file ? file._id : file.path;
+    const key = getKey(file);
     const pairs = sameChangePairs.get(key, []) || [];
     if (pairs.some((e) => e == mtime1 || e == mtime2)) {
         sameChangePairs.set(key, [...new Set([...pairs, mtime1, mtime2])]);
     } else {
         sameChangePairs.set(key, [mtime1, mtime2]);
     }
 }
+
+export function unmarkChanges(file: AnyEntry | string | UXFileInfoStub) {
+    const key = getKey(file);
+    sameChangePairs.delete(key);
+}
 export function isMarkedAsSameChanges(file: UXFileInfoStub | AnyEntry | string, mtimes: number[]) {
-    const key = typeof file == "string" ? file : "_id" in file ? file._id : file.path;
+    const key = getKey(file);
     const pairs = sameChangePairs.get(key, []) || [];
     if (mtimes.every((e) => pairs.indexOf(e) !== -1)) {
         return EVEN;
@@ -374,6 +400,95 @@ export function displayRev(rev: string) {
     return `${number}-${hash.substring(0, 6)}`;
 }
 
-// export function getPathFromUXFileInfo(file: UXFileInfoStub | UXFileInfo | string) {
-//     return (typeof file == "string" ? file : file.path) as FilePathWithPrefix;
-// }
+type DocumentProps = {
+    id: DocumentID;
+    rev?: string;
+    prefixedPath: FilePathWithPrefix;
+    path: FilePath;
+    isDeleted: boolean;
+    revDisplay: string;
+    shortenedId: string;
+    shortenedPath: string;
+};
+
+export function getDocProps(doc: AnyEntry): DocumentProps {
+    const id = doc._id;
+    const shortenedId = id.substring(0, 10);
+    const prefixedPath = getPath(doc);
+    const path = stripAllPrefixes(prefixedPath);
+    const rev = doc._rev;
+    const revDisplay = rev ? displayRev(rev) : "0-NOREVS";
+    // const prefix = prefixedPath.substring(0, prefixedPath.length - path.length);
+    const shortenedPath = path.substring(0, 10);
+    const isDeleted = doc._deleted || doc.deleted || false;
+    return { id, rev, revDisplay, prefixedPath, path, isDeleted, shortenedId, shortenedPath };
+}
+
+export function getLogLevel(showNotice: boolean) {
+    return showNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
+}
+
+export type MapLike<K, V> = {
+    set(key: K, value: V): Map<K, V>;
+    clear(): void;
+    delete(key: K): boolean;
+    get(key: K): V | undefined;
+    has(key: K): boolean;
+    keys: () => IterableIterator<K>;
+    get size(): number;
+};
+
+export async function autosaveCache<K, V>(db: KeyValueDatabase, mapKey: string): Promise<MapLike<K, V>> {
+    const savedData = (await db.get<Map<K, V>>(mapKey)) ?? new Map<K, V>();
+    const _commit = () => {
+        try {
+            scheduleTask("commit-map-save-" + mapKey, 250, async () => {
+                await db.set(mapKey, savedData);
+            });
+        } catch {
+            // NO OP.
+        }
+    };
+    return {
+        set(key: K, value: V) {
+            const modified = savedData.get(key) !== value;
+            const result = savedData.set(key, value);
+            if (modified) {
+                _commit();
+            }
+            return result;
+        },
+        clear(): void {
+            savedData.clear();
+            _commit();
+        },
+        delete(key: K): boolean {
+            const result = savedData.delete(key);
+            if (result) {
+                _commit();
+            }
+            return result;
+        },
+        get(key: K): V | undefined {
+            return savedData.get(key);
+        },
+        has(key) {
+            return savedData.has(key);
+        },
+        keys() {
+            return savedData.keys();
+        },
+        get size() {
+            return savedData.size;
+        },
+    };
+}
+
+export function onlyInNTimes(n: number, proc: (progress: number) => any) {
+    let counter = 0;
+    return function () {
+        if (counter++ % n == 0) {
+            proc(counter);
+        }
+    };
+}

src/features/ConfigSync/PluginPane.svelte

@@ -1,11 +1,24 @@
 <script lang="ts">
     import { onMount } from "svelte";
     import ObsidianLiveSyncPlugin from "../../main";
-    import { ConfigSync, type IPluginDataExDisplay, pluginIsEnumerating, pluginList, pluginManifestStore, pluginV2Progress } from "./CmdConfigSync.ts";
+    import {
+        ConfigSync,
+        type IPluginDataExDisplay,
+        pluginIsEnumerating,
+        pluginList,
+        pluginManifestStore,
+        pluginV2Progress,
+    } from "./CmdConfigSync.ts";
     import PluginCombo from "./PluginCombo.svelte";
     import { Menu, type PluginManifest } from "obsidian";
     import { unique } from "../../lib/src/common/utils";
-    import { MODE_SELECTIVE, MODE_AUTOMATIC, MODE_PAUSED, type SYNC_MODE, type PluginSyncSettingEntry, MODE_SHINY } from "../../lib/src/common/types";
+    import {
+        MODE_SELECTIVE,
+        MODE_AUTOMATIC,
+        MODE_PAUSED,
+        type SYNC_MODE,
+        MODE_SHINY,
+    } from "../../lib/src/common/types";
     import { normalizePath } from "../../deps";
     import { HiddenFileSync } from "../HiddenFileSync/CmdHiddenFileSync.ts";
     import { LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger";
@@ -16,13 +29,15 @@
 
     const addOn = plugin.getAddOn(ConfigSync.name) as ConfigSync;
     if (!addOn) {
-        const msg = "AddOn Module (ConfigSync) has not been loaded. This is very unexpected situation. Please report this issue.";
+        const msg =
+            "AddOn Module (ConfigSync) has not been loaded. This is very unexpected situation. Please report this issue.";
         Logger(msg, LOG_LEVEL_NOTICE);
         throw new Error(msg);
     }
     const addOnHiddenFileSync = plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync;
     if (!addOnHiddenFileSync) {
-        const msg = "AddOn Module (HiddenFileSync) has not been loaded. This is very unexpected situation. Please report this issue.";
+        const msg =
+            "AddOn Module (HiddenFileSync) has not been loaded. This is very unexpected situation. Please report this issue.";
         Logger(msg, LOG_LEVEL_NOTICE);
         throw new Error(msg);
     }
@@ -99,7 +114,11 @@
     async function applyData(data: IPluginDataExDisplay): Promise<boolean> {
         return await addOn.applyData(data);
     }
-    async function compareData(docA: IPluginDataExDisplay, docB: IPluginDataExDisplay, compareEach = false): Promise<boolean> {
+    async function compareData(
+        docA: IPluginDataExDisplay,
+        docB: IPluginDataExDisplay,
+        compareEach = false
+    ): Promise<boolean> {
         return await addOn.compareUsingDisplayData(docA, docB, compareEach);
     }
     async function deleteData(data: IPluginDataExDisplay): Promise<boolean> {
@@ -130,7 +149,7 @@
         setMode(key, MODE_AUTOMATIC);
         const configDir = normalizePath(plugin.app.vault.configDir);
         const files = (plugin.settings.pluginSyncExtendedSetting[key]?.files ?? []).map((e) => `${configDir}/${e}`);
-        addOnHiddenFileSync.syncInternalFilesAndDatabase(direction, true, false, files);
+        addOnHiddenFileSync.initialiseInternalFileSync(direction, true, files);
     }
     function askOverwriteModeForAutomatic(evt: MouseEvent, key: string) {
         const menu = new Menu();
@@ -199,7 +218,7 @@
                 .filter((e) => `${e.category}/${e.name}` == key)
                 .map((e) => e.files)
                 .flat()
-                .map((e) => e.filename),
+                .map((e) => e.filename)
         );
         if (mode == MODE_SELECTIVE) {
             automaticList.delete(key);
@@ -249,7 +268,15 @@
                 .map((e) => ({ category: e[0], name: e[1], displayName: e[1] })),
         ]
             .sort((a, b) => (a.displayName ?? a.name).localeCompare(b.displayName ?? b.name))
-            .reduce((p, c) => ({ ...p, [c.category]: unique(c.category in p ? [...p[c.category], c.displayName ?? c.name] : [c.displayName ?? c.name]) }), {} as Record<string, string[]>);
+            .reduce(
+                (p, c) => ({
+                    ...p,
+                    [c.category]: unique(
+                        c.category in p ? [...p[c.category], c.displayName ?? c.name] : [c.displayName ?? c.name]
+                    ),
+                }),
+                {} as Record<string, string[]>
+            );
     }
     $: {
         displayKeys = computeDisplayKeys(list);
@@ -337,7 +364,12 @@
                         </div>
                         <div class="body">
                             {#if mode == MODE_SELECTIVE || mode == MODE_SHINY}
-                                <PluginCombo {...options} isFlagged={mode == MODE_SHINY} list={list.filter((e) => e.category == key && e.name == name)} hidden={false} />
+                                <PluginCombo
+                                    {...options}
+                                    isFlagged={mode == MODE_SHINY}
+                                    list={list.filter((e) => e.category == key && e.name == name)}
+                                    hidden={false}
+                                />
                             {:else}
                                 <div class="statusnote">{TITLES[mode]}</div>
                             {/if}
@@ -359,7 +391,10 @@
                 {@const modeEtc = automaticListDisp.get(bindKeyETC) ?? MODE_SELECTIVE}
                 <div class="labelrow {hideEven ? 'hideeven' : ''}">
                     <div class="title">
-                        <button class="status" on:click={(evt) => askMode(evt, `${PREFIX_PLUGIN_ALL}/${name}`, bindKeyAll)}>
+                        <button
+                            class="status"
+                            on:click={(evt) => askMode(evt, `${PREFIX_PLUGIN_ALL}/${name}`, bindKeyAll)}
+                        >
                             {getIcon(modeAll)}
                         </button>
                         <span class="name">{nameMap.get(`plugins/${name}`) || name}</span>
@@ -373,29 +408,45 @@
                 {#if modeAll == MODE_SELECTIVE || modeAll == MODE_SHINY}
                     <div class="filerow {hideEven ? 'hideeven' : ''}">
                         <div class="filetitle">
-                            <button class="status" on:click={(evt) => askMode(evt, `${PREFIX_PLUGIN_MAIN}/${name}/MAIN`, bindKeyMain)}>
+                            <button
+                                class="status"
+                                on:click={(evt) => askMode(evt, `${PREFIX_PLUGIN_MAIN}/${name}/MAIN`, bindKeyMain)}
+                            >
                                 {getIcon(modeMain)}
                             </button>
                             <span class="name">MAIN</span>
                         </div>
                         <div class="body">
                             {#if modeMain == MODE_SELECTIVE || modeMain == MODE_SHINY}
-                                <PluginCombo {...options} isFlagged={modeMain == MODE_SHINY} list={filterList(listX, ["PLUGIN_MAIN"])} hidden={false} />
+                                <PluginCombo
+                                    {...options}
+                                    isFlagged={modeMain == MODE_SHINY}
+                                    list={filterList(listX, ["PLUGIN_MAIN"])}
+                                    hidden={false}
+                                />
                             {:else}
                                 <div class="statusnote">{TITLES[modeMain]}</div>
                             {/if}
                         </div>
                     </div>
                     <div class="filerow {hideEven ? 'hideeven' : ''}">
                         <div class="filetitle">
-                            <button class="status" on:click={(evt) => askMode(evt, `${PREFIX_PLUGIN_DATA}/${name}`, bindKeyData)}>
+                            <button
+                                class="status"
+                                on:click={(evt) => askMode(evt, `${PREFIX_PLUGIN_DATA}/${name}`, bindKeyData)}
+                            >
                                 {getIcon(modeData)}
                             </button>
                             <span class="name">DATA</span>
                         </div>
                         <div class="body">
                             {#if modeData == MODE_SELECTIVE || modeData == MODE_SHINY}
-                                <PluginCombo {...options} isFlagged={modeData == MODE_SHINY} list={filterList(listX, ["PLUGIN_DATA"])} hidden={false} />
+                                <PluginCombo
+                                    {...options}
+                                    isFlagged={modeData == MODE_SHINY}
+                                    list={filterList(listX, ["PLUGIN_DATA"])}
+                                    hidden={false}
+                                />
                             {:else}
                                 <div class="statusnote">{TITLES[modeData]}</div>
                             {/if}
@@ -404,14 +455,22 @@
                     {#if useSyncPluginEtc}
                         <div class="filerow {hideEven ? 'hideeven' : ''}">
                             <div class="filetitle">
-                                <button class="status" on:click={(evt) => askMode(evt, `${PREFIX_PLUGIN_ETC}/${name}`, bindKeyETC)}>
+                                <button
+                                    class="status"
+                                    on:click={(evt) => askMode(evt, `${PREFIX_PLUGIN_ETC}/${name}`, bindKeyETC)}
+                                >
                                     {getIcon(modeEtc)}
                                 </button>
                                 <span class="name">Other files</span>
                             </div>
                             <div class="body">
                                 {#if modeEtc == MODE_SELECTIVE || modeEtc == MODE_SHINY}
-                                    <PluginCombo {...options} isFlagged={modeEtc == MODE_SHINY} list={filterList(listX, ["PLUGIN_ETC"])} hidden={false} />
+                                    <PluginCombo
+                                        {...options}
+                                        isFlagged={modeEtc == MODE_SHINY}
+                                        list={filterList(listX, ["PLUGIN_ETC"])}
+                                        hidden={false}
+                                    />
                                 {:else}
                                     <div class="statusnote">{TITLES[modeEtc]}</div>
                                 {/if}