Skip to content

Commit

Permalink
New feature:
Browse files Browse the repository at this point in the history 
- Now we are ready for i18n.
- The setting dialogue has been refined. Very controllable, clearly displayed disabled items, and ready to i18n.
Fixed:
- Many memory leaks have been rescued.
- Chunk caches now work well.
- Many trivial but potential bugs are fixed.
- No longer error messages will be shown on retrieving checkpoint or server information.
- Now we can check and correct tweak mismatch during the setup
Improved:
- Customisation synchronisation has got more smoother.
Tidied
- Practically unused functions have been removed or are being prepared for removal.
- Many of the type-errors and lint errors have been corrected.
- Unused files have been removed.
Note:
- From this version, some test files have been included. However, they are not enabled and released in the release build.
  • Loading branch information
@vrtmrz
vrtmrz committed May 22, 2024
1 parent 7b0ac22 commit b3a85c5
Show file tree
Hide file tree
Showing 19 changed files with 2,105 additions and 1,711 deletions.
6 changes: 4 additions & 2 deletions esbuild.config.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ if you want to view the source, please visit the github repository of this plugi


const prod = process.argv[2] === "production";
const keepTest = !prod;

const terserOpt = {
sourceMap: (!prod ? {
Expand Down Expand Up @@ -142,16 +143,17 @@ const context = await esbuild.context({
sourcemap: prod ? false : "inline",
treeShaking: true,
outfile: "main_org.js",
mainFields: ["browser", "module", "main"],
minifyWhitespace: false,
minifySyntax: false,
minifyIdentifiers: false,
minify: false,

dropLabels: prod && !keepTest ? ["TEST", "DEV"] : [],
// keepNames: true,
plugins: [
sveltePlugin({
preprocess: sveltePreprocess(),
compilerOptions: { css: true, preserveComments: true },
compilerOptions: { css: "injected", preserveComments: false },
}),
...plugins
],
Expand Down
1 change: 1 addition & 0 deletions src/common/ObsHttpHandler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export class ObsHttpHandler extends FetchHttpHandler {
options === undefined ? undefined : options.requestTimeout;
this.reverseProxyNoSignUrl = reverseProxyNoSignUrl;
}
// eslint-disable-next-line require-await
async handle(
request: HttpRequest,
{ abortSignal }: HttpHandlerOptions = {}
Expand Down
140 changes: 78 additions & 62 deletions src/features/CmdConfigSync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import { Notice, type PluginManifest, parseYaml, normalizePath, type ListedFiles
import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry, SavingEntry } from "../lib/src/common/types.ts";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "../lib/src/common/types.ts";
import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "../common/types.ts";
import { createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocData, isDocContentSame, throttle } from "../lib/src/common/utils.ts";
import { createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocDataAsArray, isDocContentSame, throttle } from "../lib/src/common/utils.ts";
import { Logger } from "../lib/src/common/logger.ts";
import { readString, decodeBinary, arrayBufferToBase64, digestHash } from "../lib/src/string_and_binary/strbin.ts";
import { serialized } from "../lib/src/concurrency/lock.ts";
import { serialized, shareRunningResult } from "../lib/src/concurrency/lock.ts";
import { LiveSyncCommands } from "./LiveSyncCommands.ts";
import { stripAllPrefixes } from "../lib/src/string_and_binary/path.ts";
import { PeriodicProcessor, disposeMemoObject, memoIfNotExist, memoObject, retrieveMemoObject, scheduleTask } from "../common/utils.ts";
Expand All @@ -19,6 +19,8 @@ import type ObsidianLiveSyncPlugin from '../main.ts';

const d = "\u200b";
const d2 = "\n";
const delimiters = /(?<=[\n|\u200b])/g;


function serialize(data: PluginDataEx): string {
// For higher performance, create custom plug-in data strings.
Expand All @@ -30,7 +32,7 @@ function serialize(data: PluginDataEx): string {
ret += data.mtime + d2;
for (const file of data.files) {
ret += file.filename + d + (file.displayName ?? "") + d + (file.version ?? "") + d2;
const hash = digestHash((file.data ?? []).join());
const hash = digestHash((file.data ?? []));
ret += file.mtime + d + file.size + d + hash + d2;
for (const data of file.data ?? []) {
ret += data + d
Expand All @@ -39,41 +41,49 @@ function serialize(data: PluginDataEx): string {
}
return ret;
}
function fetchToken(source: string, from: number): [next: number, token: string] {
const limitIdx = source.indexOf(d2, from);
const limit = limitIdx == -1 ? source.length : limitIdx;
const delimiterIdx = source.indexOf(d, from);
const delimiter = delimiterIdx == -1 ? source.length : delimiterIdx;
const tokenEnd = Math.min(limit, delimiter);
let next = tokenEnd;
if (limit < delimiter) {
next = tokenEnd;
} else {
next = tokenEnd + 1
}
return [next, source.substring(from, tokenEnd)];
}
function getTokenizer(source: string) {

function getTokenizer(source: string[]) {
const sources = source.flatMap(e => e.split(delimiters))
sources[0] = sources[0].substring(1);
let pos = 0;
let lineRunOut = false;
const t = {
pos: 1,
next() {
const [next, token] = fetchToken(source, this.pos);
this.pos = next;
return token;
next(): string {
if (lineRunOut) {
return "";
}
if (pos >= sources.length) {
return "";
}
const item = sources[pos];
if (!item.endsWith(d2)) {
pos++;
} else {
lineRunOut = true;
}
if (item.endsWith(d) || item.endsWith(d2)) {
return item.substring(0, item.length - 1);
} else {
return item + this.next();
}
},
nextLine() {
const nextPos = source.indexOf(d2, this.pos);
if (nextPos == -1) {
this.pos = source.length;
if (lineRunOut) {
pos++;
} else {
this.pos = nextPos + 1;
while (!sources[pos].endsWith(d2)) {
pos++;
if (pos >= sources.length) break;
}
pos++;
}
lineRunOut = false;
}
}
return t;
}

function deserialize2(str: string): PluginDataEx {
function deserialize2(str: string[]): PluginDataEx {
const tokens = getTokenizer(str);
const ret = {} as PluginDataEx;
const category = tokens.next();
Expand Down Expand Up @@ -120,13 +130,16 @@ function deserialize2(str: string): PluginDataEx {
return result;
}

function deserialize<T>(str: string, def: T) {
function deserialize<T>(str: string[], def: T) {
try {
if (str[0] == ":") return deserialize2(str);
return JSON.parse(str) as T;
if (str[0][0] == ":") {
const o = deserialize2(str);
return o;
}
return JSON.parse(str.join("")) as T;
} catch (ex) {
try {
return parseYaml(str);
return parseYaml(str.join(""));
} catch (ex) {
return def;
}
Expand Down Expand Up @@ -277,14 +290,14 @@ export class ConfigSync extends LiveSyncCommands {
async loadPluginData(path: FilePathWithPrefix): Promise<PluginDataExDisplay | false> {
const wx = await this.localDatabase.getDBEntry(path, undefined, false, false);
if (wx) {
const data = deserialize(getDocData(wx.data), {}) as PluginDataEx;
const data = deserialize(getDocDataAsArray(wx.data), {}) as PluginDataEx;
const xFiles = [] as PluginDataExFile[];
let missingHash = false;
for (const file of data.files) {
const work = { ...file, data: [] as string[] };
if (!file.hash) {
// debugger;
const tempStr = getDocData(work.data);
const tempStr = getDocDataAsArray(work.data);
const hash = digestHash(tempStr);
file.hash = hash;
missingHash = true;
Expand Down Expand Up @@ -326,6 +339,7 @@ export class ConfigSync extends LiveSyncCommands {
if (saveRequired) {
this.plugin.saveSettingData();
}

}

pluginScanProcessor = new QueueProcessor(async (v: AnyEntry[]) => {
Expand Down Expand Up @@ -384,9 +398,9 @@ export class ConfigSync extends LiveSyncCommands {
const docB = await this.localDatabase.getDBEntry(dataB.documentPath);

if (docA && docB) {
const pluginDataA = deserialize(getDocData(docA.data), {}) as PluginDataEx;
const pluginDataA = deserialize(getDocDataAsArray(docA.data), {}) as PluginDataEx;
pluginDataA.documentPath = dataA.documentPath;
const pluginDataB = deserialize(getDocData(docB.data), {}) as PluginDataEx;
const pluginDataB = deserialize(getDocDataAsArray(docB.data), {}) as PluginDataEx;
pluginDataB.documentPath = dataB.documentPath;

// Use outer structure to wrap each data.
Expand Down Expand Up @@ -425,7 +439,7 @@ export class ConfigSync extends LiveSyncCommands {
if (dx == false) {
throw "Not found on database"
}
const loadedData = deserialize(getDocData(dx.data), {}) as PluginDataEx;
const loadedData = deserialize(getDocDataAsArray(dx.data), {}) as PluginDataEx;
for (const f of loadedData.files) {
Logger(`Applying ${f.filename} of ${data.displayName || data.name}..`);
try {
Expand Down Expand Up @@ -688,7 +702,7 @@ export class ConfigSync extends LiveSyncCommands {
}
const oldC = await this.localDatabase.getDBEntryFromMeta(old, {}, false, false);
if (oldC) {
const d = await deserialize(getDocData(oldC.data), {}) as PluginDataEx;
const d = await deserialize(getDocDataAsArray(oldC.data), {}) as PluginDataEx;
const diffs = (d.files.map(previous => ({ prev: previous, curr: dt.files.find(e => e.filename == previous.filename) })).map(async e => {
try { return await isDocContentSame(e.curr?.data ?? [], e.prev.data) } catch (_) { return false }
}))
Expand Down Expand Up @@ -750,31 +764,33 @@ export class ConfigSync extends LiveSyncCommands {


async scanAllConfigFiles(showMessage: boolean) {
const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
Logger("Scanning customizing files.", logLevel, "scan-all-config");
const term = this.plugin.deviceAndVaultName;
if (term == "") {
Logger("We have to configure the device name", LOG_LEVEL_NOTICE);
return;
}
const filesAll = await this.scanInternalFiles();
const files = filesAll.filter(e => this.isTargetPath(e)).map(e => ({ key: this.filenameToUnifiedKey(e), file: e }));
const virtualPathsOfLocalFiles = [...new Set(files.map(e => e.key))];
const filesOnDB = ((await this.localDatabase.allDocsRaw({ startkey: ICXHeader + "", endkey: `${ICXHeader}\u{10ffff}`, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted);
let deleteCandidate = filesOnDB.map(e => this.getPath(e)).filter(e => e.startsWith(`${ICXHeader}${term}/`));
for (const vp of virtualPathsOfLocalFiles) {
const p = files.find(e => e.key == vp)?.file;
if (!p) {
Logger(`scanAllConfigFiles - File not found: ${vp}`, LOG_LEVEL_VERBOSE);
continue;
await shareRunningResult("scanAllConfigFiles", async () => {
const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
Logger("Scanning customizing files.", logLevel, "scan-all-config");
const term = this.plugin.deviceAndVaultName;
if (term == "") {
Logger("We have to configure the device name", LOG_LEVEL_NOTICE);
return;
}
await this.storeCustomizationFiles(p);
deleteCandidate = deleteCandidate.filter(e => e != vp);
}
for (const vp of deleteCandidate) {
await this.deleteConfigOnDatabase(vp);
}
this.updatePluginList(false).then(/* fire and forget */);
const filesAll = await this.scanInternalFiles();
const files = filesAll.filter(e => this.isTargetPath(e)).map(e => ({ key: this.filenameToUnifiedKey(e), file: e }));
const virtualPathsOfLocalFiles = [...new Set(files.map(e => e.key))];
const filesOnDB = ((await this.localDatabase.allDocsRaw({ startkey: ICXHeader + "", endkey: `${ICXHeader}\u{10ffff}`, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted);
let deleteCandidate = filesOnDB.map(e => this.getPath(e)).filter(e => e.startsWith(`${ICXHeader}${term}/`));
for (const vp of virtualPathsOfLocalFiles) {
const p = files.find(e => e.key == vp)?.file;
if (!p) {
Logger(`scanAllConfigFiles - File not found: ${vp}`, LOG_LEVEL_VERBOSE);
continue;
}
await this.storeCustomizationFiles(p);
deleteCandidate = deleteCandidate.filter(e => e != vp);
}
for (const vp of deleteCandidate) {
await this.deleteConfigOnDatabase(vp);
}
this.updatePluginList(false).then(/* fire and forget */);
});
}
async deleteConfigOnDatabase(prefixedFileName: FilePathWithPrefix, forceWrite = false) {

Expand Down
4 changes: 2 additions & 2 deletions src/features/CmdHiddenFileSync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ export class HiddenFileSync extends LiveSyncCommands {
Logger("something went wrong on resolving all conflicted internal files");
Logger(ex, LOG_LEVEL_VERBOSE);
}
await this.conflictResolutionProcessor.startPipeline().waitForPipeline();
await this.conflictResolutionProcessor.startPipeline().waitForAllProcessed();
}

async resolveByNewerEntry(id: DocumentID, path: FilePathWithPrefix, currentDoc: EntryDoc, currentRev: string, conflictedRev: string) {
Expand Down Expand Up @@ -388,7 +388,7 @@ export class HiddenFileSync extends LiveSyncCommands {
}, { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 0 }))
.root
.enqueueAll(allFileNames)
.startPipeline().waitForPipeline();
.startPipeline().waitForAllDoneAndTerminate();

await this.kvDB.set("diff-caches-internal", caches);

Expand Down
2 changes: 1 addition & 1 deletion src/lib
Submodule lib updated 17 files
+76 −0 src/common/i18n.ts
+196 −0 src/common/rosetta.ts
+85 −24 src/common/types.ts
+2 −2 src/common/utils.ts
+20 −21 src/concurrency/lock.ts
+52 −10 src/concurrency/processor.ts
+2 −2 src/concurrency/task.ts
+1 −1 src/dataobject/reactive.ts
+4 −4 src/dataobject/store.ts
+1 −1 src/memory/LRUCache.ts
+32 −8 src/memory/memutil.ts
+1 −1 src/pouchdb/LiveSyncDBFunctions.ts
+1 −1 src/pouchdb/LiveSyncLocalDB.ts
+6 −1 src/pouchdb/pouchdb-browser.ts
+1 −1 src/pouchdb/utils_couchdb.ts
+2 −0 src/replication/journal/LiveSyncJournalReplicator.ts
+32 −15 src/string_and_binary/strbin.ts
Loading

0 comments on commit b3a85c5

Please sign in to comment.