From f80d448220f1c883d484ef47f167a2a807f890e5 Mon Sep 17 00:00:00 2001 From: gsovereignty Date: Sat, 20 Apr 2024 10:24:15 +1200 Subject: [PATCH] problem: getting replies is slow --- src/lib/snort_workers/main.ts | 2 +- src/lib/snort_workers/master_worker.ts | 230 +++++++++--------- src/lib/snort_workers/types.ts | 2 +- src/lib/snort_workers/utils.ts | 3 + src/lib/views/messages/Messages.svelte | 4 + .../messages/RenderKind1AsThreadHead.svelte | 22 +- 6 files changed, 138 insertions(+), 125 deletions(-) diff --git a/src/lib/snort_workers/main.ts b/src/lib/snort_workers/main.ts index cac10ea..a32d35a 100644 --- a/src/lib/snort_workers/main.ts +++ b/src/lib/snort_workers/main.ts @@ -24,7 +24,7 @@ export function UpdatePubkey(pubkey:string) { } -export function PushEvent(e: NostrEvent) { +export function PushEvent(e: NostrEvent[]) { if (worker) { let cmd = new Command("push_event") cmd.event = e diff --git a/src/lib/snort_workers/master_worker.ts b/src/lib/snort_workers/master_worker.ts index 9d9c3a1..52434f8 100644 --- a/src/lib/snort_workers/master_worker.ts +++ b/src/lib/snort_workers/master_worker.ts @@ -6,10 +6,26 @@ import type { NostrEvent } from '@nostr-dev-kit/ndk'; import WorkerPubkeys from './live_subs?worker'; import WorkerEvents from './fetch_events?worker'; import { seedRelays } from '@/snort_workers/seed_relays'; +import type { Nostr } from 'nostr-tools'; let workerData = new WorkerData(); let workerDataStore = writable(workerData); +const sys = new NostrSystem({ + checkSigs: false + // automaticOutboxModel: true, + // buildFollowGraph: true, +}); + +let connecting = false + +async function connect() { + if (!connecting) { + connecting = true + seedRelays.forEach((r) => sys.ConnectToRelay(r, { read: true, write: false })); + } +} + workerDataStore.subscribe((data) => { let fed = new FrontendData(); fed.basePubkey = data.ourPubkey(); @@ -64,32 +80,31 @@ onmessage = (m: MessageEvent) => { }); } if (m.data.command == 'push_event') { - - workerDataStore.update(current=>{ - console.log(m.data.event) - current.events.set(m.data.event!.id, m.data.event!) - return current - }) + let map = new Map() + if (m.data.event) { + for (let e of m.data.event) { + map.set(e.id, e) + } + if (map.size > 0) { + console.log(89, map) + updateReplies(map) + console.log(91) + } + } } }; //connect to seed relays, get our follows and relays. async function start(pubkey?: string, pubkeys?: string[]) { + connect() return new Promise((resolve, reject) => { if (pubkey) { workerData.setOurPubkey(pubkey); } else { pubkey = workerData.ourPubkey(); } - const sys = new NostrSystem({ - checkSigs: false - // automaticOutboxModel: true, - // buildFollowGraph: true, - }); + (async () => { - await sys.Init(); - seedRelays.forEach((r) => sys.ConnectToRelay(r, { read: true, write: false })); - const rb = new RequestBuilder('fetch-initial-data'); let _pukeys: string[] = []; if (pubkeys) { @@ -155,90 +170,96 @@ async function start(pubkey?: string, pubkeys?: string[]) { let permaSub: Worker | undefined = undefined; -async function PermaSub(pubkeys: string[]) { - if (pubkeys.length > 0) { - if (permaSub) { - permaSub.terminate(); +function updateReplies(newEvents?:Map) { + workerDataStore.update((current) => { + if (newEvents) { + current.events = new Map([...newEvents, ...current.events]); } - permaSub = new WorkerPubkeys(); - permaSub.onmessage = (x: MessageEvent>) => { - workerDataStore.update((current) => { - current.events = new Map([...x.data, ...current.events]); - //console.log(current.events.size) - let printed = 0; - let printedID = new Set(); - for (let [id, e] of current.events) { - current.missingEvents.delete(id); - let tagsForEvent = new tagSplits(e); - if (tagsForEvent.unknown.size > 0) { - //tell user that there's an unhandled tag - if (printed < 20 && !printedID.has(tagsForEvent.id)) { - printed++; - printedID.add(tagsForEvent.id); - //console.log('unknown tag detected', printed, tagsForEvent.rawEvent); - } + //console.log(current.events.size) + let printed = 0; + let printedID = new Set(); + for (let [id, e] of current.events) { + current.missingEvents.delete(id); + let tagsForEvent = new tagSplits(e); + if (tagsForEvent.unknown.size > 0) { + //tell user that there's an unhandled tag + if (printed < 20 && !printedID.has(tagsForEvent.id)) { + printed++; + printedID.add(tagsForEvent.id); + //console.log('unknown tag detected', printed, tagsForEvent.rawEvent); + } + } + tagsForEvent.roots.forEach((r) => { + if (!current.events.has(r)) { + current.missingEvents.add(r); + } else { + current.roots.add(r); + } + }); + if ( + (tagsForEvent.replies.size != 1 && tagsForEvent.unlabelled.size > 1) || + tagsForEvent.replies.size > 1 + ) { + //we don't know which tag is the _real_ reply (parent), let's try and find out + let possibleParents = new Map(); + let possibleReplyTags = new Set([...tagsForEvent.unlabelled, ...tagsForEvent.replies]); + let numMissing = 0; + for (let _id of possibleReplyTags) { + let _event = current.events.get(_id); + if (_event) { + possibleParents.set(_id, _event); } - tagsForEvent.roots.forEach((r) => { - if (!current.events.has(r)) { - current.missingEvents.add(r); - } else { - current.roots.add(r); - } - }); - if ( - (tagsForEvent.replies.size != 1 && tagsForEvent.unlabelled.size > 1) || - tagsForEvent.replies.size > 1 - ) { - //we don't know which tag is the _real_ reply (parent), let's try and find out - let possibleParents = new Map(); - let possibleReplyTags = new Set([...tagsForEvent.unlabelled, ...tagsForEvent.replies]); - let numMissing = 0; - for (let _id of possibleReplyTags) { - let _event = current.events.get(_id); - if (_event) { - possibleParents.set(_id, _event); - } - if (!_event) { - current.missingEvents.add(_id); - numMissing++; - } - } - if (numMissing == 0 && possibleParents.size > 0) { - let allTaggedEvents = new Set(); - for (let [_, e] of possibleParents) { - let splits = new tagSplits(e); - for (let id of splits.All()) { - allTaggedEvents.add(id); - } - } - let tagsThatAreNotInTaggedEvents = new Set(); - for (let id of possibleReplyTags) { - if (!allTaggedEvents.has(id)) { - tagsThatAreNotInTaggedEvents.add(id); - } - } - if (tagsThatAreNotInTaggedEvents.size == 1) { - //console.log("found mistagged reply") - tagsForEvent.replies = new Set([tagsThatAreNotInTaggedEvents][0]); - } - //if more than one in replies: find all the tagged events and see which tag among all these events is unique (the unique one is probably the reply, and the repeated one(s) are the root or further up in the thread) - //console.log('implement me'); - } else { - //console.log(missing) - //todo: fetch missing events by ID + if (!_event) { + current.missingEvents.add(_id); + numMissing++; + } + } + if (numMissing == 0 && possibleParents.size > 0) { + let allTaggedEvents = new Set(); + for (let [_, e] of possibleParents) { + let splits = new tagSplits(e); + for (let id of splits.All()) { + allTaggedEvents.add(id); } } - if (tagsForEvent.replies.size == 1) { - let existing = current.replies.get([...tagsForEvent.replies][0]); - if (!existing) { - existing = new Set(); + let tagsThatAreNotInTaggedEvents = new Set(); + for (let id of possibleReplyTags) { + if (!allTaggedEvents.has(id)) { + tagsThatAreNotInTaggedEvents.add(id); } - existing.add(tagsForEvent.id); - current.replies.set([...tagsForEvent.replies][0], existing); } + if (tagsThatAreNotInTaggedEvents.size == 1) { + //console.log("found mistagged reply") + tagsForEvent.replies = new Set([tagsThatAreNotInTaggedEvents][0]); + } + //if more than one in replies: find all the tagged events and see which tag among all these events is unique (the unique one is probably the reply, and the repeated one(s) are the root or further up in the thread) + //console.log('implement me'); + } else { + //console.log(missing) + //todo: fetch missing events by ID } - return current; - }); + } + if (tagsForEvent.replies.size == 1) { + let existing = current.replies.get([...tagsForEvent.replies][0]); + if (!existing) { + existing = new Set(); + } + existing.add(tagsForEvent.id); + current.replies.set([...tagsForEvent.replies][0], existing); + } + } + return current; + }); +} + +async function PermaSub(pubkeys: string[]) { + if (pubkeys.length > 0) { + if (permaSub) { + permaSub.terminate(); + } + permaSub = new WorkerPubkeys(); + permaSub.onmessage = (x: MessageEvent>) => { + updateReplies(x.data) }; let cmd = new Command('sub_to_pubkeys'); cmd.pubkeys = pubkeys; @@ -252,37 +273,26 @@ let numberOfMissingEvents = derived(workerDataStore, ($wds) => { //let fetchEventsWorker: Worker | undefined = undefined; -const missingEventSys = new NostrSystem({ - checkSigs: false - // automaticOutboxModel: true, - // buildFollowGraph: true, -}); let q: QueryLike - -let fmeStarted = false; -function fmeStart() { - if (!fmeStarted) { - fmeStarted = true; - seedRelays.forEach((r) => missingEventSys.ConnectToRelay(r, { read: true, write: false })); - } -} - numberOfMissingEvents.subscribe((n) => { if (n > 0) { - fmeStart() const rb = new RequestBuilder('fetch-missing-events'); rb.withFilter().ids([...workerData.missingEvents]) rb.withOptions({ leaveOpen: false }); if (q) {q.cancel()} - q = missingEventSys.Query(rb); + q = sys.Query(rb); q.on('event', (evs): void => { - evs.forEach(e=>{ - workerData.events.set(e.id, getNostrEvent(e)) - }) + let m = new Map() + for (let e of evs) { + m.set(e.id, e) + } + if (m.size > 0) { + updateReplies(m) + } }) // console.log(248, n) diff --git a/src/lib/snort_workers/types.ts b/src/lib/snort_workers/types.ts index a04eb58..23388cb 100644 --- a/src/lib/snort_workers/types.ts +++ b/src/lib/snort_workers/types.ts @@ -5,7 +5,7 @@ export class Command { pubkey?: string; pubkeys?: string[]; events?: string[]; - event?: NostrEvent; + event?: NostrEvent[]; constructor(command: 'start' | 'sub_to_pubkeys' | 'fetch_events' | 'push_event') { this.command = command; } diff --git a/src/lib/snort_workers/utils.ts b/src/lib/snort_workers/utils.ts index ebc70fb..5989188 100644 --- a/src/lib/snort_workers/utils.ts +++ b/src/lib/snort_workers/utils.ts @@ -66,5 +66,8 @@ export class tagSplits { if (this.replies.size == 0 && this.unlabelled.size == 1) { this.replies.add([...this.unlabelled][0]); } + if (this.replies.size == 0 && this.roots.size == 1) { + this.replies.add([...this.roots][0]); + } } } \ No newline at end of file diff --git a/src/lib/views/messages/Messages.svelte b/src/lib/views/messages/Messages.svelte index 344ee89..f460d8c 100644 --- a/src/lib/views/messages/Messages.svelte +++ b/src/lib/views/messages/Messages.svelte @@ -38,6 +38,8 @@ let ev = $fds.rawEvents.get(id); if (ev) { fullSet.set(id, ev); + } else { + throw new Error("this should not happen") } } } @@ -150,6 +152,8 @@

HUMBLE HORSE

Release Name: "Giddy Up"
Events in memory: {$FrontendDataStore.rawEvents.size}
+ + diff --git a/src/lib/views/messages/RenderKind1AsThreadHead.svelte b/src/lib/views/messages/RenderKind1AsThreadHead.svelte index 0978b91..cf54887 100644 --- a/src/lib/views/messages/RenderKind1AsThreadHead.svelte +++ b/src/lib/views/messages/RenderKind1AsThreadHead.svelte @@ -12,8 +12,6 @@ export let note: NostrEvent; export let store: Writable; - - let top: HTMLDivElement; let q: QueryLike; @@ -22,17 +20,15 @@ // ID should be unique to the use case, this is important as all data fetched from this ID will be merged into the same NoteStore const rb = new RequestBuilder(`get-${note.id}`); rb.withFilter().tag('e', [note.id]).kinds([1]); - rb.withOptions({leaveOpen: false}) - console.log(26) - console.log(rb) - console.log(28) + rb.withOptions({ leaveOpen: false }); q = System.Query(rb); // basic usage using "onEvent", fired every 100ms q.on('event', (evs) => { - console.log(35, evs); - evs.forEach(e=>{ - PushEvent(e) - }) + if (evs.length > 0) { + console.log(35, evs); + PushEvent(evs); + } + // something else.. }); })(); @@ -43,11 +39,11 @@ })(); }); - onDestroy(()=>{ + onDestroy(() => { if (q) { - q.cancel() + q.cancel(); } - }) + }); $: childrenCount = $store?.replies.get(note.id) ? $store.replies.get(note.id)!.size : 0;