Skip to content

Commit

Permalink
problem: getting replies is slow
Browse files Browse the repository at this point in the history
  • Loading branch information
@gsovereignty
gsovereignty committed Apr 19, 2024
1 parent ec11729 commit f80d448
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 125 deletions.
2 changes: 1 addition & 1 deletion src/lib/snort_workers/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export function UpdatePubkey(pubkey:string) {

}

export function PushEvent(e: NostrEvent) {
export function PushEvent(e: NostrEvent[]) {
if (worker) {
let cmd = new Command("push_event")
cmd.event = e
Expand Down
230 changes: 120 additions & 110 deletions src/lib/snort_workers/master_worker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,26 @@ import type { NostrEvent } from '@nostr-dev-kit/ndk';
import WorkerPubkeys from './live_subs?worker';
import WorkerEvents from './fetch_events?worker';
import { seedRelays } from '@/snort_workers/seed_relays';
import type { Nostr } from 'nostr-tools';

let workerData = new WorkerData();
let workerDataStore = writable(workerData);

const sys = new NostrSystem({
checkSigs: false
// automaticOutboxModel: true,
// buildFollowGraph: true,
});

let connecting = false

async function connect() {
if (!connecting) {
connecting = true
seedRelays.forEach((r) => sys.ConnectToRelay(r, { read: true, write: false }));
}
}

workerDataStore.subscribe((data) => {
let fed = new FrontendData();
fed.basePubkey = data.ourPubkey();
Expand Down Expand Up @@ -64,32 +80,31 @@ onmessage = (m: MessageEvent<Command>) => {
});
}
if (m.data.command == 'push_event') {

workerDataStore.update(current=>{
console.log(m.data.event)
current.events.set(m.data.event!.id, m.data.event!)
return current
})
let map = new Map<string, NostrEvent>()
if (m.data.event) {
for (let e of m.data.event) {
map.set(e.id, e)
}
if (map.size > 0) {
console.log(89, map)
updateReplies(map)
console.log(91)
}
}
}
};

//connect to seed relays, get our follows and relays.
async function start(pubkey?: string, pubkeys?: string[]) {
connect()
return new Promise((resolve, reject) => {
if (pubkey) {
workerData.setOurPubkey(pubkey);
} else {
pubkey = workerData.ourPubkey();
}
const sys = new NostrSystem({
checkSigs: false
// automaticOutboxModel: true,
// buildFollowGraph: true,
});

(async () => {
await sys.Init();
seedRelays.forEach((r) => sys.ConnectToRelay(r, { read: true, write: false }));

const rb = new RequestBuilder('fetch-initial-data');
let _pukeys: string[] = [];
if (pubkeys) {
Expand Down Expand Up @@ -155,90 +170,96 @@ async function start(pubkey?: string, pubkeys?: string[]) {

let permaSub: Worker | undefined = undefined;

async function PermaSub(pubkeys: string[]) {
if (pubkeys.length > 0) {
if (permaSub) {
permaSub.terminate();
function updateReplies(newEvents?:Map<string, NostrEvent>) {
workerDataStore.update((current) => {
if (newEvents) {
current.events = new Map([...newEvents, ...current.events]);
}
permaSub = new WorkerPubkeys();
permaSub.onmessage = (x: MessageEvent<Map<string, NostrEvent>>) => {
workerDataStore.update((current) => {
current.events = new Map([...x.data, ...current.events]);
//console.log(current.events.size)
let printed = 0;
let printedID = new Set<string>();
for (let [id, e] of current.events) {
current.missingEvents.delete(id);
let tagsForEvent = new tagSplits(e);
if (tagsForEvent.unknown.size > 0) {
//tell user that there's an unhandled tag
if (printed < 20 && !printedID.has(tagsForEvent.id)) {
printed++;
printedID.add(tagsForEvent.id);
//console.log('unknown tag detected', printed, tagsForEvent.rawEvent);
}
//console.log(current.events.size)
let printed = 0;
let printedID = new Set<string>();
for (let [id, e] of current.events) {
current.missingEvents.delete(id);
let tagsForEvent = new tagSplits(e);
if (tagsForEvent.unknown.size > 0) {
//tell user that there's an unhandled tag
if (printed < 20 && !printedID.has(tagsForEvent.id)) {
printed++;
printedID.add(tagsForEvent.id);
//console.log('unknown tag detected', printed, tagsForEvent.rawEvent);
}
}
tagsForEvent.roots.forEach((r) => {
if (!current.events.has(r)) {
current.missingEvents.add(r);
} else {
current.roots.add(r);
}
});
if (
(tagsForEvent.replies.size != 1 && tagsForEvent.unlabelled.size > 1) ||
tagsForEvent.replies.size > 1
) {
//we don't know which tag is the _real_ reply (parent), let's try and find out
let possibleParents = new Map<string, NostrEvent>();
let possibleReplyTags = new Set([...tagsForEvent.unlabelled, ...tagsForEvent.replies]);
let numMissing = 0;
for (let _id of possibleReplyTags) {
let _event = current.events.get(_id);
if (_event) {
possibleParents.set(_id, _event);
}
tagsForEvent.roots.forEach((r) => {
if (!current.events.has(r)) {
current.missingEvents.add(r);
} else {
current.roots.add(r);
}
});
if (
(tagsForEvent.replies.size != 1 && tagsForEvent.unlabelled.size > 1) ||
tagsForEvent.replies.size > 1
) {
//we don't know which tag is the _real_ reply (parent), let's try and find out
let possibleParents = new Map<string, NostrEvent>();
let possibleReplyTags = new Set([...tagsForEvent.unlabelled, ...tagsForEvent.replies]);
let numMissing = 0;
for (let _id of possibleReplyTags) {
let _event = current.events.get(_id);
if (_event) {
possibleParents.set(_id, _event);
}
if (!_event) {
current.missingEvents.add(_id);
numMissing++;
}
}
if (numMissing == 0 && possibleParents.size > 0) {
let allTaggedEvents = new Set<string>();
for (let [_, e] of possibleParents) {
let splits = new tagSplits(e);
for (let id of splits.All()) {
allTaggedEvents.add(id);
}
}
let tagsThatAreNotInTaggedEvents = new Set<string>();
for (let id of possibleReplyTags) {
if (!allTaggedEvents.has(id)) {
tagsThatAreNotInTaggedEvents.add(id);
}
}
if (tagsThatAreNotInTaggedEvents.size == 1) {
//console.log("found mistagged reply")
tagsForEvent.replies = new Set([tagsThatAreNotInTaggedEvents][0]);
}
//if more than one in replies: find all the tagged events and see which tag among all these events is unique (the unique one is probably the reply, and the repeated one(s) are the root or further up in the thread)
//console.log('implement me');
} else {
//console.log(missing)
//todo: fetch missing events by ID
if (!_event) {
current.missingEvents.add(_id);
numMissing++;
}
}
if (numMissing == 0 && possibleParents.size > 0) {
let allTaggedEvents = new Set<string>();
for (let [_, e] of possibleParents) {
let splits = new tagSplits(e);
for (let id of splits.All()) {
allTaggedEvents.add(id);
}
}
if (tagsForEvent.replies.size == 1) {
let existing = current.replies.get([...tagsForEvent.replies][0]);
if (!existing) {
existing = new Set();
let tagsThatAreNotInTaggedEvents = new Set<string>();
for (let id of possibleReplyTags) {
if (!allTaggedEvents.has(id)) {
tagsThatAreNotInTaggedEvents.add(id);
}
existing.add(tagsForEvent.id);
current.replies.set([...tagsForEvent.replies][0], existing);
}
if (tagsThatAreNotInTaggedEvents.size == 1) {
//console.log("found mistagged reply")
tagsForEvent.replies = new Set([tagsThatAreNotInTaggedEvents][0]);
}
//if more than one in replies: find all the tagged events and see which tag among all these events is unique (the unique one is probably the reply, and the repeated one(s) are the root or further up in the thread)
//console.log('implement me');
} else {
//console.log(missing)
//todo: fetch missing events by ID
}
return current;
});
}
if (tagsForEvent.replies.size == 1) {
let existing = current.replies.get([...tagsForEvent.replies][0]);
if (!existing) {
existing = new Set();
}
existing.add(tagsForEvent.id);
current.replies.set([...tagsForEvent.replies][0], existing);
}
}
return current;
});
}

async function PermaSub(pubkeys: string[]) {
if (pubkeys.length > 0) {
if (permaSub) {
permaSub.terminate();
}
permaSub = new WorkerPubkeys();
permaSub.onmessage = (x: MessageEvent<Map<string, NostrEvent>>) => {
updateReplies(x.data)
};
let cmd = new Command('sub_to_pubkeys');
cmd.pubkeys = pubkeys;
Expand All @@ -252,37 +273,26 @@ let numberOfMissingEvents = derived(workerDataStore, ($wds) => {

//let fetchEventsWorker: Worker | undefined = undefined;

const missingEventSys = new NostrSystem({
checkSigs: false
// automaticOutboxModel: true,
// buildFollowGraph: true,
});


let q: QueryLike



let fmeStarted = false;
function fmeStart() {
if (!fmeStarted) {
fmeStarted = true;
seedRelays.forEach((r) => missingEventSys.ConnectToRelay(r, { read: true, write: false }));
}
}

numberOfMissingEvents.subscribe((n) => {
if (n > 0) {
fmeStart()
const rb = new RequestBuilder('fetch-missing-events');
rb.withFilter().ids([...workerData.missingEvents])
rb.withOptions({ leaveOpen: false });
if (q) {q.cancel()}
q = missingEventSys.Query(rb);
q = sys.Query(rb);
q.on('event', (evs): void => {
evs.forEach(e=>{
workerData.events.set(e.id, getNostrEvent(e))
})
let m = new Map<string, NostrEvent>()
for (let e of evs) {
m.set(e.id, e)
}
if (m.size > 0) {
updateReplies(m)
}
})

// console.log(248, n)
Expand Down
2 changes: 1 addition & 1 deletion src/lib/snort_workers/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export class Command {
pubkey?: string;
pubkeys?: string[];
events?: string[];
event?: NostrEvent;
event?: NostrEvent[];
constructor(command: 'start' | 'sub_to_pubkeys' | 'fetch_events' | 'push_event') {
this.command = command;
}
Expand Down
3 changes: 3 additions & 0 deletions src/lib/snort_workers/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,5 +66,8 @@ export class tagSplits {
if (this.replies.size == 0 && this.unlabelled.size == 1) {
this.replies.add([...this.unlabelled][0]);
}
if (this.replies.size == 0 && this.roots.size == 1) {
this.replies.add([...this.roots][0]);
}
}
}
4 changes: 4 additions & 0 deletions src/lib/views/messages/Messages.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
let ev = $fds.rawEvents.get(id);
if (ev) {
fullSet.set(id, ev);
} else {
throw new Error("this should not happen")
}
}
}
Expand Down Expand Up @@ -150,6 +152,8 @@
<h3>HUMBLE HORSE</h3>
<h6>Release Name: "Giddy Up"</h6>
Events in memory: {$FrontendDataStore.rawEvents.size}<br />
<Button onClick={()=>{console.log($FrontendDataStore.replies.get($threadParentID))}}>Print root event data</Button>

</div>
</div>
</ChatLayout>
22 changes: 9 additions & 13 deletions src/lib/views/messages/RenderKind1AsThreadHead.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
export let note: NostrEvent;
export let store: Writable<FrontendData>;
let top: HTMLDivElement;
let q: QueryLike;
Expand All @@ -22,17 +20,15 @@
// ID should be unique to the use case, this is important as all data fetched from this ID will be merged into the same NoteStore
const rb = new RequestBuilder(`get-${note.id}`);
rb.withFilter().tag('e', [note.id]).kinds([1]);
rb.withOptions({leaveOpen: false})
console.log(26)
console.log(rb)
console.log(28)
rb.withOptions({ leaveOpen: false });
q = System.Query(rb);
// basic usage using "onEvent", fired every 100ms
q.on('event', (evs) => {
console.log(35, evs);
evs.forEach(e=>{
PushEvent(e)
})
if (evs.length > 0) {
console.log(35, evs);
PushEvent(evs);
}
// something else..
});
})();
Expand All @@ -43,11 +39,11 @@
})();
});
onDestroy(()=>{
onDestroy(() => {
if (q) {
q.cancel()
q.cancel();
}
})
});
$: childrenCount = $store?.replies.get(note.id) ? $store.replies.get(note.id)!.size : 0;
</script>
Expand Down

0 comments on commit f80d448

Please sign in to comment.