From 91fa45a8c6932ac8c523f38bd0c8680e775e3e33 Mon Sep 17 00:00:00 2001 From: Vishal <64505169+vishalxl@users.noreply.github.com> Date: Fri, 23 Dec 2022 20:08:41 +0530 Subject: [PATCH] reduced events fetched .. to about 30k for my own username. This reduces connection which seems to improve relay response. added file user.dart with mostly old code. --- bin/nostr_console.dart | 25 +++++++++++------- lib/console_ui.dart | 1 + lib/event_ds.dart | 48 ---------------------------------- lib/settings.dart | 6 ++--- lib/tree_ds.dart | 3 ++- lib/user.dart | 59 ++++++++++++++++++++++++++++++++++++++++++ pubspec.yaml | 1 + 7 files changed, 81 insertions(+), 62 deletions(-) create mode 100644 lib/user.dart diff --git a/bin/nostr_console.dart b/bin/nostr_console.dart index dd81c68..db26b0f 100644 --- a/bin/nostr_console.dart +++ b/bin/nostr_console.dart @@ -6,6 +6,8 @@ import 'package:nostr_console/relays.dart'; import 'package:nostr_console/console_ui.dart'; import 'package:nostr_console/settings.dart'; import 'package:nostr_console/utils.dart'; +import 'package:nostr_console/user.dart'; + import 'package:args/args.dart'; import 'package:logging/logging.dart'; @@ -322,8 +324,8 @@ Future main(List arguments) async { usersFetched = usersFetched.union(gDefaultFollows); // get group and meta info events - getKindEvents([40, 41], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents)); - getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(2)); + getKindEvents([40, 41], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents)); + getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(4)); getMultiUserEvents(gListRelayUrls1, usersFetched, 4 * limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), {0,3}); @@ -352,6 +354,7 @@ Future main(List arguments) async { contacts.add(contact.id); }); contacts = contacts.difference(usersFetched); // remove already fetched users from this list + getContactFeed(gListRelayUrls1, contacts, 3 * gLimitPerSubscription, getSecondsDaysAgo( limitOthersEvents)); usersFetched = usersFetched.union(contacts); } else { @@ -359,14 +362,19 @@ Future main(List arguments) async { } } - // calculate top mentioned ptags, and then get the events for those users - Set pTags = getpTags(initialEvents, gMaxPtagsToGet); + // fetch extra events for people who don't have too large a follow list + if( usersFetched.length < gMaxPtagsToGet * 2 ) { + // calculate top mentioned ptags, and then get the events for those users + Set pTags = getpTags(initialEvents, gMaxPtagsToGet); + pTags = pTags.difference(usersFetched); - pTags = pTags.difference(usersFetched); - getMultiUserEvents(gListRelayUrls1, pTags, 4 * gLimitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); - usersFetched = usersFetched.union(pTags); + getMultiUserEvents(gListRelayUrls1, pTags, 4 * gLimitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); + usersFetched = usersFetched.union(pTags); + } - //print("total users fetched: ${usersFetched.length}"); + // get events from channels of user + Set userChannels = getUserChannels(initialEvents, userPublicKey); + //getMentionEvents(gListRelayUrls1, userChannels, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#e"); stdout.write('Waiting for feed to come in..............'); Future.delayed(Duration(milliseconds: gDefaultNumWaitSeconds * 1), () { @@ -381,7 +389,6 @@ Future main(List arguments) async { Store node = getTree(initialEvents); gStore = node; - clearEvents(); mainMenuUi(node); }); diff --git a/lib/console_ui.dart b/lib/console_ui.dart index bd46660..8aee30c 100644 --- a/lib/console_ui.dart +++ b/lib/console_ui.dart @@ -6,6 +6,7 @@ import 'package:nostr_console/tree_ds.dart'; import 'package:nostr_console/relays.dart'; import 'package:nostr_console/settings.dart'; import 'package:nostr_console/utils.dart'; +import 'package:nostr_console/user.dart'; import 'package:bip340/bip340.dart'; Future processAnyIncomingEvents(Store node, [bool printNotifications = true]) async { diff --git a/lib/event_ds.dart b/lib/event_ds.dart index 4253a0f..dd8828d 100644 --- a/lib/event_ds.dart +++ b/lib/event_ds.dart @@ -998,54 +998,6 @@ class Event { } } -void addToHistogram(Map histogram, List pTags) { - Set tempPtags = {}; - pTags.retainWhere((x) => tempPtags.add(x)); - - for(int i = 0; i < pTags.length; i++ ) { - String pTag = pTags[i]; - if( histogram.containsKey(pTag)) { - int? val = histogram[pTag]; - if( val != null) { - histogram[pTag] = ++val; - } else { - } - } else { - histogram[pTag] = 1; - } - } - //return histogram; -} - -// return the numMostFrequent number of most frequent p tags ( user pubkeys) in the given events -Set getpTags(Set events, int numMostFrequent) { - List listHistogram = []; - Map histogramMap = {}; - for(var event in events) { - addToHistogram(histogramMap, event.eventData.pTags); - } - - histogramMap.forEach((key, value) {listHistogram.add(HistogramEntry(key, value));/* print("added to list of histogramEntry $key $value"); */}); - listHistogram.sort(HistogramEntry.histogramSorter); - List ptags = []; - for( int i = 0; i < listHistogram.length && i < numMostFrequent; i++ ) { - ptags.add(listHistogram[i].str); - } - - return ptags.toSet(); -} - -// From the list of events provided, lookup the lastst contact information for the given user/pubkey -Event? getContactEvent(String pubkey) { - - // get the latest kind 3 event for the user, which lists his 'follows' list - if( gKindONames.containsKey(pubkey)) { - Event? e = (gKindONames[pubkey]?.latestContactEvent)??null; - return e; - } - - return null; -} // for the user userPubkey, returns the relay of its contact contactPubkey String getRelayOfUser(String userPubkey, String contactPubkey) { diff --git a/lib/settings.dart b/lib/settings.dart index cd8b840..e2b55d8 100644 --- a/lib/settings.dart +++ b/lib/settings.dart @@ -41,14 +41,12 @@ const int gLimitPerSubscription = 10000; const int gDontHighlightEventsOlderThan = 4; int gDefaultNumWaitSeconds = 12000; // is used in main() -const int gMaxAuthorsInOneRequest = 200; // number of author requests to send in one request -const int gMaxPtagsToGet = 200; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent) +const int gMaxAuthorsInOneRequest = 300; // number of author requests to send in one request +const int gMaxPtagsToGet = 100; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent) // global counters of total events read or processed int numFileEvents = 0, numFilePosts = 0, numUserPosts = 0, numFeedPosts = 0, numOtherPosts = 0; -//String defaultServerUrl = 'wss://relay.damus.io'; -//const String nostrRelayUnther = 'wss://nostr-relay.untethr.me'; not working String defaultServerUrl = "wss://relay.damus.io"; const String relayNostrInfo = 'wss://relay.nostr.info'; diff --git a/lib/tree_ds.dart b/lib/tree_ds.dart index 8405dc6..361f96f 100644 --- a/lib/tree_ds.dart +++ b/lib/tree_ds.dart @@ -2,8 +2,9 @@ import 'dart:io'; import 'dart:convert'; import 'package:nostr_console/event_ds.dart'; import 'package:nostr_console/relays.dart'; -import 'package:nostr_console/settings.dart'; import 'package:nostr_console/utils.dart'; +import 'package:nostr_console/settings.dart'; +import 'package:nostr_console/user.dart'; import 'dart:math'; // for Point diff --git a/lib/user.dart b/lib/user.dart new file mode 100644 index 0000000..add089c --- /dev/null +++ b/lib/user.dart @@ -0,0 +1,59 @@ + +import 'dart:io'; +import 'package:nostr_console/event_ds.dart'; +import 'package:nostr_console/utils.dart'; + +// From the list of events provided, lookup the lastst contact information for the given user/pubkey +Event? getContactEvent(String pubkey) { + + // get the latest kind 3 event for the user, which lists his 'follows' list + if( gKindONames.containsKey(pubkey)) { + Event? e = (gKindONames[pubkey]?.latestContactEvent)??null; + return e; + } + + return null; +} + +Set getUserChannels(Set userEvents, String userPublicKey) { + Set userChannels = {}; + + return userChannels; +} + +void addToHistogram(Map histogram, List pTags) { + Set tempPtags = {}; + pTags.retainWhere((x) => tempPtags.add(x)); + + for(int i = 0; i < pTags.length; i++ ) { + String pTag = pTags[i]; + if( histogram.containsKey(pTag)) { + int? val = histogram[pTag]; + if( val != null) { + histogram[pTag] = ++val; + } else { + } + } else { + histogram[pTag] = 1; + } + } + //return histogram; +} + +// return the numMostFrequent number of most frequent p tags ( user pubkeys) in the given events +Set getpTags(Set events, int numMostFrequent) { + List listHistogram = []; + Map histogramMap = {}; + for(var event in events) { + addToHistogram(histogramMap, event.eventData.pTags); + } + + histogramMap.forEach((key, value) {listHistogram.add(HistogramEntry(key, value));/* print("added to list of histogramEntry $key $value"); */}); + listHistogram.sort(HistogramEntry.histogramSorter); + List ptags = []; + for( int i = 0; i < listHistogram.length && i < numMostFrequent; i++ ) { + ptags.add(listHistogram[i].str); + } + + return ptags.toSet(); +} \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index 47ab750..51b6a8f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,6 +12,7 @@ homepage: https://github.com/vishalxl/nostr_console # increased channel fetches from 2 days from half a day # reduced items fetched. 23/12 +# reduced items more evening 23/12 environment: sdk: '>=2.17.3 <3.0.0'