Skip to content

Commit

Permalink
improved fetching or requests
Browse files Browse the repository at this point in the history 
added zbd relay and more
added more default initial users to fetch
  • Loading branch information
@vishalxl
vishalxl committed Dec 22, 2022
1 parent c44d7b8 commit 91ddb39
Show file tree
Hide file tree
Showing 7 changed files with 179 additions and 79 deletions.
60 changes: 37 additions & 23 deletions bin/nostr_console.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ Future<void> main(List<String> arguments) async {
// start application
printIntro("Nostr");


if( argResults["debug"]) {
gDebug = 1;
}
Expand Down Expand Up @@ -133,11 +132,13 @@ Future<void> main(List<String> arguments) async {
// verify that there is at least one valid relay they provided, otherwise keep defaults
if (parsedRelays.length > 0) {
gListRelayUrls1 = parsedRelays;
defaultServerUrl = gListRelayUrls1.first;
} else {
print("No valid relays were provided, using the default relay list");
}
}
printSet( gListRelayUrls1, "Relay List: ");
printSet( gListRelayUrls1, "Primary relays that will be used: ");
print("From among them, default relay: $defaultServerUrl");

if( argResults[lastdaysArg] != null) {
gNumLastDays = int.parse(argResults[lastdaysArg]);
Expand Down Expand Up @@ -251,8 +252,8 @@ Future<void> main(List<String> arguments) async {
//log.info("after reading events");

// count events
initialEvents.forEach((element) { element.eventData.kind == 1? numFilePosts++: numFilePosts;});
print("read $numFilePosts posts from file $gEventsFilename");
initialEvents.forEach((element) { numFileEvents++;});
print("read $numFileEvents events from file $gEventsFilename");
}

// process request string. If this is blank then the application only reads from file and does not connect to internet.
Expand Down Expand Up @@ -292,71 +293,84 @@ Future<void> main(List<String> arguments) async {
// then get the events of user-id's mentioned in p-tags of received events and the contact list
// then display them all

int limitFollowPosts = gLimitFollowPosts;
int limitMetaInfoEvents = 300;
int limitSelfEvents = 300;
int limitOthersEvents = 20;

int limitPerSubscription = gLimitPerSubscription;

// if more than 1000 posts have already been read from the file, then don't get too many day's events. Only for last 3 days.
if(numFilePosts > 1000) {
limitPerSubscription = 5000;
limitSelfEvents = 5;
limitMetaInfoEvents = 3;
limitFollowPosts = 3;
limitSelfEvents = 10;
limitOthersEvents = 3;
gDefaultNumWaitSeconds = gDefaultNumWaitSeconds ~/5 ;
} else {
printInfoForNewUser();
}

// get event for user
if( userPublicKey!= "") {
getUserEvents(gListRelayUrls1, userPublicKey, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents));
getMentionEvents(gListRelayUrls2, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p"); // from relay group 2
getMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p");
}
//getKindEvents([gSecretMessageKind], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo( limitSelfEvents));

Set<String> usersFetched = {userPublicKey};
// remove user from default list if he exists in it. because theyv'e already been fetched.
gDefaultFollows = gDefaultFollows.difference(usersFetched);

//print("getting defaultfollows. usersFetched len = ${usersFetched.length} ");
// get other user events
getMultiUserEvents(gListRelayUrls1, gDefaultFollows, limitPerSubscription, getSecondsDaysAgo(limitFollowPosts));
getMultiUserEvents(gListRelayUrls1, gDefaultFollows, 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
usersFetched = usersFetched.union(gDefaultFollows);

// get group and meta info events
getKindEvents([0, 3, 40, 41], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(limitMetaInfoEvents)); // get all type 3 etc
getKindEvents([42], gListRelayUrls1, limitPerSubscription * 3, getSecondsDaysAgo( limitFollowPosts)); // get all type 3 etc
getKindEvents([40, 41], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents));
getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, 40000);

getKindEvents([gSecretMessageKind], gListRelayUrls1, limitPerSubscription * 3, getSecondsDaysAgo( 3)); // get all type 3 etc
getMultiUserEvents(gListRelayUrls1, usersFetched, 4 * limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), {0,3});

// TODO get all 40 events, and then get all #e for them ( responses to them)

stdout.write('Waiting for user posts to come in.....');
Future.delayed( Duration(milliseconds: gDefaultNumWaitSeconds), () {
//print("total users fetched: ${usersFetched.length}");

initialEvents.addAll(getRecievedEvents());
clearEvents();

initialEvents.forEach((element) { element.eventData.kind == 1? numUserPosts++: numUserPosts;});
numUserPosts -= numFilePosts;
stdout.write("...done\n");//received $numUserPosts new posts made by the user\n");
if( gDebug > 0) log.info("Received user events.");

initialEvents.forEach((e) => processKind3Event(e)); // first process the kind 3 event

Set<String> contacts = {};
if( userPublicKey != "") {
// get the latest kind 3 event for the user, which lists his 'follows' list
Event? contactEvent = getContactEvent(userPublicKey);

// if contact list was found, get user's feed; also get some default contacts
Set<String> contacts = {};
//contacts.addAll(gDefaultFollows);
if (contactEvent != null ) {
if(gDebug > 0) print("In main: found contact list: \n ${contactEvent.originalJson}");
contactEvent.eventData.contactList.forEach((contact) {
contacts.add(contact.id);
});
contacts = contacts.difference(usersFetched); // remove already fetched users from this list
getContactFeed(gListRelayUrls1, contacts, 3 * gLimitPerSubscription, getSecondsDaysAgo( limitOthersEvents));
usersFetched = usersFetched.union(contacts);
} else {
print("Could not find your contact list.");
}
getContactFeed(gListRelayUrls1, contacts, gLimitPerSubscription, getSecondsDaysAgo(2 * limitFollowPosts));
}

// calculate top mentioned ptags, and then get the events for those users
List<String> pTags = getpTags(initialEvents, gMaxPtagsToGet);
getMultiUserEvents(gListRelayUrls1, pTags, gLimitPerSubscription, getSecondsDaysAgo(limitFollowPosts));

Set<String> pTags = getpTags(initialEvents, gMaxPtagsToGet);

pTags = pTags.difference(usersFetched);
getMultiUserEvents(gListRelayUrls1, pTags, 4 * gLimitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
usersFetched = usersFetched.union(pTags);

//print("total users fetched: ${usersFetched.length}");

stdout.write('Waiting for feed to come in..............');
Future.delayed(Duration(milliseconds: gDefaultNumWaitSeconds * 1), () {

Expand Down
9 changes: 9 additions & 0 deletions lib/console_ui.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,14 @@ String mySign(String privateKey, String msg) {
return sign(privateKey, msg, randomSeed);
}

Future<void> mySleep(int duration) async {
Future<void> foo() async {
await Future.delayed(Duration(milliseconds: duration));
return;
}
await foo();
}

/* @function sendReplyPostLike Used to send Reply, Post and Like ( event 1 for reply and post, and event 7 for like/reaction)
* If replyToId is blank, then it does not reference any e/p tags, and thus becomes a top post
* otherwise e and p tags are found for the given event being replied to, if that event data is available
Expand Down Expand Up @@ -92,6 +100,7 @@ Future<void> sendReplyPostLike(Store node, String replyToId, String replyKind, S
String toSendMessage = '["EVENT",{"id":"$id","pubkey":"$userPublicKey","created_at":$createdAt,"kind":$replyKind,"tags":[$vanityTag],"content":"$content","sig":"$sig"}]';
//print("sending $toSendMessage");
sendRequest( gListRelayUrls1, toSendMessage);
await mySleep(200);
}

// Sends a public channel message
Expand Down
9 changes: 7 additions & 2 deletions lib/event_ds.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -956,6 +956,11 @@ class Event {

factory Event.fromJson(String d, String relay, [bool fromFile = false]) {
try {
if( d.length > gMaxEventLenthAccepted) {
//throw Exception("Event json is larger than max len");

}

dynamic json = jsonDecode(d);

if( json.length < 3) {
Expand Down Expand Up @@ -1010,7 +1015,7 @@ void addToHistogram(Map<String, int> histogram, List<String> pTags) {
}

// return the numMostFrequent number of most frequent p tags ( user pubkeys) in the given events
List<String> getpTags(Set<Event> events, int numMostFrequent) {
Set<String> getpTags(Set<Event> events, int numMostFrequent) {
List<HistogramEntry> listHistogram = [];
Map<String, int> histogramMap = {};
for(var event in events) {
Expand All @@ -1024,7 +1029,7 @@ List<String> getpTags(Set<Event> events, int numMostFrequent) {
ptags.add(listHistogram[i].str);
}

return ptags;
return ptags.toSet();
}

// From the list of events provided, lookup the lastst contact information for the given user/pubkey
Expand Down
33 changes: 9 additions & 24 deletions lib/relays.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ class Relays {
}

String request = getUserRequest(subscriptionId, publicKey, limit, sinceWhen);
//print("In relay: getKind events: request = $request");
sendRequest(relayUrl, request);
}

Expand All @@ -100,29 +101,12 @@ class Relays {
* @connect Connect to given relay and get all events for multiple users/publicKey and insert the
* received events in the given List<Event>
*/
void getMultiUserEvents(String relayUrl, List<String> publicKeys, int limit, int sinceWhen) {

Set<String> reqKeys = {};
if( relays.containsKey(relayUrl)) {
List<String>? users = relays[relayUrl]?.users;
if( users != null) {
// following is too restrictive. TODO improve it
for(int i = 0; i < publicKeys.length; i++) {
if( users.any( (u) => u == publicKeys[i])) {
continue;
}
if( gBots.any( (bot) => bot == publicKeys[i] )) {
continue;
}
users.add(publicKeys[i]);
reqKeys.add(publicKeys[i]);
}
}
} // if relay exists and has a user list
void getMultiUserEvents(String relayUrl, List<String> publicKeys, int limit, int sinceWhen, [Set<int>? kind = null]) {
//print("In relays: getmulti events kind = $kind len ${publicKeys.length}");

String subscriptionId = "multiple_user" + (relays[relayUrl]?.numRequestsSent??"").toString() + "_" + relayUrl.substring(6);

String request = getMultiUserRequest( subscriptionId, reqKeys, limit, sinceWhen);
String request = getMultiUserRequest( subscriptionId, publicKeys.toSet(), limit, sinceWhen, kind);
sendRequest(relayUrl, request);
}

Expand Down Expand Up @@ -233,7 +217,7 @@ void getContactFeed(Set<String> relayUrls, Set<String> setContacts, int numEvent
groupContacts.add(contacts[i + j]);
}

//print( "i = $i upperLimit = $upperLimit") ;
//print( "in getcontact feed . i = $i upperLimit = $upperLimit") ;
relayUrls.forEach((relayUrl) {
relays.getMultiUserEvents(relayUrl, groupContacts, numEventsToGet, sinceWhen);
});
Expand Down Expand Up @@ -262,7 +246,8 @@ getKindEvents(List<int> kind, Set<String> serverUrls, int limit, int sinceWhen)
});
}

void getMultiUserEvents(Set<String> serverUrls, List<String> publicKeys, int numUserEvents, int sinceWhen) {
void getMultiUserEvents(Set<String> serverUrls, Set<String> setPublicKeys, int numUserEvents, int sinceWhen, [Set<int>? kind]) {
List<String> publicKeys = setPublicKeys.toList();
if( gDebug > 0) print("Sending multi user request for ${publicKeys.length} users");

for(var serverUrl in serverUrls) {
Expand All @@ -271,9 +256,9 @@ void getMultiUserEvents(Set<String> serverUrls, List<String> publicKeys, int num
if( publicKeys.length - i <= gMaxAuthorsInOneRequest) {
getUserRequests = publicKeys.length - i;
}
//print(" sending request form $i to ${i + getUserRequests} ");
//print("In getMultiuserevents: sending request form $i to ${i + getUserRequests} ");
List<String> partialList = publicKeys.sublist(i, i + getUserRequests);
relays.getMultiUserEvents(serverUrl, partialList, numUserEvents, sinceWhen);
relays.getMultiUserEvents(serverUrl, partialList, numUserEvents, sinceWhen, kind);
}
}
}
Expand Down
57 changes: 48 additions & 9 deletions lib/settings.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ final log = Logger('ExampleLogger');
// for debugging
String gCheckEventId = "x98821b082a1d322e8cba84e8d430da300dea043348f422229f929059d1a9bb05";

int gDefaultNumWaitSeconds = 8000; // is used in main()

int gMaxEventLenthAccepted = 80000; // max event size. events larger than this are rejected.

//////////////////////////////////////////////////////////////////////////////////////////////////////////////// encrypted Group settings
const int gSecretMessageKind = 104;
Expand All @@ -39,11 +40,12 @@ const int gLimitPerSubscription = 10000;
// applicable only for notifications and not for search results. Search results set a flag in EventData and don't use this variable
const int gDontHighlightEventsOlderThan = 4;

const int gMaxAuthorsInOneRequest = 100; // number of author requests to send in one request
const int gMaxPtagsToGet = 150; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent)
int gDefaultNumWaitSeconds = 12000; // is used in main()
const int gMaxAuthorsInOneRequest = 200; // number of author requests to send in one request
const int gMaxPtagsToGet = 200; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent)

// global counters of total events read or processed
int numFilePosts = 0, numUserPosts = 0, numFeedPosts = 0, numOtherPosts = 0;
int numFileEvents = 0, numFilePosts = 0, numUserPosts = 0, numFeedPosts = 0, numOtherPosts = 0;

//String defaultServerUrl = 'wss://relay.damus.io';
//const String nostrRelayUnther = 'wss://nostr-relay.untethr.me'; not working
Expand All @@ -52,12 +54,15 @@ String defaultServerUrl = "wss://relay.damus.io";

Set<String> gListRelayUrls1 = { defaultServerUrl,
relayNostrInfo,
"wss://nostr-relay.wlvs.space"
"wss://nostr.semisol.dev",
"wss://nostr-2.zebedee.cloud",
"wss://nostr.onsats.org"

};

Set<String> gListRelayUrls2 = {
"wss://nostr.oxtr.dev",
"wss://nostr.ono.re"
"wss://nostr.bitcoiner.social"
};

// well known disposable test private key
Expand All @@ -66,9 +71,36 @@ String userPrivateKey = "";
String userPublicKey = gDefaultPublicKey;

// default follows; taken from nostr.io/stats
List<String> gDefaultFollows = [
Set<String> gDefaultFollows = {
// 21 dec 2022
"82341f882b6eabcd2ba7f1ef90aad961cf074af15b9ef44a09f9d2a8fbfbe6a2", // Jack Dorsey
"c4eabae1be3cf657bc1855ee05e69de9f059cb7a059227168b80b89761cbc4e0", // Mallers
"a341f45ff9758f570a21b000c17d4e53a3a497c8397f26c0e6d61e5acffc7a98", // Saylor
//"703e26b4f8bc0fa57f99d815dbb75b086012acc24fc557befa310f5aa08d1898", // Adam Back not sure
"04c915daefee38317fa734444acee390a8269fe5810b2241e5e6dd343dfbecc9", // ODELL
"e88a691e98d9987c964521dff60025f60700378a4879180dcbbb4a5027850411", // NVK
"85080d3bad70ccdcd7f74c29a44f55bb85cbcd3dd0cbb957da1d215bdb931204", // Preston
"83e818dfbeccea56b0f551576b3fd39a7a50e1d8159343500368fa085ccd964b", // Jeff Booth
"f728d9e6e7048358e70930f5ca64b097770d989ccd86854fe618eda9c8a38106", // Lopp
"bf2376e17ba4ec269d10fcc996a4746b451152be9031fa48e74553dde5526bce", // CARLA
"e33fe65f1fde44c6dc17eeb38fdad0fceaf1cae8722084332ed1e32496291d42", // wiz
"472f440f29ef996e92a186b8d320ff180c855903882e59d50de1b8bd5669301e", // MartyBent
"1577e4599dd10c863498fe3c20bd82aafaf829a595ce83c5cf8ac3463531b09b", // yegorPetrov
"be1d89794bf92de5dd64c1e60f6a2c70c140abac9932418fee30c5c637fe9479", // walletofsatoshi
"eaf27aa104833bcd16f671488b01d65f6da30163b5848aea99677cc947dd00aa", // grubles
"b9003833fabff271d0782e030be61b7ec38ce7d45a1b9a869fbdb34b9e2d2000", // brockm
"51b826cccd92569a6582e20982fd883fccfa78ad03e0241f7abec1830d7a2565", // Jonas Schnelli
"92de68b21302fa2137b1cbba7259b8ba967b535a05c6d2b0847d9f35ff3cf56a", // Susie bdds
"c48e29f04b482cc01ca1f9ef8c86ef8318c059e0e9353235162f080f26e14c11", // walker
"a9b9525992a486aa16b3c1d3f9d3604bca08f3c15b712d70711b9aecd8c3dc44", // Alana

"24e37c1e5b0c8ba8dde2754bcffc63b5b299f8064f8fb928bcf315b9c4965f3b", // lunaticoin
"4523be58d395b1b196a9b8c82b038b6895cb02b683d0c253a955068dba1facd0", // martii malmi
"97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322", // hodlbod

// pre dec 2022
"3efdaebb1d8923ebd99c9e7ace3b4194ab45512e2be79c1b7d68d9243e0d2681", //damus
"6b0d4c8d9dc59e110d380b0429a02891f1341a0fa2ba1b1cf83a3db4d47e3964", // dergigi
"6b0d4c8d9dc59e110d380b0429a02891f1341a0fa2ba1b1cf83a3db4d47e3964", // dergigi
"32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245", // jb55
"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d", // fiatjaf
"2ef93f01cd2493e04235a6b87b10d3c4a74e2a7eb7c3caf168268f6af73314b5", // unclebobmarting
Expand All @@ -89,7 +121,8 @@ List<String> gDefaultFollows = [
"f43c1f9bff677b8f27b602725ea0ad51af221344f69a6b352a74991a4479bac3", // manfromhighcastle
"80482e60178c2ce996da6d67577f56a2b2c47ccb1c84c81f2b7960637cb71b78", // Leo
"42a0825e980b9f97943d2501d99c3a3859d4e68cd6028c02afe58f96ba661a9d", // zerosequioso
"3235036bd0957dfb27ccda02d452d7c763be40c91a1ac082ba6983b25238388c"]; // vishalxl ];

"3235036bd0957dfb27ccda02d452d7c763be40c91a1ac082ba6983b25238388c"}; // vishalxl ];


// dummy account pubkey
Expand Down Expand Up @@ -368,6 +401,12 @@ lines.forEach((line) {print(line.length > terminalColumns ? line.substring(0, te

}

void printInfoForNewUser() {
print("""\nFor new users: The app only gets kind 1 events from people you follow or some popular well known pubkeys.
If you see a message such as 'event not loaded' it implies its from someone you don't follow. Such events
are eventually loaded; however, the ideal way to use this app is to follow people whose posts you want to read or follow.\n""");
}

/////////////////////////////////////////////////////////other settings related functions
void printUsage() {
Expand Down
Loading

0 comments on commit 91ddb39

Please sign in to comment.