Skip to content

Commit

Permalink
spiders for the relay events but doesn't save them yet
Browse files Browse the repository at this point in the history
  • Loading branch information
@mleku
mleku committed Dec 21, 2024
1 parent 3a95456 commit a485d24
Show file tree
Hide file tree
Showing 17 changed files with 516 additions and 287 deletions.
24 changes: 13 additions & 11 deletions cmd/realy/app/implementation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ type Relay struct {
sync.Mutex
Ctx cx
*config.C
Store store.I
Store store.I
MaxLimit no
// Owners' pubkeys
Owners []by
Followed, OwnersFollowed map[st]struct{}
Expand Down Expand Up @@ -63,7 +64,7 @@ func (r *Relay) Init() (err er) {
r.ZeroLists()
r.CheckOwnerLists()
// start up the spider, if configured
r.Spider()
go r.Spider()
return nil
}

Expand Down Expand Up @@ -268,8 +269,9 @@ func (r *Relay) CheckOwnerLists() {
// need to search DB for moderator npub follow lists, followed npubs are allowed access.
if len(r.Followed) < 1 {
// add the owners themselves of course
for i := range r.Owners {
r.Followed[st(r.Owners[i])] = struct{}{}
for _, v := range r.Owners {
r.Followed[st(v)] = struct{}{}
r.OwnersFollowed[st(v)] = struct{}{}
}
log.D.Ln("regenerating owners follow lists")
if evs, err = r.Store.QueryEvents(r.Ctx,
Expand All @@ -280,12 +282,12 @@ func (r *Relay) CheckOwnerLists() {
r.OwnersFollowLists = append(r.OwnersFollowLists, ev.ID)
for _, t := range ev.Tags.F() {
if equals(t.Key(), by("p")) {
var p by
if p, err = hex.Dec(st(t.Value())); chk.E(err) {
var v by
if v, err = hex.Dec(st(t.Value())); chk.E(err) {
continue
}
r.Followed[st(p)] = struct{}{}
r.OwnersFollowed[st(p)] = struct{}{}
r.Followed[st(v)] = struct{}{}
r.OwnersFollowed[st(v)] = struct{}{}
}
}
}
Expand All @@ -306,11 +308,11 @@ func (r *Relay) CheckOwnerLists() {
r.OwnersFollowLists = append(r.OwnersFollowLists, ev.ID)
for _, t := range ev.Tags.F() {
if equals(t.Key(), by("p")) {
var p by
if p, err = hex.Dec(st(t.Value())); err != nil {
var v by
if v, err = hex.Dec(st(t.Value())); err != nil {
continue
}
r.Followed[st(p)] = struct{}{}
r.Followed[st(v)] = struct{}{}
}
}
}
Expand Down
173 changes: 168 additions & 5 deletions cmd/realy/app/spider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ import (
"time"
"realy.lol/kinds"
"realy.lol/kind"
"realy.lol/event"
"realy.lol/filter"
"realy.lol/tag"
"strings"
"realy.lol/ws"
)

func (r *Relay) Spider() {
Expand All @@ -20,7 +24,7 @@ func (r *Relay) Spider() {
// re-run the spider every hour to catch any updates that for whatever
// reason permitted users may have uploaded to other relays via other
// clients that may not be sending to us.
ticker := time.NewTicker(time.Hour)
ticker := time.NewTicker(time.Hour * 24)
for {
select {
case <-r.Ctx.Done():
Expand All @@ -41,9 +45,168 @@ var RelayKinds = &kinds.T{

// spider is the actual function that does a spider run
func (r *Relay) spider() {
// first find all the relays that we currently know about.
filter := filter.T{Kinds: RelayKinds}
var err er
var evs event.Ts
sto := r.Storage()
_ = filter
_ = sto
// count how many npubs we need to search with
r.Lock()
nUsers := len(r.Followed)
// n := r.MaxLimit / 2
// we probably want to be conservative with how many we query at once
// on rando relays, so make `n` small
n := 20
nQueries := nUsers / n
// make the list
users := make([]st, 0, nUsers)
for v := range r.Followed {
users = append(users, v)
}
r.Unlock()
// break into chunks for each query
chunks := make([][]st, 0, nQueries)
// starting from the nearest integer (from the total divided by the number
// per chunk) we know
for i := nQueries * n; i > 0; i -= n {
// take the last segment
last := users[i:]
// if it happens to be a round number, don't collect an empty slice
if len(last) < 1 {
continue
}
chunks = append(chunks, users[i:])
// snip what we took out of the main slice
users = users[:i]
}
relays := make(map[st]struct{})
usersWithRelays := make(map[st]struct{})
for _, v := range chunks {
f := &filter.T{Kinds: RelayKinds, Authors: tag.New(v...)}
if evs, err = sto.QueryEvents(r.Ctx, f); chk.E(err) {
// fatal
return
}
// log.I.F("%d relay events found", len(evs))
for _, ev := range evs {
relays, usersWithRelays = filterRelays(ev, relays, usersWithRelays)
}
}
log.I.F("%d relays found in db, of %d users",
len(relays), len(usersWithRelays))
log.W.F("****************** starting spider ******************")
// now spider all these relays for the users, and get even moar relays
spide:
for rely := range relays {
select {
case <-r.Ctx.Done():
var o st
for v := range relays {
o += v + "\n"
}
// log.I.F("found relays: %d\n%s", len(relays), o)
log.W.F("shutting down")
return
default:
}
rl := ws.NewRelay(r.Ctx, rely)
if err = rl.Connect(r.Ctx); chk.E(err) {
// chk.E(rl.Close())
continue spide
}
log.D.F("connected to '%s'", rely)
// first get some estimate of how many of these events the relay has, if
// possible
var count no
for i, v := range chunks {
log.D.F("chunk %d/%d from %s so far: %d relays %d users %d",
i, len(chunks), rely, count, len(relays), len(usersWithRelays))
f := &filter.T{Kinds: RelayKinds, Authors: tag.New(v...)}
if evs, err = rl.QuerySync(r.Ctx, f); chk.E(err) {
chk.E(rl.Close())
continue spide
}
count += len(evs)
for _, ev := range evs {
relays, usersWithRelays = filterRelays(ev, relays,
usersWithRelays)
}
}
log.I.F("got %d results from %s", count, rely)
chk.E(rl.Close())
}
// var o st
// for v := range relays {
// o += v + "\n"
// }
log.I.F("%d relays found, of %d users",
len(relays), len(usersWithRelays))
}

func filterRelays(ev *event.T,
relays, usersWithRelays map[st]struct{}) (r, u map[st]struct{}) {
// log.I.S(ev)
var foundSome bo
t := ev.Tags.GetAll(tag.New("r"))
next:
for _, tr := range t.F() {
v := st(tr.Value())
if len(v) < 5 {
continue
}
// we only want wss, very often ws:// is not routeable address.
if !strings.HasPrefix(v, "wss") {
continue
}
// remove ones with extra shit after the relay
if strings.ContainsAny(v, " \n\r\f\t") {
for i := range v {
switch v[i] {
case ' ', '\n', '\t', '\r', '\f':
// log.I.F("%s", v)
// log.I.F("%s", ev.Serialize())
v = v[:i]
continue next
}
}
}
// we don't want URLs with query parameters, mostly nostr.wine
// these are not interesting. also, if they have @ symbols. or
// = in case the user didn't put the ? in properly also.
if strings.Contains(v, "\"") {
log.E.F("%s", v)
panic("wtf")
continue
}
if strings.Contains(v, "?") ||
strings.Contains(v, "@") ||
strings.Contains(v, "=") {
// log.E.F("%s", v)
continue
}

// get rid of the slashes
if strings.HasSuffix(v, "/") {
// trim it off
v = v[:len(v)-1]
}
// and lastly, we aren't going to use tor for this, so, nope.
// also no .local this is not routeable
if strings.Contains(v, ".onion") ||
strings.Contains(v, ".local") {
continue
}
// weirdly sometimes there is addresses with multiple mangled
// protocol things in them, this is a waste of time also.
if len(strings.Split(v, "//")) > 2 {
continue
}
// finally, because people are dumb and don't know that URLs are
// case-insensitive, standardise them
v = strings.ToLower(v)
relays[v] = struct{}{}
foundSome = true
}
if foundSome {
usersWithRelays[st(ev.PubKey)] = struct{}{}
}
return relays, usersWithRelays
}
28 changes: 23 additions & 5 deletions cmd/realy/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ import (
"realy.lol/realy"
"realy.lol/realy/config"
"realy.lol/units"
"realy.lol/bech32encoding"
"realy.lol/hex"
"realy.lol/ec/secp256k1"
)

func main() {
Expand All @@ -36,6 +39,22 @@ func main() {
os.Exit(0)
}
log.I.Ln("log level", cfg.LogLevel)
var prf by
var val any
spiderKey := make(by, secp256k1.SecKeyBytesLen)
if prf, val, err = bech32encoding.Decode(by(cfg.SpiderKey)); chk.E(err) {
log.E.F("SPIDER_KEY decode error: '%s' hrp: %s", err.Error(), prf)
spiderKey = nil
} else {
if sk, ok := val.(by); ok {
var n no
if n, err = hex.DecBytes(spiderKey, sk); chk.E(err) {
log.E.F("failed to decode hex: '%s' at %d",
err.Error(), n)
spiderKey = nil
}
}
}
lol.SetLogLevel(cfg.LogLevel)
if cfg.Pprof {
defer profile.Start(profile.MemProfile).Stop()
Expand All @@ -52,7 +71,7 @@ func main() {
WG: &wg,
BlockCacheSize: units.Gb,
LogLevel: lol.GetLogLevel(cfg.DbLogLevel),
MaxLimit: ratel.DefaultMaxLimit,
MaxLimit: cfg.MaxLimit,
UseCompact: cfg.UseCompact,
Compression: cfg.Compression,
Extra: []no{
Expand All @@ -63,20 +82,19 @@ func main() {
},
},
)
r := &app.Relay{Ctx: c, C: cfg, Store: storage}
r := &app.Relay{Ctx: c, C: cfg, Store: storage, MaxLimit: cfg.MaxLimit}
go app.MonitorResources(c)
var server *realy.Server
if server, err = realy.NewServer(realy.ServerParams{
Ctx: c,
Cancel: cancel,
Rl: r,
DbPath: cfg.Profile,
MaxLimit: ratel.DefaultMaxLimit,
MaxLimit: cfg.MaxLimit,
AdminUser: cfg.AdminUser,
AdminPass: cfg.AdminPass,
SpiderKey: cfg.SpiderKey,
SpiderKey: spiderKey,
}); chk.E(err) {

os.Exit(1)
}
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion envelopes/eventenvelope/eventenvelope.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,16 @@ type Result struct {

var _ codec.Envelope = (*Result)(nil)

func NewResult() *Result { return &Result{} }
func NewResult() *Result { return &Result{Event: &event.T{}} }

func NewResultWith[V st | by](s V, ev *event.T) (res *Result, err er) {
if len(s) < 0 || len(s) > 64 {
err = errorf.E("subscription id must be length > 0 and <= 64")
return
}
return &Result{subscription.MustNew(s), ev}, nil
}

func (en *Result) Label() st { return L }

func (en *Result) Write(w io.Writer) (err er) {
Expand Down Expand Up @@ -114,6 +116,7 @@ func (en *Result) Unmarshal(b by) (r by, err er) {
if r, err = envelopes.SkipToTheEnd(r); chk.E(err) {
return
}
// log.I.S(en.Event)
return
}

Expand Down
22 changes: 21 additions & 1 deletion envelopes/eventenvelope/eventenvelope_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ func TestResult(t *testing.T) {
rem)
}
var ea *Result
if ea, err = NewResultWith(subscription.NewStd().String(), ev); chk.E(err) {
if ea, err = NewResultWith(subscription.NewStd().String(),
ev); chk.E(err) {
t.Fatal(err)
}
rem = ea.Marshal(rem)
Expand All @@ -92,3 +93,22 @@ func TestResult(t *testing.T) {
rem, c, out = rem[:0], c[:0], out[:0]
}
}

func TestResult_Unmarshal(t *testing.T) {
var err er
evbs := []by{
// by(`["EVENT",":9",{"id":"8031e5c66be1bb00ee55e4902c74fb37d447b9cddda2e2591d21c9bb30a17f60","kind":10002,"pubkey":"146bda4ec6932830503ee4f8e8b626bd7b3a5784232b8240ba15c8cbff9a07cd","created_at":1733221976,"content":"","tags":[["r","wss://relay.primal.net/"],["r","wss://nostr.mom/"],["r","wss://nos.lol/"],["r","wss://bitcoiner.social/"],["r","wss://relay.nostr.bg/"],["r","wss://nostr.oxtr.dev/"],["r","wss://relay.wellorder.net/"],["r","wss://nostr.wine/"],["r","wss://eden.nostr.land/"],["r","wss://offchain.pub/"],["r","wss://nostr-pub.wellorder.net/"],["r","wss://atlas.nostr.land/"],["r","wss://relay.stoner.com/"],["r","wss://puravida.nostr.land/"],["r","wss://nostr.coinfundit.com/"],["r","wss://relay.plebstr.com/"],["r","wss://relay.snort.social/"],["r","wss://nostr.zebedee.cloud/"],["r","wss://nostr.walletofsatoshi.com/"],["r","wss://purplepag.es/"],["r","wss://relay.nostrich.de/"],["r","wss://brb.io/"],["r","wss://relay.siamstr.com/"],["r","wss://relay.nostr.com.au/"],["r","wss://nostr.milou.lol/"],["r","wss://relay.orangepill.dev/"],["r","wss://nostr.bitcoiner.social/"],["r","wss://relay.nostr.band/"],["r","wss://relay.damus.io/"],["r","wss://relay.0xchat.com/"]],"sig":"7a3a1a16ed30e0f3f64d1ce2425cb68ed6cd383afde16f7422dbe691bc745a85b8c59d3292fb6daad22604699552abad9090b534900a33400ed175a3acdf2521"}]`),
// by(`["EVENT",":4",{"content":"","created_at":1713250740,"id":"79c22b961e808bb94a4c8b92e681f0c72175878c4f4c3d72c47bb179b0dbfb4e","kind":10002,"pubkey":"43debddd908e677f0b559fff1f5b2d99daf8672eb7f88e5ddf7190c9a65e0ca8","sig":"4e690d173abacf70d0bd56cace795622a7ef0a4b513bc5ebe73c7173fd013003051f430e8d6bf189ce6d46251bb814f55f0487b23da31fb35e82d0610a212c13","tags":[["r","wss://nos.lol/"],["r","wss://nostr.bitcoiner.social/"],["r","wss://relay.nostr.band/"],["r","wss://nostr-pub.wellorder.net/"],["r","wss://offchain.pub/"],["r","wss://relay.mostr.pub/"],["r","wss://purplepag.es/"],["r","wss://relay.damus.io/"],["r","wss://nostr.oxtr.dev/"],["r","wss://xmr.usenostr.org/"],["r","wss://relay.snort.social/"],["r","wss://relay.nostrss.re/"],["r","wss://relay.nostr.directory/"],["r","wss://relayable.org/"],["r","wss://nostr.skitso.business/"],["r","wss://nostr.sidnlabs.nl/"],["r","wss://relay.blackbyte.nl/"],["r","wss://milwaukie.nostr1.com/"],["r","wss://GALAXY13.nostr1.com/"],["r","wss://21ideas.nostr1.com/"],["r","wss://support.nostr1.com/"],["r","wss://pater.nostr1.com/"],["r","wss://ryan.nostr1.com/"],["r","wss://nostr21.com/"]]}]`),
// by(`["EVENT",":3",{"content":"","created_at":1685471660,"id":"b8cb76ed134c0d245d2ed97ae5fae916d6f727dafa6ca25777caa4d6bdcc6e62","kind":10002,"pubkey":"340e4aef86cd5240aaf5fc550fa3f4291e6a03281a469b6ef34f060f6944033f","sig":"4e1661f8561c820aee150b6e841b309e3772eb7f89123aa7b3a3ffc6e88445fa6c3368d88d002987dd883227908aaaa40eee1819ac53ba632ae4df32a695e7b9","tags":[["r","wss://relay.damus.io"],["r","wss://relay.nostr.band"],["r","wss://btc-italia.online"],["r","wss://bitcoiner.social"],["client","coracle"]]}]`),
// by(`["EVENT",":2",{"content":"","created_at":1707267379,"id":"fd9b42c626277d763877d526043552e0c01fd7cea34b68a191b6981aeb20cb90","kind":10002,"pubkey":"c5fb6ecc876e0458e3eca9918e370cbcd376901c58460512fe537a46e58c38bb","sig":"a4db470fed2d18a341e7bd4e2b15ab9ed9c2e6c5df9d213d3f9ec1b0d848a18b39358fc31be97e78a2d1e19016ea6ce5603bd1645bc50ccfe76d566a40bf1a2f","tags":[["r","wss://nostr21.com"],["r","wss://blastr.f7z.xyz"],["r","wss://relay.orangepill.dev/"],["r","wss://relay.nostriches.org/"],["r","wss://relayable.org"],["r","wss://relay.nostr.band/"],["r","wss://purplepag.es"],["r","wss://welcome.nostr.wine"],["r","wss://nos.lol"],["r","wss://relay.nostrview.com/"],["r","wss://relay.damus.io"],["r","wss://filter.nostr.wine/npub1chakany8dcz93clv4xgcudcvhnfhdyqutprq2yh72daydevv8zasmuhf02?broadcast=true"],["r","wss://nostr.fmt.wiz.biz"],["r","wss://eden.nostr.land"],["r","wss://nostr.plebchain.org/"],["r","wss://nostr.wine"],["r","wss://atlas.nostr.land"],["r","wss://offchain.pub/"],["r","wss://relay.nostrati.com/"],["r","wss://pyramid.fiatjaf.com"]]}]`),
by(`["EVENT",":3",{"kind":10002,"id":"6ce2cb371eb16d3ba50dc49c5290d05b84ff5dd9c50b89e3e74ca89af9fc89c4","pubkey":"e58143f793e4bf805a4df6cdc0289e352b3cf08a7b3e6afaaf89dd497bf0f4a6","created_at":1731866484,"tags":[["r","wss://eden.nostr.land"],["r","wss://nos.lol"],["r","wss://nostr.fmt.wiz.biz"],["r","wss://nostr.wine"],["r","wss://relay.damus.io"],["r","wss://relay.mostr.pub"],["r","wss://nostrelites.org"],["r","wss://relay.bitcoinpark.com"],["r","wss://sendit.nosflare.com"],["r","wss://wot.nostr.party"]],"content":"","sig":"eb73a422adaacdf9ded42e49ab09bc1227b06b8d2c286b0775d9e2b532f8cc8e6c32de1eb27d89bc70159469d79473e9a0150b29c618abe8684ad524ee82d941"}]`),
by(`["EVENT",":2",{"content":"","created_at":1727465947,"id":"47bf33c78d58be44b63bac818e2e19597972937d79c54b84ae0bf2a08622edd2","kind":10002,"pubkey":"34ca937f6e91550633ff4d8381b388b0cca22d212ff8e7b953f0f458cb16e915","sig":"c59260219bf0fbe0767d9157183c83b3e1d89f7689c92892a4f904dce0017f993f84b006cfe0b113d70b0332b55d1d3c9bfc0b78accb6aa9bb4915648a05c572","tags":[["r","wss://nos.lol/"],["r","wss://relay.primal.net/"],["r","wss://relay.nostr.band/"],["r","wss://relay.damus.io/"],["r","wss://relay.nostrplebs.com/"],["r","wss://nostr-pub.wellorder.net/","write"],["r","wss://nostr.nodeofsven.com/"],["r","wss://nostr.einundzwanzig.space/"],["r","wss://nostr.walletofsatoshi.com/"],["r","wss://relay.snort.social/"],["r","wss://nostr.thank.eu/"],["r","wss://nostr.hifish.org/"],["r","wss://pyramid.fiatjaf.com/"],["r","wss://relay.wellorder.net/","read"]]}]`),
}
for _, b := range evbs {
ev := NewResult()
if _, err = ev.Unmarshal(b); chk.E(err) {
t.Fatal(err)
}
log.I.S(ev.Event)
}
}
Loading

0 comments on commit a485d24

Please sign in to comment.