spider is pretty much right

auth/nip42.go

@@ -25,13 +25,12 @@ func GenerateChallenge() (b by) {
 
 // CreateUnsigned creates an event which should be sent via an "AUTH" command.
 // If the authentication succeeds, the user will be authenticated as pubkey.
-func CreateUnsigned(pubkey, challenge by, relayURL string) (ev *event.T) {
+func CreateUnsigned(challenge by, relayURL st) (ev *event.T) {
 	return &event.T{
-		PubKey:    pubkey,
 		CreatedAt: timestamp.Now(),
 		Kind:      kind.ClientAuthentication,
 		Tags: tags.New(tag.New("relay", relayURL),
-			tag.New("challenge", string(challenge))),
+			tag.New("challenge", st(challenge))),
 	}
 }
 

auth/nip42_test.go

@@ -16,7 +16,7 @@ func TestCreateUnsigned(t *testing.T) {
 	const relayURL = "wss://example.com"
 	for _ = range 100 {
 		challenge := GenerateChallenge()
-		ev := CreateUnsigned(signer.Pub(), challenge, relayURL)
+		ev := CreateUnsigned(challenge, relayURL)
 		if err = ev.Sign(signer); chk.E(err) {
 			t.Fatal(err)
 		}

bech32encoding/keys.go

@@ -22,12 +22,16 @@ var (
 )
 
 // ConvertForBech32 performs the bit expansion required for encoding into Bech32.
-func ConvertForBech32(b8 by) (b5 by, err er) { return bech32.ConvertBits(b8, 8,
-	5, true) }
+func ConvertForBech32(b8 by) (b5 by, err er) {
+	return bech32.ConvertBits(b8, 8,
+		5, true)
+}
 
 // ConvertFromBech32 collapses together the bit expanded 5 bit numbers encoded in bech32.
-func ConvertFromBech32(b5 by) (b8 by, err er) { return bech32.ConvertBits(b5, 5,
-	8, true) }
+func ConvertFromBech32(b5 by) (b8 by, err er) {
+	return bech32.ConvertBits(b5, 5,
+		8, true)
+}
 
 // SecretKeyToNsec encodes an secp256k1 secret key as a Bech32 string (nsec).
 func SecretKeyToNsec(sk *secp256k1.SecretKey) (encoded by, err er) {
@@ -170,6 +174,7 @@ func BinToNsec(sk by) (nsec by, err er) {
 
 // SecretKeyToHex converts a secret key to the hex encoding.
 func SecretKeyToHex(sk *btcec.SecretKey) (hexSec by) {
+	hexSec = make(by, btcec.SecKeyBytesLen*2)
 	hex.EncBytes(hexSec, sk.Serialize())
 	return
 }

bech32encoding/keys_test.go

@@ -2,9 +2,8 @@ package bech32encoding
 
 import (
 	"crypto/rand"
-	"encoding/hex"
 	"testing"
-
+	"realy.lol/hex"
 	"realy.lol/ec/schnorr"
 	"realy.lol/ec/secp256k1"
 )
@@ -51,7 +50,7 @@ func TestSecretKeyToNsec(t *testing.T) {
 		reSecBytes = reSec.Serialize()
 		if st(secBytes) != st(reSecBytes) {
 			t.Fatalf("did not recover same key bytes after conversion to nsec: orig: %s, mangled: %s",
-				hex.EncodeToString(secBytes), hex.EncodeToString(reSecBytes))
+				hex.Enc(secBytes), hex.Enc(reSecBytes))
 		}
 		if reNsec, err = SecretKeyToNsec(reSec); chk.E(err) {
 			t.Fatalf("error recovered secret key from converted to nsec: %s",
@@ -87,10 +86,11 @@ func TestPublicKeyToNpub(t *testing.T) {
 		rePubBytes = schnorr.SerializePubKey(rePub)
 		if st(pubBytes) != st(rePubBytes) {
 			t.Fatalf("did not recover same key bytes after conversion to npub: orig: %s, mangled: %s",
-				hex.EncodeToString(pubBytes), hex.EncodeToString(rePubBytes))
+				hex.Enc(pubBytes), hex.Enc(rePubBytes))
 		}
 		if reNpub, err = PublicKeyToNpub(rePub); chk.E(err) {
-			t.Fatalf("error recovered secret key from converted to nsec: %s", err)
+			t.Fatalf("error recovered secret key from converted to nsec: %s",
+				err)
 		}
 		if !equals(reNpub, npub) {
 			t.Fatalf("recovered public key did not regenerate npub of original: %s mangled: %s",

cmd/realy/app/implementation.go

@@ -302,7 +302,7 @@ func (r *Relay) CheckOwnerLists() {
 			}
 			if evs, err = r.Store.QueryEvents(r.Ctx,
 				&filter.T{Authors: tag.New(followed...),
-					Kinds: kinds.New(kind.FollowList)}); chk.E(err) {
+					Kinds: kinds.New(kind.FollowList)}, true); chk.E(err) {
 			}
 			for _, ev := range evs {
 				// we want to protect the follow lists of users as well, they
@@ -321,7 +321,8 @@ func (r *Relay) CheckOwnerLists() {
 			evs = evs[:0]
 		}
 		// log this info
-		log.I.F("%d allowed npubs", len(r.Followed))
+		log.I.F("%d owner followed; %d allowed npubs",
+			len(r.OwnersFollowed), len(r.Followed))
 		// r.Followed
 		// r.OwnersFollowed
 		// o := "followed:\n"

cmd/realy/app/spider.go

@@ -22,17 +22,17 @@ func (r *Relay) Spider() {
 		return
 	}
 	// we run at first startup
-	r.spider()
+	r.spider(true)
 	// re-run the spider every hour to catch any updates that for whatever
 	// reason permitted users may have uploaded to other relays via other
 	// clients that may not be sending to us.
-	ticker := time.NewTicker(time.Hour * 24)
+	ticker := time.NewTicker(time.Hour)
 	for {
 		select {
 		case <-r.Ctx.Done():
 			return
 		case <-ticker.C:
-			r.spider()
+			r.spider(false)
 		}
 	}
 }
@@ -46,7 +46,7 @@ var RelayKinds = &kinds.T{
 }
 
 // spider is the actual function that does a spider run
-func (r *Relay) spider() {
+func (r *Relay) spider(full bo) {
 	log.I.F("spidering")
 	if r.SpiderSigner == nil {
 		panic("bro the signer still not hear")
@@ -60,14 +60,27 @@ func (r *Relay) spider() {
 	// n := r.MaxLimit / 2
 	// we probably want to be conservative with how many we query at once
 	// on rando relays, so make `n` small
-	n := 100
+	n := r.MaxLimit
 	nQueries := nUsers / n
 	// make the list
 	users := make([]st, 0, nUsers)
 	for v := range r.Followed {
 		users = append(users, v)
 	}
 	r.Unlock()
+	relays := make(map[st]struct{})
+	relaysUsed := make(map[st]struct{})
+	usersWithRelays := make(map[st]struct{})
+	log.I.F("finding relay events")
+	f := &filter.T{Kinds: RelayKinds, Authors: tag.New(users...)}
+	if evs, err = sto.QueryEvents(r.Ctx, f, true); chk.E(err) {
+		// fatal
+		return
+	}
+	for _, ev := range evs {
+		relays, usersWithRelays = filterRelays(ev, relays, usersWithRelays)
+	}
+
 	log.I.F("making query chunks")
 	// break into chunks for each query
 	chunks := make([][]st, 0, nQueries)
@@ -84,20 +97,17 @@ func (r *Relay) spider() {
 		// snip what we took out of the main slice
 		users = users[:i]
 	}
-	relays := make(map[st]struct{})
-	relaysUsed := make(map[st]struct{})
-	usersWithRelays := make(map[st]struct{})
-	for _, v := range chunks {
-		f := &filter.T{Kinds: RelayKinds, Authors: tag.New(v...)}
-		if evs, err = sto.QueryEvents(r.Ctx, f); chk.E(err) {
-			// fatal
-			return
-		}
-		log.D.F("%d relay events found", len(evs))
-		for _, ev := range evs {
-			relays, usersWithRelays = filterRelays(ev, relays, usersWithRelays)
-		}
-	}
+	// for i, v := range chunks {
+	// 	f := &filter.T{Kinds: RelayKinds, Authors: tag.New(v...)}
+	// 	if evs, err = sto.QueryEvents(r.Ctx, f); chk.E(err) {
+	// 		// fatal
+	// 		return
+	// 	}
+	// 	log.D.F("%d relay events found %d/%d", len(evs), i, len(chunks))
+	// 	for _, ev := range evs {
+	// 		relays, usersWithRelays = filterRelays(ev, relays, usersWithRelays)
+	// 	}
+	// }
 	log.I.F("%d relays found in db, of %d users",
 		len(relays), len(usersWithRelays))
 	log.W.F("****************** starting spider ******************")
@@ -106,9 +116,8 @@ func (r *Relay) spider() {
 	var found no
 spide:
 	for rely := range relays {
-		if found > 5 {
-			// that's enough for now
-			log.I.S("got events from %d relays queried, "+
+		if found > 2 {
+			log.W.F("got events from %d relays queried, "+
 				"finishing spider for today", found)
 			return
 		}
@@ -131,11 +140,12 @@ spide:
 				rely, len(relays))
 			continue spide
 		}
-		if !inf.Limitation.AuthRequired {
-			continue spide
-		}
+		// if !inf.Limitation.AuthRequired {
+		// 	continue spide
+		// }
 		log.I.S(inf)
-		rl := ws.NewRelay(r.Ctx, rely, r.SpiderSigner)
+		var rl *ws.Client
+		rl, err = ws.ConnectWithAuth(r.Ctx, rely, r.SpiderSigner)
 		if err = rl.Connect(r.Ctx); chk.E(err) {
 			// chk.E(rl.Close())
 			continue spide
@@ -146,11 +156,18 @@ spide:
 		// possible
 		var count no
 		var average time.Duration
-		var looked bo
 		for i, v := range chunks {
 			log.D.F("chunk %d/%d from %s so far: %d relays %d users %d, av response %v",
 				i, len(chunks), rely, count, len(relays), len(usersWithRelays),
 				average)
+			if i > 3 {
+				if average > time.Second {
+					log.I.F("relay %s is throttling us, move on", rely)
+					chk.E(rl.Close())
+					continue spide
+				}
+				found++
+			}
 			f := &filter.T{
 				Kinds:   &kinds.T{K: kind.Directory},
 				Authors: tag.New(v...),
@@ -163,34 +180,18 @@ spide:
 			average += time.Now().Sub(started)
 			average /= 2
 			count += len(evs)
-			if !looked && count > 5 {
-				if len(evs) > 0 {
-					looked = true
-					found++
-				}
-			}
 			for _, ev := range evs {
 				relays, usersWithRelays = filterRelays(ev, relays,
 					usersWithRelays)
-				if err = r.Storage().SaveEvent(r.Ctx, ev); chk.E(err) {
+				if err = r.Storage().SaveEvent(r.Ctx, ev); err != nil {
 					continue
 				}
 			}
-			if i > 5 {
-				if average > time.Second {
-					log.I.F("relay %s is throttling us, move on", rely)
-					chk.E(rl.Close())
-					continue spide
-				}
-			}
 		}
-		log.I.F("got %d results from %s", count, rely)
+		log.I.F("%d found so far in this spider run; "+
+			"got %d results from %s", found, count, rely)
 		chk.E(rl.Close())
 	}
-	// var o st
-	// for v := range relays {
-	// 	o += v + "\n"
-	// }
 	log.I.F("%d relays found, of %d users",
 		len(relays), len(usersWithRelays))
 	// filter out the relays we used

cmd/realy/main.go

@@ -105,7 +105,7 @@ func main() {
 	if server, err = realy.NewServer(realy.ServerParams{
 		Ctx:       c,
 		Cancel:    cancel,
-		Rl:        r,
+		I:         r,
 		DbPath:    cfg.Profile,
 		MaxLimit:  cfg.MaxLimit,
 		AdminUser: cfg.AdminUser,

envelopes/authenvelope/authenvelope_test.go

@@ -46,8 +46,7 @@ func TestAuth(t *testing.T) {
 		if !equals(oChal, b2) {
 			t.Fatalf("challenge mismatch\n%s\n%s", oChal, b2)
 		}
-		resp := Response{Event: auth.CreateUnsigned(signer.Pub(), ch,
-			relayURL)}
+		resp := Response{Event: auth.CreateUnsigned(ch, relayURL)}
 		if err = resp.Event.Sign(signer); chk.E(err) {
 			t.Fatal(err)
 		}

envelopes/countenvelope/countenvelope.go

@@ -21,13 +21,14 @@ type Request struct {
 
 var _ codec.Envelope = (*Request)(nil)
 
-func New() *Request {
-	return &Request{Subscription: subscription.NewStd(),
-		Filters: filters.New()}
-}
-func NewRequest(id *subscription.Id, filters *filters.T) *Request {
-	return &Request{Subscription: id,
-		Filters: filters}
+func NewRequest(id *subscription.Id, ff *filters.T) *Request {
+	if id == nil {
+		id = subscription.NewStd()
+	}
+	if ff == nil {
+		ff = filters.New()
+	}
+	return &Request{Subscription: id, Filters: ff}
 }
 func (en *Request) Label() string { return L }
 func (en *Request) Write(w io.Writer) (err er) {
@@ -71,7 +72,7 @@ func (en *Request) Unmarshal(b by) (r by, err er) {
 }
 
 func ParseRequest(b by) (t *Request, rem by, err er) {
-	t = New()
+	t = NewRequest(nil, nil)
 	if rem, err = t.Unmarshal(b); chk.E(err) {
 		return
 	}
@@ -87,7 +88,8 @@ type Response struct {
 var _ codec.Envelope = (*Response)(nil)
 
 func NewResponse() *Response { return &Response{ID: subscription.NewStd()} }
-func NewResponseFrom[V st | by](s V, cnt no, approx ...bo) (res *Response, err er) {
+func NewResponseFrom[V st | by](s V, cnt no, approx ...bo) (res *Response,
+	err er) {
 	var a bo
 	if len(approx) > 0 {
 		a = approx[0]

envelopes/countenvelope/countenvelope_test.go

@@ -32,7 +32,7 @@ func TestRequest(t *testing.T) {
 		if l != L {
 			t.Fatalf("invalid sentinel %s, expect %s", l, L)
 		}
-		req2 := New()
+		req2 := NewRequest(nil, nil)
 		if rem, err = req2.Unmarshal(rb); chk.E(err) {
 			t.Fatal(err)
 		}

filter/filter.go

@@ -18,6 +18,7 @@ import (
 	"realy.lol/tags"
 	"realy.lol/text"
 	"realy.lol/timestamp"
+	"golang.org/x/exp/constraints"
 )
 
 func Present(i *uint) bo { return i != nil }
@@ -42,6 +43,11 @@ type T struct {
 	Limit   *uint        `json:"limit,omitempty"`
 }
 
+func L[V constraints.Integer](l V) (u *uint) {
+	uu := uint(l)
+	return &uu
+}
+
 func New() (f *T) {
 	return &T{
 		IDs:     tag.NewWithCap(10),
@@ -317,7 +323,8 @@ func (f *T) Unmarshal(b by) (r by, err er) {
 					goto invalid
 				}
 				var ff []by
-				if ff, r, err = text.UnmarshalHexArray(r, schnorr.PubKeyBytesLen); chk.E(err) {
+				if ff, r, err = text.UnmarshalHexArray(r,
+					schnorr.PubKeyBytesLen); chk.E(err) {
 					return
 				}
 				f.Authors = tag.New(ff...)

go.mod

@@ -13,12 +13,14 @@ require (
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
 	github.com/klauspost/cpuid/v2 v2.2.9
 	github.com/pkg/profile v1.7.0
+	github.com/puzpuzpuz/xsync/v2 v2.5.1
 	github.com/puzpuzpuz/xsync/v3 v3.4.0
 	github.com/rs/cors v1.11.1
 	github.com/stretchr/testify v1.10.0
 	github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b
 	go-simpler.org/env v0.12.0
 	golang.org/x/crypto v0.31.0
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	golang.org/x/lint v0.0.0-20241112194109-818c5a804067
 	golang.org/x/net v0.32.0
 	golang.org/x/sync v0.10.0