cleaning up signer

bech32encoding/decoders.go

@@ -0,0 +1,39 @@
+package bech32encoding
+
+import (
+	"realy.lol/ec/bech32"
+)
+
+func DecodeNsec[V st | by](nsec V) (skb by, err er) {
+	sks := by(nsec)
+	var prefix, bits5 by
+	if prefix, bits5, err = bech32.DecodeNoLimit(sks); chk.D(err) {
+		return
+	}
+	if !equals(prefix, NsecHRP) {
+		err = errorf.E("incorrect prefix for nsec: %s", prefix)
+		return
+	}
+	if skb, err = bech32.ConvertBits(bits5, 5, 8,
+		false); chk.D(err) {
+
+		return
+	}
+	return
+}
+
+func DecodeNpub[V st | by](nsec V) (skb by, err er) {
+	pks := by(nsec)
+	var prefix, bits5 by
+	if prefix, bits5, err = bech32.DecodeNoLimit(pks); chk.D(err) {
+		return
+	}
+	if !equals(prefix, NpubHRP) {
+		err = errorf.E("incorrect prefix for npub: %s", prefix)
+		return
+	}
+	if skb, err = bech32.Convert5to8(bits5, false); chk.D(err) {
+		return
+	}
+	return
+}

bech32encoding/keys.go

@@ -22,10 +22,12 @@ var (
 )
 
 // ConvertForBech32 performs the bit expansion required for encoding into Bech32.
-func ConvertForBech32(b8 by) (b5 by, err er) { return bech32.ConvertBits(b8, 8, 5, true) }
+func ConvertForBech32(b8 by) (b5 by, err er) { return bech32.ConvertBits(b8, 8,
+	5, true) }
 
 // ConvertFromBech32 collapses together the bit expanded 5 bit numbers encoded in bech32.
-func ConvertFromBech32(b5 by) (b8 by, err er) { return bech32.ConvertBits(b5, 5, 8, true) }
+func ConvertFromBech32(b5 by) (b8 by, err er) { return bech32.ConvertBits(b5, 5,
+	8, true) }
 
 // SecretKeyToNsec encodes an secp256k1 secret key as a Bech32 string (nsec).
 func SecretKeyToNsec(sk *secp256k1.SecretKey) (encoded by, err er) {
@@ -136,9 +138,9 @@ func HexToNpub(publicKeyHex by) (s by, err er) {
 	return bech32.Encode(NpubHRP, bits5)
 }
 
-func BinToNpub(b by) (s by, err er) {
+func BinToNpub(b by) (npub by, err er) {
 	var bits5 by
-	if bits5, err = bech32.ConvertBits(b, 8, 5, true); chk.D(err) {
+	if bits5, err = bech32.Convert8to5(b, true); chk.D(err) {
 		return nil, err
 	}
 	return bech32.Encode(NpubHRP, bits5)

cmd/realy/app/implementation.go

@@ -20,6 +20,7 @@ import (
 	"realy.lol/realy/config"
 	"realy.lol/store"
 	"realy.lol/tag"
+	"realy.lol/signer"
 )
 
 type Relay struct {
@@ -36,7 +37,8 @@ type Relay struct {
 	OwnersFollowLists []by
 	// KnownRelays is a map populated by the Spider from all RelayKinds events
 	// found on the relay.
-	KnownRelays map[st]struct{}
+	KnownRelays  map[st]struct{}
+	SpiderSigner signer.I
 }
 
 func (r *Relay) Name() st { return r.C.AppName }

cmd/realy/app/spider.go

@@ -9,6 +9,8 @@ import (
 	"realy.lol/tag"
 	"strings"
 	"realy.lol/ws"
+	"realy.lol/relayinfo"
+	"net/url"
 )
 
 func (r *Relay) Spider() {
@@ -45,6 +47,10 @@ var RelayKinds = &kinds.T{
 
 // spider is the actual function that does a spider run
 func (r *Relay) spider() {
+	log.I.F("spidering")
+	if r.SpiderSigner == nil {
+		panic("bro the signer still not hear")
+	}
 	var err er
 	var evs event.Ts
 	sto := r.Storage()
@@ -54,14 +60,15 @@ func (r *Relay) spider() {
 	// n := r.MaxLimit / 2
 	// we probably want to be conservative with how many we query at once
 	// on rando relays, so make `n` small
-	n := 20
+	n := 100
 	nQueries := nUsers / n
 	// make the list
 	users := make([]st, 0, nUsers)
 	for v := range r.Followed {
 		users = append(users, v)
 	}
 	r.Unlock()
+	log.I.F("making query chunks")
 	// break into chunks for each query
 	chunks := make([][]st, 0, nQueries)
 	// starting from the nearest integer (from the total divided by the number
@@ -78,14 +85,15 @@ func (r *Relay) spider() {
 		users = users[:i]
 	}
 	relays := make(map[st]struct{})
+	relaysUsed := make(map[st]struct{})
 	usersWithRelays := make(map[st]struct{})
 	for _, v := range chunks {
 		f := &filter.T{Kinds: RelayKinds, Authors: tag.New(v...)}
 		if evs, err = sto.QueryEvents(r.Ctx, f); chk.E(err) {
 			// fatal
 			return
 		}
-		// log.I.F("%d relay events found", len(evs))
+		log.D.F("%d relay events found", len(evs))
 		for _, ev := range evs {
 			relays, usersWithRelays = filterRelays(ev, relays, usersWithRelays)
 		}
@@ -94,8 +102,16 @@ func (r *Relay) spider() {
 		len(relays), len(usersWithRelays))
 	log.W.F("****************** starting spider ******************")
 	// now spider all these relays for the users, and get even moar relays
+	var second bo
+	var found no
 spide:
 	for rely := range relays {
+		if found > 5 {
+			// that's enough for now
+			log.I.S("got events from %d relays queried, "+
+				"finishing spider for today", found)
+			return
+		}
 		select {
 		case <-r.Ctx.Done():
 			var o st
@@ -107,31 +123,66 @@ spide:
 			return
 		default:
 		}
-		rl := ws.NewRelay(r.Ctx, rely)
+		// fetch the relay info
+		var inf *relayinfo.T
+		if inf, err = relayinfo.Fetch(r.Ctx, by(rely)); chk.E(err) {
+			delete(relays, rely)
+			log.I.F("deleted relay %s now %d relays on list",
+				rely, len(relays))
+			continue spide
+		}
+		if !inf.Limitation.AuthRequired {
+			continue spide
+		}
+		log.I.S(inf)
+		rl := ws.NewRelay(r.Ctx, rely, r.SpiderSigner)
 		if err = rl.Connect(r.Ctx); chk.E(err) {
 			// chk.E(rl.Close())
 			continue spide
 		}
 		log.D.F("connected to '%s'", rely)
+		relaysUsed[rely] = struct{}{}
 		// first get some estimate of how many of these events the relay has, if
 		// possible
 		var count no
+		var average time.Duration
+		var looked bo
 		for i, v := range chunks {
-			log.D.F("chunk %d/%d from %s so far: %d relays %d users %d",
-				i, len(chunks), rely, count, len(relays), len(usersWithRelays))
-			f := &filter.T{Kinds: RelayKinds, Authors: tag.New(v...)}
+			log.D.F("chunk %d/%d from %s so far: %d relays %d users %d, av response %v",
+				i, len(chunks), rely, count, len(relays), len(usersWithRelays),
+				average)
+			f := &filter.T{
+				Kinds:   &kinds.T{K: kind.Directory},
+				Authors: tag.New(v...),
+			}
+			started := time.Now()
 			if evs, err = rl.QuerySync(r.Ctx, f); chk.E(err) {
 				chk.E(rl.Close())
 				continue spide
 			}
+			average += time.Now().Sub(started)
+			average /= 2
 			count += len(evs)
+			if !looked && count > 5 {
+				if len(evs) > 0 {
+					looked = true
+					found++
+				}
+			}
 			for _, ev := range evs {
 				relays, usersWithRelays = filterRelays(ev, relays,
 					usersWithRelays)
 				if err = r.Storage().SaveEvent(r.Ctx, ev); chk.E(err) {
 					continue
 				}
 			}
+			if i > 5 {
+				if average > time.Second {
+					log.I.F("relay %s is throttling us, move on", rely)
+					chk.E(rl.Close())
+					continue spide
+				}
+			}
 		}
 		log.I.F("got %d results from %s", count, rely)
 		chk.E(rl.Close())
@@ -142,11 +193,33 @@ spide:
 	// }
 	log.I.F("%d relays found, of %d users",
 		len(relays), len(usersWithRelays))
+	// filter out the relays we used
+	for rely := range relaysUsed {
+		delete(relays, rely)
+	}
+	if !second {
+		log.I.F("%d new relays found, spidering these", len(relays))
+		second = true
+		goto spide
+	} else {
+		// we only will spider the additional ones found
+		o := "relays found:\n"
+		for v := range relaysUsed {
+			o = v + "\n"
+		}
+		log.I.F("%s", o)
+		return
+	}
 }
 
 func filterRelays(ev *event.T,
 	relays, usersWithRelays map[st]struct{}) (r, u map[st]struct{}) {
 	// log.I.S(ev)
+	if !(ev.Kind.Equal(kind.RelayListMetadata) ||
+		ev.Kind.Equal(kind.DMRelaysList)) {
+
+		return relays, usersWithRelays
+	}
 	var foundSome bo
 	t := ev.Tags.GetAll(tag.New("r"))
 next:
@@ -174,18 +247,20 @@ next:
 		// we don't want URLs with query parameters, mostly nostr.wine
 		// these are not interesting. also, if they have @ symbols. or
 		// = in case the user didn't put the ? in properly also.
-		if strings.Contains(v, "\"") {
-			log.E.F("%s", v)
-			panic("wtf")
-			continue
-		}
 		if strings.Contains(v, "?") ||
 			strings.Contains(v, "@") ||
 			strings.Contains(v, "=") {
 			// log.E.F("%s", v)
 			continue
 		}
 
+		// this means some kind of parsing error or format error. it shouldn't
+		// happen, but this check was here because the client code used to have
+		// a concurrency issue with overwriting event bytes.
+		if strings.Contains(v, "\"") {
+			log.E.F("%s", v)
+			continue
+		}
 		// get rid of the slashes
 		if strings.HasSuffix(v, "/") {
 			// trim it off
@@ -202,9 +277,22 @@ next:
 		if len(strings.Split(v, "//")) > 2 {
 			continue
 		}
+		// some relays have subpaths for specific users, so we will just ignore
+		// relay URLs that contain `npub1` and more than 3 `/` characters, which
+		// is a match on wss://filter.nostr.wine/npub1xxxxxx
+		if strings.Contains(v, "/npub1") &&
+			!strings.Contains(v, "//npub1") &&
+			strings.Count(v, "/") > 2 {
+			continue
+		}
 		// finally, because people are dumb and don't know that URLs are
 		// case-insensitive, standardise them
 		v = strings.ToLower(v)
+		// penultimate test, does it validate as a URL at all
+		_, err := url.Parse(v)
+		if chk.E(err) {
+			return relays, usersWithRelays
+		}
 		relays[v] = struct{}{}
 		foundSome = true
 	}

cmd/realy/main.go

@@ -22,6 +22,7 @@ import (
 	"realy.lol/bech32encoding"
 	"realy.lol/hex"
 	"realy.lol/ec/secp256k1"
+	"realy.lol/p256k"
 )
 
 func main() {
@@ -39,19 +40,29 @@ func main() {
 		os.Exit(0)
 	}
 	log.I.Ln("log level", cfg.LogLevel)
-	var prf by
+	var prf, spiderKey by
 	var val any
-	spiderKey := make(by, secp256k1.SecKeyBytesLen)
-	if prf, val, err = bech32encoding.Decode(by(cfg.SpiderKey)); chk.E(err) {
-		log.E.F("SPIDER_KEY decode error: '%s' hrp: %s", err.Error(), prf)
-		spiderKey = nil
-	} else {
-		if sk, ok := val.(by); ok {
-			var n no
-			if n, err = hex.DecBytes(spiderKey, sk); chk.E(err) {
-				log.E.F("failed to decode hex: '%s' at %d",
-					err.Error(), n)
-				spiderKey = nil
+	var sign *p256k.Signer
+	if len(cfg.SpiderKey) > 60 {
+		spiderKey = make(by, secp256k1.SecKeyBytesLen)
+		log.I.F("%s", cfg.SpiderKey)
+		if prf, val, err = bech32encoding.Decode(by(cfg.SpiderKey)); chk.E(err) {
+			log.E.F("SPIDER_KEY decode error: '%s' hrp: %s", err.Error(), prf)
+			spiderKey = nil
+		} else {
+			if sk, ok := val.(by); ok {
+				if spiderKey, err = hex.Dec(st(sk)); chk.E(err) {
+					log.E.F("failed to decode hex: '%s'", err.Error())
+					spiderKey = nil
+				}
+				sign = &p256k.Signer{}
+				if err = sign.InitSec(spiderKey); chk.E(err) {
+					// if this didn't work, disable the spider key
+					cfg.SpiderKey = ""
+					// also nil the signer so it doesn't panic.
+					sign = nil
+				}
+				log.I.F("signer enabled for %x", sign.Pub())
 			}
 		}
 	}
@@ -82,7 +93,13 @@ func main() {
 			},
 		},
 	)
-	r := &app.Relay{Ctx: c, C: cfg, Store: storage, MaxLimit: cfg.MaxLimit}
+	r := &app.Relay{
+		Ctx:          c,
+		C:            cfg,
+		Store:        storage,
+		MaxLimit:     cfg.MaxLimit,
+		SpiderSigner: sign,
+	}
 	go app.MonitorResources(c)
 	var server *realy.Server
 	if server, err = realy.NewServer(realy.ServerParams{
@@ -93,7 +110,6 @@ func main() {
 		MaxLimit:  cfg.MaxLimit,
 		AdminUser: cfg.AdminUser,
 		AdminPass: cfg.AdminPass,
-		SpiderKey: spiderKey,
 	}); chk.E(err) {
 		os.Exit(1)
 	}

ec/bech32/bech32.go

@@ -378,6 +378,14 @@ func ConvertBits(data by, fromBits, toBits uint8, pad bo) (by,
 	return regrouped, nil
 }
 
+func Convert5to8(data by, pad bo) (by, er) {
+	return ConvertBits(data, 5, 8, pad)
+}
+
+func Convert8to5(data by, pad bo) (by, er) {
+	return ConvertBits(data, 8, 5, pad)
+}
+
 // EncodeFromBase256 converts a base256-encoded byte slice into a base32-encoded
 // byte slice and then encodes it into a bech32 string with the given
 // human-readable part (HRP).  The HRP will be converted to lowercase if needed