refactored the storage method, starting on the directory spider

removed owner muting in favour of filtering this in subscriptions

cmd/realy/app/implementation.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 	"time"
 
-	"realy.lol/context"
 	"realy.lol/ec/schnorr"
 	"realy.lol/event"
 	"realy.lol/filter"
@@ -25,22 +24,23 @@ import (
 
 type Relay struct {
 	sync.Mutex
+	Ctx cx
 	*config.C
 	Store store.I
 	// Owners' pubkeys
-	Owners                          []by
-	Followed, OwnersFollowed, Muted map[st]struct{}
-	// OwnersFollowLists are the event IDs of owners follow lists, which must not be deleted,
-	// only replaced.
+	Owners                   []by
+	Followed, OwnersFollowed map[st]struct{}
+	// OwnersFollowLists are the event IDs of owners follow lists, which must
+	// not be deleted, only replaced.
 	OwnersFollowLists []by
-	// OwnersMuteLists are the event IDs of owners mute lists, which must not be deleted, only
-	// replaced.
-	OwnersMuteLists []by
+	// KnownRelays is a map populated by the Spider from all RelayKinds events
+	// found on the relay.
+	KnownRelays map[st]struct{}
 }
 
 func (r *Relay) Name() st { return r.C.AppName }
 
-func (r *Relay) Storage(c cx) store.I { return r.Store }
+func (r *Relay) Storage() store.I { return r.Store }
 
 func (r *Relay) Init() (err er) {
 	for _, src := range r.C.Owners {
@@ -61,16 +61,16 @@ func (r *Relay) Init() (err er) {
 		return fmt.Sprintf("%v", ownerIds)
 	})
 	r.ZeroLists()
-	r.CheckOwnerLists(context.Bg())
+	r.CheckOwnerLists()
+	// start up the spider, if configured
+	r.Spider()
 	return nil
 }
 
 func (r *Relay) ZeroLists() {
 	r.Followed = make(map[st]struct{})
 	r.OwnersFollowed = make(map[st]struct{})
 	r.OwnersFollowLists = r.OwnersFollowLists[:0]
-	r.Muted = make(map[st]struct{})
-	r.OwnersMuteLists = r.OwnersMuteLists[:0]
 }
 
 func (r *Relay) AcceptEvent(c cx, evt *event.T, hr *http.Request, origin st,
@@ -85,7 +85,8 @@ func (r *Relay) AcceptEvent(c cx, evt *event.T, hr *http.Request, origin st,
 			nil
 	}
 	if len(authedPubkey) != 32 {
-		return false, fmt.Sprintf("client not authed with auth required %s", origin), nil
+		return false, fmt.Sprintf("client not authed with auth required %s",
+			origin), nil
 	}
 	if len(r.Owners) > 0 {
 		r.Lock()
@@ -99,7 +100,7 @@ func (r *Relay) AcceptEvent(c cx, evt *event.T, hr *http.Request, origin st,
 				if equals(by(o), evt.PubKey) {
 					return true, "", func() {
 						r.ZeroLists()
-						r.CheckOwnerLists(context.Bg())
+						r.CheckOwnerLists()
 					}
 				}
 			}
@@ -110,7 +111,7 @@ func (r *Relay) AcceptEvent(c cx, evt *event.T, hr *http.Request, origin st,
 				if equals(o, evt.PubKey) {
 					return true, "", func() {
 						r.ZeroLists()
-						r.CheckOwnerLists(context.Bg())
+						r.CheckOwnerLists()
 					}
 				}
 			}
@@ -121,9 +122,10 @@ func (r *Relay) AcceptEvent(c cx, evt *event.T, hr *http.Request, origin st,
 				// prevent owners from deleting their own mute/follow lists in case of bad
 				// client implementation
 				if evt.Kind.Equal(kind.Deletion) {
-					// we don't accept deletes on owners' follow or mute lists because of the
-					// potential for a malicious action causing this, first check for the list:
-					tt := tag.New(append(r.OwnersFollowLists, r.OwnersMuteLists...)...)
+					// we don't accept deletes on owners' follow or mute lists
+					// because of the potential for a malicious action causing
+					// this, first check for the list:
+					tt := tag.New(r.OwnersFollowLists...)
 					if evt.Tags.ContainsAny(by("e"), tt) {
 						return false,
 							"cannot delete owner's follow, owners' follows follow or mute events",
@@ -167,14 +169,6 @@ func (r *Relay) AcceptEvent(c cx, evt *event.T, hr *http.Request, origin st,
 					return true, "", nil
 				}
 			}
-			// check the mute list, and reject events authored by muted pubkeys, even if
-			// they come from a pubkey that is on the follow list.
-			for pk := range r.Muted {
-				if equals(evt.PubKey, by(pk)) {
-					return false, "rejecting event with pubkey " + hex.Enc(evt.PubKey) +
-						" because on owner mute list", nil
-				}
-			}
 			// for all else, check the authed pubkey is in the follow list
 			for pk := range r.Followed {
 				// allow all events from follows of owners
@@ -262,10 +256,10 @@ func (r *Relay) AcceptReq(c cx, hr *http.Request, id by, ff *filters.T,
 	return
 }
 
-// CheckOwnerLists regenerates the owner follow and mute lists if they are empty.
+// CheckOwnerLists regenerates the owner follow lists if they are empty.
 //
 // It also adds the followed npubs of the follows.
-func (r *Relay) CheckOwnerLists(c cx) {
+func (r *Relay) CheckOwnerLists() {
 	if len(r.Owners) > 0 {
 		r.Lock()
 		defer r.Unlock()
@@ -278,7 +272,7 @@ func (r *Relay) CheckOwnerLists(c cx) {
 				r.Followed[st(r.Owners[i])] = struct{}{}
 			}
 			log.D.Ln("regenerating owners follow lists")
-			if evs, err = r.Store.QueryEvents(c,
+			if evs, err = r.Store.QueryEvents(r.Ctx,
 				&filter.T{Authors: tag.New(r.Owners...),
 					Kinds: kinds.New(kind.FollowList)}); chk.E(err) {
 			}
@@ -302,13 +296,13 @@ func (r *Relay) CheckOwnerLists(c cx) {
 			for f := range r.Followed {
 				followed = append(followed, f)
 			}
-			if evs, err = r.Store.QueryEvents(c,
+			if evs, err = r.Store.QueryEvents(r.Ctx,
 				&filter.T{Authors: tag.New(followed...),
 					Kinds: kinds.New(kind.FollowList)}); chk.E(err) {
 			}
 			for _, ev := range evs {
-				// we want to protect the follow lists of users as well so they also cannot be
-				// deleted, only replaced.
+				// we want to protect the follow lists of users as well, they
+				// also cannot be deleted, only replaced.
 				r.OwnersFollowLists = append(r.OwnersFollowLists, ev.ID)
 				for _, t := range ev.Tags.F() {
 					if equals(t.Key(), by("p")) {
@@ -322,37 +316,14 @@ func (r *Relay) CheckOwnerLists(c cx) {
 			}
 			evs = evs[:0]
 		}
-		if len(r.Muted) < 1 {
-			log.D.Ln("regenerating owners mute lists")
-			r.Muted = make(map[st]struct{})
-			if evs, err = r.Store.QueryEvents(c,
-				&filter.T{Authors: tag.New(r.Owners...),
-					Kinds: kinds.New(kind.MuteList)}); chk.E(err) {
-			}
-			for _, ev := range evs {
-				r.OwnersMuteLists = append(r.OwnersMuteLists, ev.ID)
-				for _, t := range ev.Tags.F() {
-					if equals(t.Key(), by("p")) {
-						var p by
-						if p, err = hex.Dec(st(t.Value())); chk.E(err) {
-							continue
-						}
-						r.Muted[st(p)] = struct{}{}
-					}
-				}
-			}
-			evs = evs[:0]
-		}
-		log.I.F("%d allowed npubs, %d blocked", len(r.Followed), len(r.Muted))
-		// // log this info
+		// log this info
+		log.I.F("%d allowed npubs", len(r.Followed))
+		// r.Followed
+		// r.OwnersFollowed
 		// o := "followed:\n"
 		// for pk := range r.Followed {
 		// 	o += fmt.Sprintf("%0x,", pk)
 		// }
-		// o += "\nmuted:\n"
-		// for pk := range r.Muted {
-		// 	o += fmt.Sprintf("%0x,", pk)
-		// }
 		// log.T.F("%s\n", o)
 	}
 }

cmd/realy/app/spider.go

@@ -0,0 +1,49 @@
+package app
+
+import (
+	"time"
+	"realy.lol/kinds"
+	"realy.lol/kind"
+	"realy.lol/filter"
+)
+
+func (r *Relay) Spider() {
+	// Don't start the spider if the spider key is not configured, many relays
+	// require auth, whether the spider key is allowed, those that do,
+	// and are, it must be there and this wraps the toggle together with the
+	// configuration neatly.
+	if len(r.C.SpiderKey) == 0 {
+		return
+	}
+	// we run at first startup
+	r.spider()
+	// re-run the spider every hour to catch any updates that for whatever
+	// reason permitted users may have uploaded to other relays via other
+	// clients that may not be sending to us.
+	ticker := time.NewTicker(time.Hour)
+	for {
+		select {
+		case <-r.Ctx.Done():
+			return
+		case <-ticker.C:
+			r.spider()
+		}
+	}
+}
+
+// RelayKinds are the types of events that we want to search and fetch.
+var RelayKinds = &kinds.T{
+	K: []*kind.T{
+		kind.RelayListMetadata,
+		kind.DMRelaysList,
+	},
+}
+
+// spider is the actual function that does a spider run
+func (r *Relay) spider() {
+	// first find all the relays that we currently know about.
+	filter := filter.T{Kinds: RelayKinds}
+	sto := r.Storage()
+	_ = filter
+	_ = sto
+}

cmd/realy/main.go

@@ -63,7 +63,7 @@ func main() {
 			},
 		},
 	)
-	r := &app.Relay{C: cfg, Store: storage}
+	r := &app.Relay{Ctx: c, C: cfg, Store: storage}
 	go app.MonitorResources(c)
 	var server *realy.Server
 	if server, err = realy.NewServer(realy.ServerParams{
@@ -73,7 +73,9 @@ func main() {
 		DbPath:    cfg.Profile,
 		MaxLimit:  ratel.DefaultMaxLimit,
 		AdminUser: cfg.AdminUser,
-		AdminPass: cfg.AdminPass}); chk.E(err) {
+		AdminPass: cfg.AdminPass,
+		SpiderKey: cfg.SpiderKey,
+	}); chk.E(err) {
 
 		os.Exit(1)
 	}

kind/kind.go

@@ -230,7 +230,8 @@ var (
 	BlockList = &T{10000}
 	// PinList is an event type that...
 	PinList = &T{10001}
-	// RelayListMetadata is an event type that...
+	// RelayListMetadata designates the relays a user uses, with optional
+	// additional read/write (nip-65)
 	RelayListMetadata     = &T{10002}
 	BookmarkList          = &T{10003}
 	CommunitiesList       = &T{10004}
@@ -259,6 +260,15 @@ var (
 	WalletResponse     = NWCWalletResponse
 	NWCNotification    = &T{23196}
 	WalletNotification = NWCNotification
+	// NRCMessage is a message type for an as yet unnumbered NIP for
+	// simple, public IRC style messages.
+	NRCMessage = &T{23514}
+	// NRCStatus is a message type that is used to signal that a user is online
+	// and available to receive messages using NRC. A periodic "online" message
+	// that also enables a dead-man switch propagation to other users, and an
+	// "offline" message that indicates a user is disconnecting from the NRC of
+	// a relay.
+	NRCStatus = &T{K: 23515}
 	// NostrConnect is an event type that...
 	NostrConnect = &T{24133}
 	HTTPAuth     = &T{27235}

ratel/keys/index/prefixes.go

@@ -11,7 +11,6 @@ type P byte
 func (p P) Key(element ...keys.Element) (b by) {
 	b = keys.Write(
 		append([]keys.Element{New(byte(p))}, element...)...)
-	log.T.F("key %x", b)
 	return
 }
 

realy/addEvent.go

@@ -14,12 +14,13 @@ import (
 	"realy.lol/store"
 )
 
-func (s *Server) addEvent(c cx, rl relay.I, ev *event.T, hr *http.Request, origin st,
+func (s *Server) addEvent(c cx, rl relay.I, ev *event.T, hr *http.Request,
+	origin st,
 	authedPubkey by) (accepted bo, message by) {
 	if ev == nil {
 		return false, normalize.Invalid.F("empty event")
 	}
-	sto := rl.Storage(c)
+	sto := rl.Storage()
 	wrap := &wrapper.Relay{I: sto}
 	advancedSaver, _ := sto.(relay.AdvancedSaver)
 	accept, notice, after := rl.AcceptEvent(c, ev, hr, origin, authedPubkey)

realy/config/config.go

@@ -28,7 +28,8 @@ type C struct {
 	LogLevel     st   `env:"LOG_LEVEL" default:"info" usage:"debug level: fatal error warn info debug trace"`
 	DbLogLevel   st   `env:"DB_LOG_LEVEL" default:"info" usage:"debug level: fatal error warn info debug trace"`
 	AuthRequired bo   `env:"AUTH_REQUIRED" default:"false" usage:"requires auth for all access"`
-	Owners       []st `env:"OWNERS" usage:"list of npubs of users in hex format whose follow and mute list dictate accepting requests and events with AUTH_REQUIRED enabled - follows and follows follows are allowed to read/write, owners mutes events are rejected"`
+	Owners       []st `env:"OWNERS" usage:"list of npubs of users in hex format whose follow and mute list dictate accepting requests and events with AUTH_REQUIRED enabled - follows and follows follows are allowed to read/write"`
+	SpiderKey    st   `env:"SPIDER_KEY" usage:"secret key that is used for directory spidering to auth-required relays for whitelisted users of the relay, implicitly enables directory spider"`
 	DBSizeLimit  no   `env:"DB_SIZE_LIMIT" default:"0" usage:"the number of gigabytes (1,000,000,000 bytes) we want to keep the data store from exceeding, 0 means disabled"`
 	DBLowWater   no   `env:"DB_LOW_WATER" default:"60" usage:"the percentage of DBSizeLimit a GC run will reduce the used storage down to"`
 	DBHighWater  no   `env:"DB_HIGH_WATER" default:"80" usage:"the trigger point at which a GC run should start if exceeded"`
@@ -173,7 +174,8 @@ func PrintHelp(cfg *C, printer io.Writer) {
 			" this file will be created on first startup.\nenvironment overrides it and "+
 			"you can also edit the file to set configuration options\n\n"+
 			"use the parameter 'env' to print out the current configuration to the terminal\n\n"+
-			"set the environment using\n\n\t%s env>%s/%s/.env\n\n", os.Args[0], cfg.Profile,
+			"set the environment using\n\n\t%s env>%s/%s/.env\n\n", os.Args[0],
+		cfg.Profile,
 		cfg.Profile)
 	return
 }

realy/handleAdmin.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"realy.lol/cmd/realy/app"
-	"realy.lol/context"
 	"realy.lol/hex"
 	"realy.lol/sha256"
 )
@@ -36,7 +35,8 @@ func (s *Server) auth(r *http.Request) (authed bo) {
 }
 
 func (s *Server) unauthorized(w http.ResponseWriter) {
-	w.Header().Set("WWW-Authenticate", `Basic realm="restricted", charset="UTF-8"`)
+	w.Header().Set("WWW-Authenticate",
+		`Basic realm="restricted", charset="UTF-8"`)
 	http.Error(w, "Unauthorized", http.StatusUnauthorized)
 	fmt.Fprintf(w, "you may have not configured your admin username/password")
 }
@@ -49,11 +49,12 @@ func (s *Server) handleAdmin(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		log.I.F("export of event data requested on admin port")
-		sto := s.relay.Storage(context.Bg())
+		sto := s.relay.Storage()
 		if strings.Count(r.URL.Path, "/") > 1 {
 			split := strings.Split(r.URL.Path, "/")
 			if len(split) != 3 {
-				fprintf(w, "incorrectly formatted export parameter: '%s'", r.URL.Path)
+				fprintf(w, "incorrectly formatted export parameter: '%s'",
+					r.URL.Path)
 				return
 			}
 			switch split[2] {
@@ -87,12 +88,12 @@ func (s *Server) handleAdmin(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		log.I.F("import of event data requested on admin port %s", r.RequestURI)
-		sto := s.relay.Storage(context.Bg())
+		sto := s.relay.Storage()
 		read := io.LimitReader(r.Body, r.ContentLength)
 		sto.Import(read)
 		if realy, ok := s.relay.(*app.Relay); ok {
 			realy.ZeroLists()
-			realy.CheckOwnerLists(context.Bg())
+			realy.CheckOwnerLists()
 		}
 	case strings.HasPrefix(r.URL.Path, "/shutdown"):
 		if ok := s.auth(r); !ok {