separate indexes and ratel's index key spec

ratel/countevents.go

@@ -8,10 +8,10 @@ import (
 	"realy.lol/event"
 	"realy.lol/filter"
 	"realy.lol/ratel/keys/createdat"
-	"realy.lol/ratel/keys/index"
 	"realy.lol/ratel/keys/serial"
 	"realy.lol/sha256"
 	"realy.lol/tag"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 func (r *T) CountEvents(c cx, f *filter.T) (count no, approx bo, err er) {
@@ -53,7 +53,7 @@ func (r *T) CountEvents(c cx, f *filter.T) (count no, approx bo, err er) {
 				// todo: matches that are replaceable/parameterized replaceable ones to decode
 				// todo: to check for replacements so we can actually not set the approx flag.
 				ser := serial.FromKey(k)
-				eventKey = index.Event.Key(ser)
+				eventKey = prefixes.Event.Key(ser)
 				// eventKeys = append(eventKeys, idx)
 			}
 			return

ratel/deleteevent.go

@@ -12,14 +12,15 @@ import (
 	"realy.lol/ratel/keys/serial"
 	"realy.lol/ratel/keys/tombstone"
 	"realy.lol/timestamp"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er) {
 	var foundSerial by
 	seri := serial.New(nil)
 	err = r.View(func(txn *badger.Txn) (err er) {
 		// query event by id to ensure we don't try to save duplicates
-		prf := index.Id.Key(id.New(eid))
+		prf := prefixes.Id.Key(id.New(eid))
 		it := txn.NewIterator(badger.IteratorOptions{})
 		defer it.Close()
 		it.Seek(prf)
@@ -46,7 +47,7 @@ func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er) {
 	// fetch the event to get its index keys
 	err = r.View(func(txn *badger.Txn) (err er) {
 		// retrieve the event record
-		evKey = keys.Write(index.New(index.Event), seri)
+		evKey = keys.Write(index.New(prefixes.Event), seri)
 		it := txn.NewIterator(badger.IteratorOptions{})
 		defer it.Close()
 		it.Seek(evKey)
@@ -66,7 +67,7 @@ func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er) {
 			indexKeys = GetIndexKeysForEvent(ev, seri)
 			counterKey = GetCounterKey(seri)
 			ts := tombstone.NewWith(ev.EventID())
-			tombstoneKey = index.Tombstone.Key(ts, createdat.New(timestamp.Now()))
+			tombstoneKey = prefixes.Tombstone.Key(ts, createdat.New(timestamp.Now()))
 			return
 		}
 		return

ratel/export.go

@@ -11,11 +11,11 @@ import (
 	"realy.lol/filter"
 	"realy.lol/hex"
 	"realy.lol/qu"
-	"realy.lol/ratel/keys/index"
 	"realy.lol/ratel/keys/serial"
 	"realy.lol/sha256"
 	"realy.lol/tag"
 	"realy.lol/tags"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 func (r *T) Export(c cx, w io.Writer, pubkeys ...by) {
@@ -124,7 +124,7 @@ func (r *T) Export(c cx, w io.Writer, pubkeys ...by) {
 				for it.Seek(q.start); it.ValidForPrefix(q.searchPrefix); it.Next() {
 					item := it.Item()
 					k := item.KeyCopy(nil)
-					evKey := index.Event.Key(serial.FromKey(k))
+					evKey := prefixes.Event.Key(serial.FromKey(k))
 					counter++
 					keyChan <- evKey
 				}
@@ -140,7 +140,7 @@ func (r *T) Export(c cx, w io.Writer, pubkeys ...by) {
 	} else {
 		// blanket download requested
 		err = r.View(func(txn *badger.Txn) (err er) {
-			it := txn.NewIterator(badger.IteratorOptions{Prefix: index.Event.Key()})
+			it := txn.NewIterator(badger.IteratorOptions{Prefix: prefixes.Event.Key()})
 			defer it.Close()
 			for it.Rewind(); it.Valid(); it.Next() {
 				select {

ratel/gccount.go

@@ -15,6 +15,7 @@ import (
 	"realy.lol/sha256"
 	"realy.lol/timestamp"
 	"realy.lol/units"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 const KeyLen = serial.Len + 1
@@ -32,7 +33,7 @@ func (r *T) GCCount() (unpruned, pruned count.Items, unprunedTotal,
 
 	// log.D.Ln("running GC count", r.Path())
 	overallStart := time.Now()
-	prf := by{byte(index.Event)}
+	prf := prefixes.Event.Key()
 	evStream := r.DB.NewStream()
 	evStream.Prefix = prf
 	var countMx sync.Mutex
@@ -74,7 +75,7 @@ func (r *T) GCCount() (unpruned, pruned count.Items, unprunedTotal,
 	var countFresh count.Freshes
 	// pruneStarted := time.Now()
 	counterStream := r.DB.NewStream()
-	counterStream.Prefix = by{index.Counter.B()}
+	counterStream.Prefix = by{prefixes.Counter.B()}
 	v := make(by, createdat.Len)
 	countFresh = make(count.Freshes, 0, totalCounter)
 	counterStream.ChooseKey = func(item *badger.Item) (b bo) {
@@ -139,7 +140,7 @@ func (r *T) GCCount() (unpruned, pruned count.Items, unprunedTotal,
 	if r.HasL2 {
 		// lastly, we need to count the size of all relevant transactions from the
 		// pruned set
-		for _, fp := range index.FilterPrefixes {
+		for _, fp := range prefixes.FilterPrefixes {
 			// this can all be done concurrently
 			go func(fp by) {
 				evStream = r.DB.NewStream()

ratel/gcsweep.go

@@ -6,9 +6,9 @@ import (
 	"github.com/dgraph-io/badger/v4"
 
 	"realy.lol/event"
-	"realy.lol/ratel/keys/index"
 	"realy.lol/ratel/keys/serial"
 	"realy.lol/sha256"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 func (r *T) GCSweep(evs, idxs DelItems) (err er) {
@@ -33,7 +33,7 @@ func (r *T) GCSweep(evs, idxs DelItems) (err er) {
 	// 	defer wg.Done()
 	stream := r.DB.NewStream()
 	// get all the event indexes to delete/prune
-	stream.Prefix = by{index.Event.B()}
+	stream.Prefix = prefixes.Event.Key()
 	stream.ChooseKey = func(item *badger.Item) (boo bo) {
 		if item.KeySize() != 1+serial.Len {
 			return
@@ -96,9 +96,9 @@ func (r *T) GCSweep(evs, idxs DelItems) (err er) {
 	if len(idxs) > 0 && r.HasL2 {
 		log.I.Ln("pruning indexes")
 		// we have to remove everything
-		prfs := []by{{index.Event.B()}}
-		prfs = append(prfs, index.FilterPrefixes...)
-		prfs = append(prfs, by{index.Counter.B()})
+		prfs := []by{prefixes.Event.Key()}
+		prfs = append(prfs, prefixes.FilterPrefixes...)
+		prfs = append(prfs, by{prefixes.Counter.B()})
 		for _, prf := range prfs {
 			stream = r.DB.NewStream()
 			stream.Prefix = prf

ratel/getecounterkey.go

@@ -1,13 +1,13 @@
 package ratel
 
 import (
-	"realy.lol/ratel/keys/index"
 	"realy.lol/ratel/keys/serial"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 // GetCounterKey returns the proper counter key for a given event ID.
 func GetCounterKey(ser *serial.T) (key by) {
-	key = index.Counter.Key(ser)
+	key = prefixes.Counter.Key(ser)
 	// log.T.F("counter key %d %d", index.Counter, ser.Uint64())
 	return
 }

ratel/getindexkeysforevent.go

@@ -11,6 +11,7 @@ import (
 	"realy.lol/ratel/keys/pubkey"
 	"realy.lol/ratel/keys/serial"
 	"realy.lol/tag"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 // GetIndexKeysForEvent generates all the index keys required to filter for
@@ -26,24 +27,24 @@ func GetIndexKeysForEvent(ev *event.T, ser *serial.T) (keyz []by) {
 	PK, _ := pubkey.New(ev.PubKey)
 	// indexes
 	{ // ~ by id
-		k := index.Id.Key(ID, ser)
+		k := prefixes.Id.Key(ID, ser)
 		// log.T.F("id key: %x %0x %0x", k[0], k[1:9], k[9:])
 		keyz = append(keyz, k)
 	}
 	{ // ~ by pubkey+date
-		k := index.Pubkey.Key(PK, CA, ser)
+		k := prefixes.Pubkey.Key(PK, CA, ser)
 		// log.T.F("pubkey + date key: %x %0x %0x %0x",
 		// 	k[0], k[1:9], k[9:17], k[17:])
 		keyz = append(keyz, k)
 	}
 	{ // ~ by kind+date
-		k := index.Kind.Key(K, CA, ser)
+		k := prefixes.Kind.Key(K, CA, ser)
 		// log.T.F("kind + date key: %x %0x %0x %0x",
 		// 	k[0], k[1:3], k[3:11], k[11:])
 		keyz = append(keyz, k)
 	}
 	{ // ~ by pubkey+kind+date
-		k := index.PubkeyKind.Key(PK, K, CA, ser)
+		k := prefixes.PubkeyKind.Key(PK, K, CA, ser)
 		// log.T.F("pubkey + kind + date key: %x %0x %0x %0x %0x",
 		// 	k[0], k[1:9], k[9:11], k[11:19], k[19:])
 		keyz = append(keyz, k)
@@ -85,7 +86,7 @@ func GetIndexKeysForEvent(ev *event.T, ser *serial.T) (keyz []by) {
 		keyz = append(keyz, k)
 	}
 	{ // ~ by date only
-		k := index.CreatedAt.Key(CA, ser)
+		k := prefixes.CreatedAt.Key(CA, ser)
 		// log.T.F("date key: %x %0x %0x", k[0], k[1:9], k[9:])
 		keyz = append(keyz, k)
 	}

ratel/gettagkeyelements.go

@@ -13,6 +13,7 @@ import (
 	"realy.lol/ratel/keys/kinder"
 	"realy.lol/ratel/keys/pubkey"
 	"realy.lol/ratel/keys/serial"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T,
@@ -30,7 +31,7 @@ func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T,
 			if pkk, err = pubkey.NewFromBytes(pkb); chk.E(err) {
 				return
 			}
-			prf, elems = index.Tag32, keys.Make(pkk, ser)
+			prf, elems = prefixes.Tag32, keys.Make(pkk, ser)
 			return
 		} else {
 			err = nil
@@ -55,15 +56,15 @@ func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T,
 				if pk, err = pubkey.NewFromBytes(pkb); chk.E(err) {
 					return
 				}
-				prf = index.TagAddr
+				prf = prefixes.TagAddr
 				elems = keys.Make(kinder.New(k), pk, arb.NewFromString(d), CA,
 					ser)
 				return
 			}
 		}
 	}
 	// store whatever as utf-8
-	prf = index.Tag
+	prf = prefixes.Tag
 	elems = keys.Make(arb.NewFromString(tagValue), CA, ser)
 	return
 }

ratel/gettagkeyprefix.go

@@ -4,10 +4,10 @@ import (
 	"realy.lol/hex"
 	"realy.lol/ratel/keys"
 	"realy.lol/ratel/keys/arb"
-	"realy.lol/ratel/keys/index"
 	"realy.lol/ratel/keys/kinder"
 	"realy.lol/ratel/keys/pubkey"
 	eventstore "realy.lol/store"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 // GetTagKeyPrefix returns tag index prefixes based on the initial field of a
@@ -33,22 +33,22 @@ func GetTagKeyPrefix(tagValue string) (key by, err er) {
 		if len(d) > 0 {
 			els = append(els, arb.NewFromString(d))
 		}
-		key = index.TagAddr.Key(els...)
+		key = prefixes.TagAddr.Key(els...)
 	} else if pkb, _ := hex.Dec(tagValue); len(pkb) == 32 {
 		// store value as bytes
 		var pkk *pubkey.T
 		if pkk, err = pubkey.NewFromBytes(pkb); chk.E(err) {
 			return
 		}
-		key = index.Tag32.Key(pkk)
+		key = prefixes.Tag32.Key(pkk)
 	} else {
 		// store whatever as utf-8
 		if len(tagValue) > 0 {
 			var a *arb.T
 			a = arb.NewFromString(tagValue)
-			key = index.Tag.Key(a)
+			key = prefixes.Tag.Key(a)
 		} else {
-			key = index.Tag.Key()
+			key = prefixes.Tag.Key()
 		}
 	}
 	return

ratel/init.go

@@ -8,8 +8,8 @@ import (
 	"github.com/dgraph-io/badger/v4"
 	"github.com/dgraph-io/badger/v4/options"
 
-	"realy.lol/ratel/keys/index"
 	"realy.lol/units"
+	"realy.lol/ratel/keys/prefixes"
 )
 
 func (r *T) Init(path st) (err er) {
@@ -56,7 +56,7 @@ func (r *T) runMigrations() (err er) {
 	return r.Update(func(txn *badger.Txn) (err er) {
 		var version uint16
 		var item *badger.Item
-		item, err = txn.Get(by{index.Version.B()})
+		item, err = txn.Get(prefixes.Version.Key())
 		if errors.Is(err, badger.ErrKeyNotFound) {
 			version = 0
 		} else if chk.E(err) {
@@ -71,7 +71,7 @@ func (r *T) runMigrations() (err er) {
 		if version < Version {
 			// if there is any data in the relay we will stop and notify the user, otherwise we
 			// just set version to 1 and proceed
-			prefix := by{index.Id.B()}
+			prefix := prefixes.Id.Key()
 			it := txn.NewIterator(badger.IteratorOptions{
 				PrefetchValues: true,
 				PrefetchSize:   100,
@@ -99,5 +99,5 @@ func (r *T) runMigrations() (err er) {
 func (r *T) bumpVersion(txn *badger.Txn, version uint16) er {
 	buf := make(by, 2)
 	binary.BigEndian.PutUint16(buf, version)
-	return txn.Set(by{index.Version.B()}, buf)
+	return txn.Set(prefixes.Version.Key(), buf)
 }