diff --git a/ratel/countevents.go b/ratel/countevents.go index e93c00e..451a044 100644 --- a/ratel/countevents.go +++ b/ratel/countevents.go @@ -8,10 +8,10 @@ import ( "realy.lol/event" "realy.lol/filter" "realy.lol/ratel/keys/createdat" - "realy.lol/ratel/keys/index" "realy.lol/ratel/keys/serial" "realy.lol/sha256" "realy.lol/tag" + "realy.lol/ratel/keys/prefixes" ) func (r *T) CountEvents(c cx, f *filter.T) (count no, approx bo, err er) { @@ -53,7 +53,7 @@ func (r *T) CountEvents(c cx, f *filter.T) (count no, approx bo, err er) { // todo: matches that are replaceable/parameterized replaceable ones to decode // todo: to check for replacements so we can actually not set the approx flag. ser := serial.FromKey(k) - eventKey = index.Event.Key(ser) + eventKey = prefixes.Event.Key(ser) // eventKeys = append(eventKeys, idx) } return diff --git a/ratel/deleteevent.go b/ratel/deleteevent.go index b9a0ace..302d7ca 100644 --- a/ratel/deleteevent.go +++ b/ratel/deleteevent.go @@ -12,6 +12,7 @@ import ( "realy.lol/ratel/keys/serial" "realy.lol/ratel/keys/tombstone" "realy.lol/timestamp" + "realy.lol/ratel/keys/prefixes" ) func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er) { @@ -19,7 +20,7 @@ func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er) { seri := serial.New(nil) err = r.View(func(txn *badger.Txn) (err er) { // query event by id to ensure we don't try to save duplicates - prf := index.Id.Key(id.New(eid)) + prf := prefixes.Id.Key(id.New(eid)) it := txn.NewIterator(badger.IteratorOptions{}) defer it.Close() it.Seek(prf) @@ -46,7 +47,7 @@ func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er) { // fetch the event to get its index keys err = r.View(func(txn *badger.Txn) (err er) { // retrieve the event record - evKey = keys.Write(index.New(index.Event), seri) + evKey = keys.Write(index.New(prefixes.Event), seri) it := txn.NewIterator(badger.IteratorOptions{}) defer it.Close() it.Seek(evKey) @@ -66,7 +67,7 @@ func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er) { indexKeys = GetIndexKeysForEvent(ev, seri) counterKey = GetCounterKey(seri) ts := tombstone.NewWith(ev.EventID()) - tombstoneKey = index.Tombstone.Key(ts, createdat.New(timestamp.Now())) + tombstoneKey = prefixes.Tombstone.Key(ts, createdat.New(timestamp.Now())) return } return diff --git a/ratel/export.go b/ratel/export.go index 8c3e982..86b771f 100644 --- a/ratel/export.go +++ b/ratel/export.go @@ -11,11 +11,11 @@ import ( "realy.lol/filter" "realy.lol/hex" "realy.lol/qu" - "realy.lol/ratel/keys/index" "realy.lol/ratel/keys/serial" "realy.lol/sha256" "realy.lol/tag" "realy.lol/tags" + "realy.lol/ratel/keys/prefixes" ) func (r *T) Export(c cx, w io.Writer, pubkeys ...by) { @@ -124,7 +124,7 @@ func (r *T) Export(c cx, w io.Writer, pubkeys ...by) { for it.Seek(q.start); it.ValidForPrefix(q.searchPrefix); it.Next() { item := it.Item() k := item.KeyCopy(nil) - evKey := index.Event.Key(serial.FromKey(k)) + evKey := prefixes.Event.Key(serial.FromKey(k)) counter++ keyChan <- evKey } @@ -140,7 +140,7 @@ func (r *T) Export(c cx, w io.Writer, pubkeys ...by) { } else { // blanket download requested err = r.View(func(txn *badger.Txn) (err er) { - it := txn.NewIterator(badger.IteratorOptions{Prefix: index.Event.Key()}) + it := txn.NewIterator(badger.IteratorOptions{Prefix: prefixes.Event.Key()}) defer it.Close() for it.Rewind(); it.Valid(); it.Next() { select { diff --git a/ratel/gccount.go b/ratel/gccount.go index 2592d3e..2934c27 100644 --- a/ratel/gccount.go +++ b/ratel/gccount.go @@ -15,6 +15,7 @@ import ( "realy.lol/sha256" "realy.lol/timestamp" "realy.lol/units" + "realy.lol/ratel/keys/prefixes" ) const KeyLen = serial.Len + 1 @@ -32,7 +33,7 @@ func (r *T) GCCount() (unpruned, pruned count.Items, unprunedTotal, // log.D.Ln("running GC count", r.Path()) overallStart := time.Now() - prf := by{byte(index.Event)} + prf := prefixes.Event.Key() evStream := r.DB.NewStream() evStream.Prefix = prf var countMx sync.Mutex @@ -74,7 +75,7 @@ func (r *T) GCCount() (unpruned, pruned count.Items, unprunedTotal, var countFresh count.Freshes // pruneStarted := time.Now() counterStream := r.DB.NewStream() - counterStream.Prefix = by{index.Counter.B()} + counterStream.Prefix = by{prefixes.Counter.B()} v := make(by, createdat.Len) countFresh = make(count.Freshes, 0, totalCounter) counterStream.ChooseKey = func(item *badger.Item) (b bo) { @@ -139,7 +140,7 @@ func (r *T) GCCount() (unpruned, pruned count.Items, unprunedTotal, if r.HasL2 { // lastly, we need to count the size of all relevant transactions from the // pruned set - for _, fp := range index.FilterPrefixes { + for _, fp := range prefixes.FilterPrefixes { // this can all be done concurrently go func(fp by) { evStream = r.DB.NewStream() diff --git a/ratel/gcsweep.go b/ratel/gcsweep.go index 9847d42..b7bf824 100644 --- a/ratel/gcsweep.go +++ b/ratel/gcsweep.go @@ -6,9 +6,9 @@ import ( "github.com/dgraph-io/badger/v4" "realy.lol/event" - "realy.lol/ratel/keys/index" "realy.lol/ratel/keys/serial" "realy.lol/sha256" + "realy.lol/ratel/keys/prefixes" ) func (r *T) GCSweep(evs, idxs DelItems) (err er) { @@ -33,7 +33,7 @@ func (r *T) GCSweep(evs, idxs DelItems) (err er) { // defer wg.Done() stream := r.DB.NewStream() // get all the event indexes to delete/prune - stream.Prefix = by{index.Event.B()} + stream.Prefix = prefixes.Event.Key() stream.ChooseKey = func(item *badger.Item) (boo bo) { if item.KeySize() != 1+serial.Len { return @@ -96,9 +96,9 @@ func (r *T) GCSweep(evs, idxs DelItems) (err er) { if len(idxs) > 0 && r.HasL2 { log.I.Ln("pruning indexes") // we have to remove everything - prfs := []by{{index.Event.B()}} - prfs = append(prfs, index.FilterPrefixes...) - prfs = append(prfs, by{index.Counter.B()}) + prfs := []by{prefixes.Event.Key()} + prfs = append(prfs, prefixes.FilterPrefixes...) + prfs = append(prfs, by{prefixes.Counter.B()}) for _, prf := range prfs { stream = r.DB.NewStream() stream.Prefix = prf diff --git a/ratel/getecounterkey.go b/ratel/getecounterkey.go index 6d18f79..cb18aa0 100644 --- a/ratel/getecounterkey.go +++ b/ratel/getecounterkey.go @@ -1,13 +1,13 @@ package ratel import ( - "realy.lol/ratel/keys/index" "realy.lol/ratel/keys/serial" + "realy.lol/ratel/keys/prefixes" ) // GetCounterKey returns the proper counter key for a given event ID. func GetCounterKey(ser *serial.T) (key by) { - key = index.Counter.Key(ser) + key = prefixes.Counter.Key(ser) // log.T.F("counter key %d %d", index.Counter, ser.Uint64()) return } diff --git a/ratel/getindexkeysforevent.go b/ratel/getindexkeysforevent.go index cc1fc91..b9d326a 100644 --- a/ratel/getindexkeysforevent.go +++ b/ratel/getindexkeysforevent.go @@ -11,6 +11,7 @@ import ( "realy.lol/ratel/keys/pubkey" "realy.lol/ratel/keys/serial" "realy.lol/tag" + "realy.lol/ratel/keys/prefixes" ) // GetIndexKeysForEvent generates all the index keys required to filter for @@ -26,24 +27,24 @@ func GetIndexKeysForEvent(ev *event.T, ser *serial.T) (keyz []by) { PK, _ := pubkey.New(ev.PubKey) // indexes { // ~ by id - k := index.Id.Key(ID, ser) + k := prefixes.Id.Key(ID, ser) // log.T.F("id key: %x %0x %0x", k[0], k[1:9], k[9:]) keyz = append(keyz, k) } { // ~ by pubkey+date - k := index.Pubkey.Key(PK, CA, ser) + k := prefixes.Pubkey.Key(PK, CA, ser) // log.T.F("pubkey + date key: %x %0x %0x %0x", // k[0], k[1:9], k[9:17], k[17:]) keyz = append(keyz, k) } { // ~ by kind+date - k := index.Kind.Key(K, CA, ser) + k := prefixes.Kind.Key(K, CA, ser) // log.T.F("kind + date key: %x %0x %0x %0x", // k[0], k[1:3], k[3:11], k[11:]) keyz = append(keyz, k) } { // ~ by pubkey+kind+date - k := index.PubkeyKind.Key(PK, K, CA, ser) + k := prefixes.PubkeyKind.Key(PK, K, CA, ser) // log.T.F("pubkey + kind + date key: %x %0x %0x %0x %0x", // k[0], k[1:9], k[9:11], k[11:19], k[19:]) keyz = append(keyz, k) @@ -85,7 +86,7 @@ func GetIndexKeysForEvent(ev *event.T, ser *serial.T) (keyz []by) { keyz = append(keyz, k) } { // ~ by date only - k := index.CreatedAt.Key(CA, ser) + k := prefixes.CreatedAt.Key(CA, ser) // log.T.F("date key: %x %0x %0x", k[0], k[1:9], k[9:]) keyz = append(keyz, k) } diff --git a/ratel/gettagkeyelements.go b/ratel/gettagkeyelements.go index b75dbb7..e37ce65 100644 --- a/ratel/gettagkeyelements.go +++ b/ratel/gettagkeyelements.go @@ -13,6 +13,7 @@ import ( "realy.lol/ratel/keys/kinder" "realy.lol/ratel/keys/pubkey" "realy.lol/ratel/keys/serial" + "realy.lol/ratel/keys/prefixes" ) func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T, @@ -30,7 +31,7 @@ func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T, if pkk, err = pubkey.NewFromBytes(pkb); chk.E(err) { return } - prf, elems = index.Tag32, keys.Make(pkk, ser) + prf, elems = prefixes.Tag32, keys.Make(pkk, ser) return } else { err = nil @@ -55,7 +56,7 @@ func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T, if pk, err = pubkey.NewFromBytes(pkb); chk.E(err) { return } - prf = index.TagAddr + prf = prefixes.TagAddr elems = keys.Make(kinder.New(k), pk, arb.NewFromString(d), CA, ser) return @@ -63,7 +64,7 @@ func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T, } } // store whatever as utf-8 - prf = index.Tag + prf = prefixes.Tag elems = keys.Make(arb.NewFromString(tagValue), CA, ser) return } diff --git a/ratel/gettagkeyprefix.go b/ratel/gettagkeyprefix.go index 9bf2b35..0555424 100644 --- a/ratel/gettagkeyprefix.go +++ b/ratel/gettagkeyprefix.go @@ -4,10 +4,10 @@ import ( "realy.lol/hex" "realy.lol/ratel/keys" "realy.lol/ratel/keys/arb" - "realy.lol/ratel/keys/index" "realy.lol/ratel/keys/kinder" "realy.lol/ratel/keys/pubkey" eventstore "realy.lol/store" + "realy.lol/ratel/keys/prefixes" ) // GetTagKeyPrefix returns tag index prefixes based on the initial field of a @@ -33,22 +33,22 @@ func GetTagKeyPrefix(tagValue string) (key by, err er) { if len(d) > 0 { els = append(els, arb.NewFromString(d)) } - key = index.TagAddr.Key(els...) + key = prefixes.TagAddr.Key(els...) } else if pkb, _ := hex.Dec(tagValue); len(pkb) == 32 { // store value as bytes var pkk *pubkey.T if pkk, err = pubkey.NewFromBytes(pkb); chk.E(err) { return } - key = index.Tag32.Key(pkk) + key = prefixes.Tag32.Key(pkk) } else { // store whatever as utf-8 if len(tagValue) > 0 { var a *arb.T a = arb.NewFromString(tagValue) - key = index.Tag.Key(a) + key = prefixes.Tag.Key(a) } else { - key = index.Tag.Key() + key = prefixes.Tag.Key() } } return diff --git a/ratel/init.go b/ratel/init.go index 9c8bdfd..e4f2841 100644 --- a/ratel/init.go +++ b/ratel/init.go @@ -8,8 +8,8 @@ import ( "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/badger/v4/options" - "realy.lol/ratel/keys/index" "realy.lol/units" + "realy.lol/ratel/keys/prefixes" ) func (r *T) Init(path st) (err er) { @@ -56,7 +56,7 @@ func (r *T) runMigrations() (err er) { return r.Update(func(txn *badger.Txn) (err er) { var version uint16 var item *badger.Item - item, err = txn.Get(by{index.Version.B()}) + item, err = txn.Get(prefixes.Version.Key()) if errors.Is(err, badger.ErrKeyNotFound) { version = 0 } else if chk.E(err) { @@ -71,7 +71,7 @@ func (r *T) runMigrations() (err er) { if version < Version { // if there is any data in the relay we will stop and notify the user, otherwise we // just set version to 1 and proceed - prefix := by{index.Id.B()} + prefix := prefixes.Id.Key() it := txn.NewIterator(badger.IteratorOptions{ PrefetchValues: true, PrefetchSize: 100, @@ -99,5 +99,5 @@ func (r *T) runMigrations() (err er) { func (r *T) bumpVersion(txn *badger.Txn, version uint16) er { buf := make(by, 2) binary.BigEndian.PutUint16(buf, version) - return txn.Set(by{index.Version.B()}, buf) + return txn.Set(prefixes.Version.Key(), buf) } diff --git a/ratel/keys/index/prefixes.go b/ratel/keys/index/prefixes.go index bf8e529..3e987eb 100644 --- a/ratel/keys/index/prefixes.go +++ b/ratel/keys/index/prefixes.go @@ -2,11 +2,6 @@ package index import ( "realy.lol/ratel/keys" - "realy.lol/ratel/keys/createdat" - "realy.lol/ratel/keys/id" - "realy.lol/ratel/keys/kinder" - "realy.lol/ratel/keys/pubkey" - "realy.lol/ratel/keys/serial" ) type P byte @@ -34,117 +29,3 @@ func GetAsBytes(prf ...P) (b []by) { } return } - -const ( - // Version is the key that stores the version number, the value is a 16-bit - // integer (2 bytes) - // - // [ 255 ][ 2 byte/16 bit version code ] - Version P = 255 -) -const ( - // Event is the prefix used with a Serial counter value provided by badgerDB to - // provide conflict-free 8 byte 64-bit unique keys for event records, which - // follows the prefix. - // - // [ 0 ][ 8 bytes Serial ] - Event P = iota - - // CreatedAt creates an index key that contains the unix - // timestamp of the event record serial. - // - // [ 1 ][ 8 bytes timestamp.T ][ 8 bytes Serial ] - CreatedAt - - // Id contains the first 8 bytes of the ID of the event and the 8 - // byte Serial of the event record. - // - // [ 2 ][ 8 bytes eventid.T prefix ][ 8 bytes Serial ] - Id - - // Kind contains the kind and datestamp. - // - // [ 3 ][ 2 bytes kind.T ][ 8 bytes timestamp.T ][ 8 bytes Serial ] - Kind - - // Pubkey contains pubkey prefix and timestamp. - // - // [ 4 ][ 8 bytes pubkey prefix ][ 8 bytes timestamp.T ][ 8 bytes Serial ] - Pubkey - - // PubkeyKind contains pubkey prefix, kind and timestamp. - // - // [ 5 ][ 8 bytes pubkey prefix ][ 2 bytes kind.T ][ 8 bytes timestamp.T ][ 8 bytes Serial ] - PubkeyKind - - // Tag is for miscellaneous arbitrary length tags, with timestamp and event - // serial after. - // - // [ 6 ][ tag string 1 <= 100 bytes ][ 8 bytes timestamp.T ][ 8 bytes Serial ] - Tag - - // Tag32 contains the 8 byte pubkey prefix, timestamp and serial. - // - // [ 7 ][ 8 bytes pubkey prefix ][ 8 bytes timestamp.T ][ 8 bytes Serial ] - Tag32 - - // TagAddr contains the kind, pubkey prefix, value (index 2) of address tag (eg - // relay address), followed by timestamp and serial. - // - // [ 8 ][ 2 byte kind.T][ 8 byte pubkey prefix ][ network address ][ 8 byte timestamp.T ][ 8 byte Serial ] - TagAddr - - // Counter is the eventid.T prefix, value stores the average time of access - // (average of all access timestamps) and the size of the record. - // - // [ 9 ][ 8 bytes Serial ] : value: [ 8 bytes timestamp ] - Counter - - // Tombstone is an index that contains the left half of an event ID that has - // been deleted. The purpose of this event is to stop the event being - // republished, as a delete event may not be respected by other relays and - // eventually lead to a republication. The timestamp is added at the end to - // enable pruning the oldest tombstones. - // - // [ 10 ][ 16 bytes first/left half of event ID ][ 8 bytes timestamp ] - Tombstone -) - -// FilterPrefixes is a slice of the prefixes used by filter index to enable a loop -// for pulling events matching a serial -var FilterPrefixes = []by{ - {CreatedAt.B()}, - {Id.B()}, - {Kind.B()}, - {Pubkey.B()}, - {PubkeyKind.B()}, - {Tag.B()}, - {Tag32.B()}, - {TagAddr.B()}, -} - -// KeySizes are the byte size of keys of each type of key prefix. int(P) or call the P.I() method -// corresponds to the index 1:1. For future index additions be sure to add the -// relevant KeySizes sum as it describes the data for a programmer. -var KeySizes = []no{ - // Event - 1 + serial.Len, - // CreatedAt - 1 + createdat.Len + serial.Len, - // Id - 1 + id.Len + serial.Len, - // Kind - 1 + kinder.Len + createdat.Len + serial.Len, - // Pubkey - 1 + pubkey.Len + createdat.Len + serial.Len, - // PubkeyKind - 1 + pubkey.Len + kinder.Len + createdat.Len + serial.Len, - // Tag (worst case scenario) - 1 + 100 + createdat.Len + serial.Len, - // Tag32 - 1 + pubkey.Len + createdat.Len + serial.Len, - // TagAddr - 1 + kinder.Len + pubkey.Len + 100 + createdat.Len + serial.Len, - // Counter - 1 + serial.Len, -} diff --git a/ratel/keys/keys_test.go b/ratel/keys/keys_test.go index c759ded..adcc5a5 100644 --- a/ratel/keys/keys_test.go +++ b/ratel/keys/keys_test.go @@ -20,6 +20,7 @@ import ( "realy.lol/ratel/keys/pubkey" "realy.lol/ratel/keys/serial" "realy.lol/timestamp" + "realy.lol/ratel/keys/prefixes" ) func TestElement(t *testing.T) { @@ -27,7 +28,7 @@ func TestElement(t *testing.T) { var failed bo { // construct a typical key type of structure // a prefix - np := index.Version + np := prefixes.Version vp := index.New(byte(np)) // an id fakeIdBytes := frand.Bytes(sha256.Size) diff --git a/ratel/keys/index/index_test.go b/ratel/keys/prefixes/index_test.go similarity index 71% rename from ratel/keys/index/index_test.go rename to ratel/keys/prefixes/index_test.go index 5f95f62..2b9941b 100644 --- a/ratel/keys/index/index_test.go +++ b/ratel/keys/prefixes/index_test.go @@ -1,8 +1,9 @@ -package index +package prefixes import ( "bytes" "testing" + "realy.lol/ratel/keys/index" ) func TestT(t *testing.T) { @@ -11,8 +12,8 @@ func TestT(t *testing.T) { // buf := new(bytes.Buffer) // v.Write(buf) buf2 := bytes.NewBuffer(v) - v2 := New(0) - el := v2.Read(buf2).(*T) + v2 := index.New(0) + el := v2.Read(buf2).(*index.T) if el.Val[0] != v[0] { t.Fatalf("expected %d got %d", v[0], el.Val) } diff --git a/ratel/keys/prefixes/prefixes.go b/ratel/keys/prefixes/prefixes.go new file mode 100644 index 0000000..2738d2b --- /dev/null +++ b/ratel/keys/prefixes/prefixes.go @@ -0,0 +1,123 @@ +package prefixes + +import ( + "realy.lol/ratel/keys/index" + "realy.lol/ratel/keys/serial" + "realy.lol/ratel/keys/createdat" + "realy.lol/ratel/keys/id" + "realy.lol/ratel/keys/kinder" + "realy.lol/ratel/keys/pubkey" +) + +const ( + // Version is the key that stores the version number, the value is a 16-bit + // integer (2 bytes) + // + // [ 255 ][ 2 byte/16 bit version code ] + Version index.P = 255 + + // Event is the prefix used with a Serial counter value provided by badgerDB to + // provide conflict-free 8 byte 64-bit unique keys for event records, which + // follows the prefix. + // + // [ 0 ][ 8 bytes Serial ] + Event index.P = iota + + // CreatedAt creates an index key that contains the unix + // timestamp of the event record serial. + // + // [ 1 ][ 8 bytes timestamp.T ][ 8 bytes Serial ] + CreatedAt + + // Id contains the first 8 bytes of the ID of the event and the 8 + // byte Serial of the event record. + // + // [ 2 ][ 8 bytes eventid.T prefix ][ 8 bytes Serial ] + Id + + // Kind contains the kind and datestamp. + // + // [ 3 ][ 2 bytes kind.T ][ 8 bytes timestamp.T ][ 8 bytes Serial ] + Kind + + // Pubkey contains pubkey prefix and timestamp. + // + // [ 4 ][ 8 bytes pubkey prefix ][ 8 bytes timestamp.T ][ 8 bytes Serial ] + Pubkey + + // PubkeyKind contains pubkey prefix, kind and timestamp. + // + // [ 5 ][ 8 bytes pubkey prefix ][ 2 bytes kind.T ][ 8 bytes timestamp.T ][ 8 bytes Serial ] + PubkeyKind + + // Tag is for miscellaneous arbitrary length tags, with timestamp and event + // serial after. + // + // [ 6 ][ tag string 1 <= 100 bytes ][ 8 bytes timestamp.T ][ 8 bytes Serial ] + Tag + + // Tag32 contains the 8 byte pubkey prefix, timestamp and serial. + // + // [ 7 ][ 8 bytes pubkey prefix ][ 8 bytes timestamp.T ][ 8 bytes Serial ] + Tag32 + + // TagAddr contains the kind, pubkey prefix, value (index 2) of address tag (eg + // relay address), followed by timestamp and serial. + // + // [ 8 ][ 2 byte kind.T][ 8 byte pubkey prefix ][ network address ][ 8 byte timestamp.T ][ 8 byte Serial ] + TagAddr + + // Counter is the eventid.T prefix, value stores the average time of access + // (average of all access timestamps) and the size of the record. + // + // [ 9 ][ 8 bytes Serial ] : value: [ 8 bytes timestamp ] + Counter + + // Tombstone is an index that contains the left half of an event ID that has + // been deleted. The purpose of this event is to stop the event being + // republished, as a delete event may not be respected by other relays and + // eventually lead to a republication. The timestamp is added at the end to + // enable pruning the oldest tombstones. + // + // [ 10 ][ 16 bytes first/left half of event ID ][ 8 bytes timestamp ] + Tombstone +) + +// FilterPrefixes is a slice of the prefixes used by filter index to enable a loop +// for pulling events matching a serial +var FilterPrefixes = []by{ + {CreatedAt.B()}, + {Id.B()}, + {Kind.B()}, + {Pubkey.B()}, + {PubkeyKind.B()}, + {Tag.B()}, + {Tag32.B()}, + {TagAddr.B()}, +} + +// KeySizes are the byte size of keys of each type of key prefix. int(P) or call the P.I() method +// corresponds to the index 1:1. For future index additions be sure to add the +// relevant KeySizes sum as it describes the data for a programmer. +var KeySizes = []no{ + // Event + 1 + serial.Len, + // CreatedAt + 1 + createdat.Len + serial.Len, + // Id + 1 + id.Len + serial.Len, + // Kind + 1 + kinder.Len + createdat.Len + serial.Len, + // Pubkey + 1 + pubkey.Len + createdat.Len + serial.Len, + // PubkeyKind + 1 + pubkey.Len + kinder.Len + createdat.Len + serial.Len, + // Tag (worst case scenario) + 1 + 100 + createdat.Len + serial.Len, + // Tag32 + 1 + pubkey.Len + createdat.Len + serial.Len, + // TagAddr + 1 + kinder.Len + pubkey.Len + 100 + createdat.Len + serial.Len, + // Counter + 1 + serial.Len, +} diff --git a/ratel/keys/prefixes/util.go b/ratel/keys/prefixes/util.go new file mode 100644 index 0000000..9ed3ba5 --- /dev/null +++ b/ratel/keys/prefixes/util.go @@ -0,0 +1,22 @@ +package prefixes + +import ( + "bytes" + + "realy.lol/context" + "realy.lol/lol" +) + +type ( + bo = bool + by = []byte + st = string + er = error + no = int + cx = context.T +) + +var ( + log, chk, errorf = lol.Main.Log, lol.Main.Check, lol.Main.Errorf + equals = bytes.Equal +) diff --git a/ratel/main.go b/ratel/main.go index 5186a23..742783b 100644 --- a/ratel/main.go +++ b/ratel/main.go @@ -7,10 +7,10 @@ import ( "github.com/dgraph-io/badger/v4" - "realy.lol/ratel/keys/index" "realy.lol/ratel/keys/serial" "realy.lol/store" "realy.lol/units" + "realy.lol/ratel/keys/prefixes" ) const DefaultMaxLimit = 512 @@ -127,7 +127,7 @@ func (r *T) SerialKey() (idx by, ser *serial.T) { panic(err) } ser = serial.New(s) - return index.Event.Key(ser), ser + return prefixes.Event.Key(ser), ser } // Serial returns the next monotonic conflict free unique serial on the database. diff --git a/ratel/nuke.go b/ratel/nuke.go index 3d7008c..cea31c1 100644 --- a/ratel/nuke.go +++ b/ratel/nuke.go @@ -1,22 +1,22 @@ package ratel import ( - "realy.lol/ratel/keys/index" + "realy.lol/ratel/keys/prefixes" ) func (r *T) Nuke() (err er) { - log.W.F("nukening database at %s", r.dataDir) + log.W.F("nuking database at %s", r.dataDir) if err = r.DB.DropPrefix([]by{ - {index.Event.B()}, - {index.CreatedAt.B()}, - {index.Id.B()}, - {index.Kind.B()}, - {index.Pubkey.B()}, - {index.PubkeyKind.B()}, - {index.Tag.B()}, - {index.Tag32.B()}, - {index.TagAddr.B()}, - {index.Counter.B()}, + {prefixes.Event.B()}, + {prefixes.CreatedAt.B()}, + {prefixes.Id.B()}, + {prefixes.Kind.B()}, + {prefixes.Pubkey.B()}, + {prefixes.PubkeyKind.B()}, + {prefixes.Tag.B()}, + {prefixes.Tag32.B()}, + {prefixes.TagAddr.B()}, + {prefixes.Counter.B()}, }...); chk.E(err) { return } diff --git a/ratel/preparequeries.go b/ratel/preparequeries.go index 91a85c1..43bed2a 100644 --- a/ratel/preparequeries.go +++ b/ratel/preparequeries.go @@ -9,11 +9,11 @@ import ( "realy.lol/eventid" "realy.lol/filter" "realy.lol/ratel/keys/id" - "realy.lol/ratel/keys/index" "realy.lol/ratel/keys/kinder" "realy.lol/ratel/keys/pubkey" "realy.lol/ratel/keys/serial" "realy.lol/timestamp" + "realy.lol/ratel/keys/prefixes" ) type Results struct { @@ -52,7 +52,7 @@ func PrepareQueries(f *filter.T) ( // just ignore it, clients will be clients continue } - prf := index.Id.Key(ih) + prf := prefixes.Id.Key(ih) // log.T.F("id prefix to search on %0x from key %0x", prf, ih.Val) qs[i] = query{ index: i, @@ -73,7 +73,7 @@ func PrepareQueries(f *filter.T) ( // bogus filter, continue anyway continue } - sp := index.Pubkey.Key(pk) + sp := prefixes.Pubkey.Key(pk) // log.I.F("search only for authors %0x from pub key %0x", sp, pk.Val) qs[i] = query{ index: i, @@ -95,7 +95,7 @@ func PrepareQueries(f *filter.T) ( continue authors } ki := kinder.New(kind.K) - sp := index.PubkeyKind.Key(pk, ki) + sp := prefixes.PubkeyKind.Key(pk, ki) // log.T.F("search for authors from pub key %0x and kind %0x", pk.Val, ki.Val) qs[i] = query{index: i, queryFilter: f, searchPrefix: sp} i++ @@ -142,7 +142,7 @@ func PrepareQueries(f *filter.T) ( qs = make([]query, f.Kinds.Len()) for i, kind := range f.Kinds.K { kk := kinder.New(kind.K) - ki := index.Kind.Key(kk) + ki := prefixes.Kind.Key(kk) qs[i] = query{ index: i, queryFilter: f, @@ -153,7 +153,7 @@ func PrepareQueries(f *filter.T) ( default: if len(qs) > 0 { qs[0] = query{index: 0, queryFilter: f, - searchPrefix: index.CreatedAt.Key()} + searchPrefix: prefixes.CreatedAt.Key()} ext = nil } // log.T.S("other", qs) diff --git a/ratel/queryevents.go b/ratel/queryevents.go index 3526be3..da4d592 100644 --- a/ratel/queryevents.go +++ b/ratel/queryevents.go @@ -11,11 +11,11 @@ import ( "realy.lol/filter" "realy.lol/hex" "realy.lol/ratel/keys/createdat" - "realy.lol/ratel/keys/index" "realy.lol/ratel/keys/serial" "realy.lol/sha256" "realy.lol/tag" "realy.lol/timestamp" + "realy.lol/ratel/keys/prefixes" ) func (r *T) QueryEvents(c cx, f *filter.T) (evs event.Ts, err er) { @@ -64,7 +64,7 @@ func (r *T) QueryEvents(c cx, f *filter.T) (evs event.Ts, err er) { } } ser := serial.FromKey(k) - idx := index.Event.Key(ser) + idx := prefixes.Event.Key(ser) eventKeys[st(idx)] = struct{}{} } return diff --git a/ratel/saveevent.go b/ratel/saveevent.go index 25e91e3..ef47c71 100644 --- a/ratel/saveevent.go +++ b/ratel/saveevent.go @@ -13,6 +13,7 @@ import ( "realy.lol/sha256" eventstore "realy.lol/store" "realy.lol/timestamp" + "realy.lol/ratel/keys/prefixes" ) func (r *T) SaveEvent(c cx, ev *event.T) (err er) { @@ -30,7 +31,7 @@ func (r *T) SaveEvent(c cx, ev *event.T) (err er) { var ts by err = r.View(func(txn *badger.Txn) (err er) { // query event by id to ensure we don't try to save duplicates - prf := index.Id.Key(id.New(eventid.NewWith(ev.ID))) + prf := prefixes.Id.Key(id.New(eventid.NewWith(ev.ID))) it := txn.NewIterator(badger.IteratorOptions{}) defer it.Close() it.Seek(prf) @@ -44,7 +45,7 @@ func (r *T) SaveEvent(c cx, ev *event.T) (err er) { foundSerial = seri.Val } // if the event was deleted we don't want to save it again - ts = index.Tombstone.Key(id.New(eventid.NewWith(ev.ID))) + ts = prefixes.Tombstone.Key(id.New(eventid.NewWith(ev.ID))) it.Seek(ts) if it.ValidForPrefix(ts) { deleted = true @@ -61,7 +62,7 @@ func (r *T) SaveEvent(c cx, ev *event.T) (err er) { // log.D.F("found possible duplicate or stub for %s", ev.Serialize()) err = r.Update(func(txn *badger.Txn) (err er) { // retrieve the event record - evKey := keys.Write(index.New(index.Event), seri) + evKey := keys.Write(index.New(prefixes.Event), seri) it := txn.NewIterator(badger.IteratorOptions{}) defer it.Close() it.Seek(evKey) diff --git a/realy/version b/realy/version index 8185624..b06a074 100644 --- a/realy/version +++ b/realy/version @@ -1 +1 @@ -v1.3.9 \ No newline at end of file +v1.24.12.15.0 \ No newline at end of file diff --git a/tag/tag_test.go b/tag/tag_test.go index 83498ab..d712bd8 100644 --- a/tag/tag_test.go +++ b/tag/tag_test.go @@ -27,7 +27,7 @@ func TestMarshalUnmarshal(t *testing.T) { t.Fatal(err) } bc = tg2.Marshal(bc) - log.I.F("\n\norig\n%s\n\ncopy\n%s\n", bo, bc) + // log.I.F("\n\norig\n%s\n\ncopy\n%s\n", bo, bc) if !equals(bo, bc) { t.Fatalf("got\n%s\nwant\n%s", bo, bc) } diff --git a/tags/tags_test.go b/tags/tags_test.go index e7aa9f5..a3eb496 100644 --- a/tags/tags_test.go +++ b/tags/tags_test.go @@ -205,10 +205,10 @@ func TestT_ContainsAny(t *testing.T) { x := tag.New(by{'b'}, c, b, a) y := tag.New(by{'b'}, b, a, c) z := tag.New(by{'b'}, v) - _, _ = v, err tt := New(w, x, y) ttt := New(x, y) - log.I.S(tt.ContainsAny(by{'b'}, z)) - log.I.S(ttt.ContainsAny(by{'b'}, z)) + _, _, _, _, _ = v, err, z, tt, ttt + // log.I.S(tt.ContainsAny(by{'b'}, z)) + // log.I.S(ttt.ContainsAny(by{'b'}, z)) }