add compact database encoding and compression options

cmd/realy/app/implementation.go

@@ -244,6 +244,8 @@ func (r *Relay) AcceptReq(c cx, hr *http.Request, id by, ff *filters.T,
 	// not need logic and can just use the returned filter.
 	allowed = ff
 	// check that the client is authed to a pubkey in the owner follow list
+	r.Lock()
+	defer r.Unlock()
 	if len(r.Owners) > 0 {
 		for pk := range r.Followed {
 			if equals(authedPubkey, by(pk)) {

cmd/realy/main.go

@@ -50,7 +50,7 @@ func main() {
 		ratel.BackendParams{
 			Ctx:            c,
 			WG:             &wg,
-			BlockCacheSize: units.Gb * 16,
+			BlockCacheSize: units.Gb,
 			LogLevel:       lol.GetLogLevel(cfg.DbLogLevel),
 			MaxLimit:       ratel.DefaultMaxLimit,
 			Extra: []no{

event/canonical.go

@@ -46,19 +46,19 @@ func NewCanonical() (a *json.Array) {
 	return
 }
 
-func FromCanonical(b by) (ev *T, rem by, err er) {
-	id := Hash(b)
+// FromCanonical reverses the process of creating the canonical encoding, note that the signature is missing in this
+// form. Allocate an event.T before calling this.
+func (ev *T) FromCanonical(b by) (rem by, err er) {
+	rem = b
+	id := Hash(rem)
 	c := NewCanonical()
-	if rem, err = c.Unmarshal(b); chk.E(err) {
+	if rem, err = c.Unmarshal(rem); chk.E(err) {
 		return
 	}
-	if len(rem) > 0 {
-		log.I.F("rem %s", rem)
-	}
 	// unwrap the array
 	x := (*c).V
 	if v, ok := x[0].(*json.Unsigned); !ok {
-		err = errorf.E("did not encode expected type in first field of canonical event %v %v",
+		err = errorf.E("did not decode expected type in first field of canonical event %v %v",
 			reflect.TypeOf(x[0]), x[0])
 		return
 	} else {
@@ -68,7 +68,7 @@ func FromCanonical(b by) (ev *T, rem by, err er) {
 		}
 	}
 	// create the event, use the ID hash to populate the ID
-	ev = &T{ID: id}
+	ev.ID = id
 	// unwrap the pubkey
 	if v, ok := x[1].(*json.Hex); !ok {
 		err = errorf.E("failed to decode pubkey from canonical form of event %s", b)
@@ -78,28 +78,28 @@ func FromCanonical(b by) (ev *T, rem by, err er) {
 	}
 	// populate the timestamp field
 	if v, ok := x[2].(*timestamp.T); !ok {
-		err = errorf.E("did not encode expected type in third (created_at) field of canonical event %v %v",
+		err = errorf.E("did not decode expected type in third (created_at) field of canonical event %v %v",
 			reflect.TypeOf(x[0]), x[0])
 	} else {
 		ev.CreatedAt = v
 	}
 	// populate the kind field
 	if v, ok := x[3].(*kind.T); !ok {
-		err = errorf.E("did not encode expected type in fourth (kind) field of canonical event %v %v",
+		err = errorf.E("did not decode expected type in fourth (kind) field of canonical event %v %v",
 			reflect.TypeOf(x[0]), x[0])
 	} else {
 		ev.Kind = v
 	}
 	// populate the tags field
-	if v, ok := x[3].(*tags.T); !ok {
-		err = errorf.E("did not encode expected type in fourth (tags) field of canonical event %v %v",
+	if v, ok := x[4].(*tags.T); !ok {
+		err = errorf.E("did not decode expected type in fifth (tags) field of canonical event %v %v",
 			reflect.TypeOf(x[0]), x[0])
 	} else {
 		ev.Tags = v
 	}
 	// populate the content field
-	if v, ok := x[3].(*json.String); !ok {
-		err = errorf.E("did not encode expected type in fourth (content) field of canonical event %v %v",
+	if v, ok := x[5].(*json.String); !ok {
+		err = errorf.E("did not decode expected type in sixth (content) field of canonical event %v %v",
 			reflect.TypeOf(x[0]), x[0])
 	} else {
 		ev.Content = v.V

event/canonical_test.go

@@ -0,0 +1,42 @@
+package event
+
+import (
+	"bufio"
+	"bytes"
+	"testing"
+
+	"realy.lol/event/examples"
+)
+
+func TestFromCanonical(t *testing.T) {
+	scanner := bufio.NewScanner(bytes.NewBuffer(examples.Cache))
+	var rem, out, can by
+	var err er
+	for scanner.Scan() {
+		b := scanner.Bytes()
+		c := make(by, 0, len(b))
+		c = append(c, b...)
+		ea := New()
+		if rem, err = ea.Unmarshal(b); chk.E(err) {
+			t.Fatal(err)
+		}
+		if len(rem) != 0 {
+			t.Fatalf("some of input remaining after marshal/unmarshal: '%s'", rem)
+		}
+		can = ea.ToCanonical(can)
+		ea.Sig = ea.Sig[:0]
+		mrsh := ea.Marshal(nil)
+		eb := &T{}
+		if rem, err = eb.FromCanonical(can); chk.E(err) {
+			t.Fatal(err)
+		}
+		if len(rem) != 0 {
+			t.Fatalf("some of input remaining after FromCanonical: '%s'", rem)
+		}
+		mrsh2 := eb.Marshal(nil)
+		if !bytes.Equal(mrsh, mrsh2) {
+			t.Fatalf("canonical mismatch:\n\n%s\n%s", mrsh, mrsh2)
+		}
+		out, can, rem = out[:0], can[:0], rem[:0]
+	}
+}

event/compact.go

@@ -0,0 +1,24 @@
+package event
+
+import (
+	"realy.lol/ec/schnorr"
+)
+
+// MarshalCompact encodes an event as the canonical form folowed by the raw binary
+// signature (64 bytes) which hashes to form the ID, thus a compact form for the
+// database that is smaller and fast to decode.
+func (ev *T) MarshalCompact(dst by) (b by) {
+	b = dst
+	b = ev.ToCanonical(b)
+	b = append(b, ev.Sig...)
+	return
+}
+
+func (ev *T) UnmarshalCompact(b by) (rem by, err er) {
+	if rem, err = ev.FromCanonical(b); chk.E(err) {
+		return
+	}
+	ev.Sig = rem[:schnorr.SignatureSize]
+	rem = rem[schnorr.SignatureSize:]
+	return
+}

event/compact_test.go

@@ -0,0 +1,41 @@
+package event
+
+import (
+	"bufio"
+	"bytes"
+	"testing"
+
+	"realy.lol/event/examples"
+)
+
+func TestFromCompact(t *testing.T) {
+	scanner := bufio.NewScanner(bytes.NewBuffer(examples.Cache))
+	var rem, com by
+	var err er
+	for scanner.Scan() {
+		b := scanner.Bytes()
+		c := make(by, 0, len(b))
+		c = append(c, b...)
+		ea := New()
+		if rem, err = ea.Unmarshal(b); chk.E(err) {
+			t.Fatalf("error: %s", err)
+		}
+		if len(rem) != 0 {
+			t.Fatalf("some of input remaining after marshal/unmarshal: '%s'", rem)
+		}
+		com = ea.MarshalCompact(com)
+		mrsh := ea.Marshal(nil)
+		eb := &T{}
+		if rem, err = eb.UnmarshalCompact(com); chk.E(err) {
+			t.Fatalf("error: %s", err)
+		}
+		if len(rem) != 0 {
+			t.Fatalf("some of input remaining after FromCanonical: '%0x'", rem)
+		}
+		mrsh2 := eb.Marshal(nil)
+		if !bytes.Equal(mrsh, mrsh2) {
+			t.Fatalf("compact mismatch:\n\n%s\n%s", mrsh, mrsh2)
+		}
+		com, rem = com[:0], rem[:0]
+	}
+}

event/event_test.go

@@ -19,7 +19,7 @@ func TestTMarshal_Unmarshal(t *testing.T) {
 		c := make(by, 0, len(b))
 		c = append(c, b...)
 		ea := New()
-		if _, err = ea.Unmarshal(b); chk.E(err) {
+		if rem, err = ea.Unmarshal(b); chk.E(err) {
 			t.Fatal(err)
 		}
 		if len(rem) != 0 {

event/json.go

@@ -58,6 +58,8 @@ func (ev *T) Marshal(dst by) (b by) {
 	return
 }
 
+func Marshal(ev *T, dst by) (b by) { return ev.Marshal(dst) }
+
 func (ev *T) Unmarshal(b by) (r by, err er) {
 	key := make(by, 0, 9)
 	r = b
@@ -210,3 +212,5 @@ eof:
 	err = io.EOF
 	return
 }
+
+func Unmarshal(ev *T, b by) (r by, err er) { return ev.Unmarshal(b) }

json/array.go

@@ -1,5 +1,9 @@
 package json
 
+import (
+	"io"
+)
+
 // An Array is an ordered list of values.
 type Array struct{ V []I }
 
@@ -38,11 +42,20 @@ func (a *Array) Unmarshal(dst by) (rem by, err er) {
 			if rem, err = a.V[element].Unmarshal(rem); chk.E(err) {
 				return
 			}
+			element++
+			if len(rem) < 1 {
+				err = io.EOF
+				return
+			}
 			if rem[0] == ']' {
 				rem = rem[1:]
 				// done
 				return
 			}
+			if element == len(a.V) {
+				err = io.EOF
+				return
+			}
 		}
 	}
 	return

json/unsigned.go

@@ -27,7 +27,7 @@ func (u *Unsigned) Marshal(dst by) (b by) { return ints.New(u.V).Marshal(dst) }
 func (u *Unsigned) Unmarshal(dst by) (rem by, err er) {
 	rem = dst
 	n := ints.New(u.V)
-	if rem, err = n.Unmarshal(dst); chk.E(err) {
+	if rem, err = n.Unmarshal(rem); chk.E(err) {
 		return
 	}
 	u.V = n.N

ratel/compact.go

@@ -0,0 +1,31 @@
+package ratel
+
+import (
+	"realy.lol/event"
+)
+
+func (r *T) Unmarshal(ev *event.T, evb by) (rem by, err er) {
+	if r.UseCompact {
+		if rem, err = ev.UnmarshalCompact(evb); chk.E(err) {
+			ev = nil
+			evb = evb[:0]
+			return
+		}
+	} else {
+		if rem, err = ev.Unmarshal(evb); chk.E(err) {
+			ev = nil
+			evb = evb[:0]
+			return
+		}
+	}
+	return
+}
+
+func (r *T) Marshal(ev *event.T, dst by) (b by) {
+	if r.UseCompact {
+		b = ev.MarshalCompact(dst)
+	} else {
+		b = ev.Marshal(dst)
+	}
+	return
+}

ratel/countevents.go

@@ -85,9 +85,7 @@ func (r *T) CountEvents(c cx, f *filter.T) (count no, approx bo, err er) {
 					var appr bo
 					if err = item.Value(func(eventValue by) (err er) {
 						var rem by
-						if rem, err = ev.Unmarshal(eventValue); chk.E(err) {
-							ev = nil
-							eventValue = eventValue[:0]
+						if rem, err = r.Unmarshal(ev, eventValue); chk.E(err) {
 							return
 						}
 						if len(rem) > 0 {

ratel/deleteevent.go

@@ -56,10 +56,12 @@ func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er) {
 			}
 			// log.I.S(evb)
 			var rem by
-			if rem, err = ev.Unmarshal(evb); chk.E(err) {
+			if rem, err = r.Unmarshal(ev, evb); chk.E(err) {
 				return
 			}
-			_ = rem
+			if len(rem) != 0 {
+				log.I.S(rem)
+			}
 			// log.I.S(rem, ev, seri)
 			indexKeys = GetIndexKeysForEvent(ev, seri)
 			counterKey = GetCounterKey(seri)

ratel/gcsweep.go

@@ -65,7 +65,7 @@ func (r *T) GCSweep(evs, idxs DelItems) (err er) {
 			}
 			ev := &event.T{}
 			var rem by
-			if rem, err = ev.Unmarshal(evb); chk.E(err) {
+			if rem, err = r.Unmarshal(ev, evb); chk.E(err) {
 				return
 			}
 			if len(rem) != 0 {

ratel/import.go

@@ -19,6 +19,9 @@ func (r *T) Import(rr io.Reader) {
 	var count no
 	for scan.Scan() {
 		b := scan.Bytes()
+		if len(b) < 1 {
+			continue
+		}
 		ev := &event.T{}
 		if _, err = ev.Unmarshal(b); err != nil {
 			continue

ratel/init.go

@@ -17,10 +17,17 @@ func (r *T) Init(path st) (err er) {
 	log.I.Ln("opening ratel event store at", r.Path())
 	opts := badger.DefaultOptions(r.dataDir)
 	opts.BlockCacheSize = int64(r.BlockCacheSize)
-	opts.BlockSize = units.Mb
+	opts.BlockSize = 128 * units.Mb
 	opts.CompactL0OnClose = true
 	opts.LmaxCompaction = true
-	opts.Compression = options.None
+	switch r.Compression {
+	case "none":
+		opts.Compression = options.None
+	case "snappy":
+		opts.Compression = options.Snappy
+	case "zstd":
+		opts.Compression = options.ZSTD
+	}
 	r.Logger = NewLogger(r.InitLogLevel, r.dataDir)
 	opts.Logger = r.Logger
 	if r.DB, err = badger.Open(opts); chk.E(err) {