Object Storage (#38)

.github/pull_request_template.md

@@ -2,7 +2,7 @@
 
 [Brief Description of the Changes]
 
-Fixes SC-XXXXX
+<!-- Fixes SC-XXXXX -->
 
 ### Type of change
 
@@ -15,20 +15,23 @@ Fixes SC-XXXXX
 
 ### Acceptance criteria
 
-[Copy acceptance criteria checklist from story for reviewer, or add a brief acceptance criteria here]
+This PR will be merged without review.
 
 ### Definition of Done
 
 - [ ] I have manually tested the change running it locally (having rebuilt all containers) or via unit tests
 - [ ] I have added unit and/or integration tests that cover my changes
 - [ ] I have added new test fixtures as needed to support added tests
 - [ ] I have updated the dependencies list if necessary (including updating yarn.lock and/or go.sum)
-- [ ] I have recompiled and included new protocol buffers to reflect changes I made if necessary
+
+<!--
 - [ ] Check this box if a reviewer can merge this pull request after approval (leave it unchecked if you want to do it yourself)
 - [ ] I have notified the reviewer via Shortcut or Slack that this is ready for review
 - [ ] Documented service configuration changes or created related devops stories
 
 ### Reviewer(s) checklist
 
 - [ ] Any new user-facing content that has been added for this PR has been QA'ed to ensure correct grammar, spelling, and understandability.
-- [ ] Are there any TODOs in this PR that should be turned into stories?
+- [ ] Are there any TODOs in this PR that should be turned into stories?
+
+-->

README.md

@@ -17,12 +17,16 @@ Protocol Buffers are a compact, cross-language compatible data serialization for
 ```
 goos: darwin
 goarch: arm64
-pkg: github.com/rotationalio/honu/pkg/store
+pkg: github.com/rotationalio/honu/pkg/store/object
 cpu: Apple M1 Max
-BenchmarkSerialization/Small/Encode/Honu-10         	  653840	      1905 ns/op	      1471 bytes	    4319 B/op	       2 allocs/op
-BenchmarkSerialization/Small/Encode/Protobuf-10     	  346150	      3164 ns/op	      2426 bytes	    4044 B/op	       1 allocs/op
-BenchmarkSerialization/Small/Decode/Honu-10         	  343032	      3421 ns/op	    4784 B/op	      62 allocs/op
-BenchmarkSerialization/Small/Decode/Protobuf-10     	  161241	      7889 ns/op	    8367 B/op	     127 allocs/op
+BenchmarkSerialization/Small/Encode-10  	  578818	      1768 ns/op	      4520 bytes	    4487 B/op	       2 allocs/op
+BenchmarkSerialization/Small/Decode-10  	  402945	      2686 ns/op	    2341 B/op	      62 allocs/op
+BenchmarkSerialization/Medium/Encode-10 	  363483	      3308 ns/op	     10128 bytes	   28081 B/op	       2 allocs/op
+BenchmarkSerialization/Medium/Decode-10 	  471076	      2562 ns/op	    2322 B/op	      61 allocs/op
+BenchmarkSerialization/Large/Encode-10  	   93942	     12124 ns/op	    303630 bytes	  207933 B/op	       2 allocs/op
+BenchmarkSerialization/Large/Decode-10  	  467475	      2736 ns/op	    2334 B/op	      62 allocs/op
+BenchmarkSerialization/XLarge/Encode-10 	    7250	    138013 ns/op	   4926099 bytes	 3247592 B/op	       2 allocs/op
+BenchmarkSerialization/XLarge/Decode-10 	  407468	      2749 ns/op	    2333 B/op	      62 allocs/op
 ```
 
 ![Benchmarks](./docs/static/img/serialization-benchmark.png)

pkg/store/decode.go → pkg/store/lani/decode.go

@@ -1,4 +1,4 @@
-package store
+package lani
 
 import (
 	"encoding/binary"
@@ -8,17 +8,6 @@ import (
 	"github.com/oklog/ulid/v2"
 )
 
-// Unmarshal a decodable object from a byte slice for deserialization.
-func Unmarshal(data []byte, v Decodable) (err error) {
-	decoder := NewDecoder(data)
-	return v.Decode(decoder)
-}
-
-// All objects in this package must be decodable.
-type Decodable interface {
-	Decode(*Decoder) error
-}
-
 // Decoder is similar to a bytes.Reader, allowing a sequential decoding of byte frames,
 // so that every repeated decoding advances the internal index to the next frame.
 type Decoder struct {
@@ -243,7 +232,8 @@ func (d *Decoder) DecodeTime() (_ time.Time, err error) {
 		return time.Time{}, nil
 	}
 
-	return time.Unix(0, ts), nil
+	t := time.Unix(0, ts).In(time.UTC)
+	return t, nil
 }
 
 // Decode a struct (must implement the Decodable interface). This function performs

pkg/store/decode_test.go → pkg/store/lani/decode_test.go

@@ -1,4 +1,4 @@
-package store_test
+package lani_test
 
 import (
 	"crypto/rand"
@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/oklog/ulid/v2"
-	. "github.com/rotationalio/honu/pkg/store"
+	. "github.com/rotationalio/honu/pkg/store/lani"
 	"github.com/stretchr/testify/require"
 )
 
@@ -344,12 +344,12 @@ func TestDecoder(t *testing.T) {
 
 	t.Run("Struct", func(t *testing.T) {
 		t.Run("Nil", func(t *testing.T) {
-			var obj *Compression
+			var obj *Mock
 
 			_, err := enc.EncodeStruct(obj)
 			require.NoError(t, err, "could not encode nil struct input")
 
-			var cmp *Compression
+			var cmp *Mock
 			isNil, err := NewDecoder(enc.Bytes()).DecodeStruct(cmp)
 			require.NoError(t, err, "could not decode nil struct")
 			require.True(t, isNil, "expected isNil to be true")

pkg/store/encode.go → pkg/store/lani/encode.go

@@ -1,8 +1,8 @@
-package store
+package lani
 
 import (
 	"encoding/binary"
-	"fmt"
+	"reflect"
 	"time"
 
 	"github.com/oklog/ulid/v2"
@@ -13,22 +13,6 @@ const (
 	smallBufferSize = 64
 )
 
-// Marshal an encodable object into a byte slice for storage or serialization.
-func Marshal(v Encodable) (_ []byte, err error) {
-	encoder := &Encoder{}
-	encoder.Grow(v.Size())
-	if _, err = v.Encode(encoder); err != nil {
-		return nil, err
-	}
-	return encoder.Bytes(), nil
-}
-
-// All objects in this package must be encodable.
-type Encodable interface {
-	Size() int
-	Encode(*Encoder) (int, error)
-}
-
 // Encoder is similar to a bytes.Buffer in that it maintains an internal buffer that
 // it keeps at the largest capacity its seen for repeated encodings. To use the
 // Encoder, sequentially encode values or objects to the buffer then read the encoded
@@ -242,23 +226,16 @@ func (e *Encoder) EncodeStruct(s Encodable) (n int, err error) {
 }
 
 func isNilEncodable(s Encodable) bool {
-	switch t := s.(type) {
-	case *Object:
-		return t == nil
-	case *Version:
-		return t == nil
-	case *SchemaVersion:
-		return t == nil
-	case *AccessControl:
-		return t == nil
-	case *Publisher:
-		return t == nil
-	case *Encryption:
-		return t == nil
-	case *Compression:
-		return t == nil
+	iv := reflect.ValueOf(s)
+	if !iv.IsValid() {
+		return true
+	}
+
+	switch iv.Kind() {
+	case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Func, reflect.Interface:
+		return iv.IsNil()
 	default:
-		panic(fmt.Errorf("unknown type %T", t))
+		return false
 	}
 }
 

pkg/store/encode_test.go → pkg/store/lani/encode_test.go

@@ -1,4 +1,4 @@
-package store_test
+package lani_test
 
 import (
 	"crypto/rand"
@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/oklog/ulid/v2"
-	. "github.com/rotationalio/honu/pkg/store"
+	. "github.com/rotationalio/honu/pkg/store/lani"
 	"github.com/stretchr/testify/require"
 )
 
@@ -396,7 +396,7 @@ func TestEncoder(t *testing.T) {
 
 	t.Run("Struct", func(t *testing.T) {
 		t.Run("Nil", func(t *testing.T) {
-			var obj *Compression
+			var obj *Mock
 			enc := &Encoder{}
 			n, err := enc.EncodeStruct(obj)
 			require.NoError(t, err, "could not encode nil struct")

pkg/store/errors.go → pkg/store/lani/errors.go

@@ -1,4 +1,4 @@
-package store
+package lani
 
 import "errors"
 

pkg/store/lani/lani.go

@@ -0,0 +1,33 @@
+/*
+Lani is sky or heavens in Hawaiian and also sounds like line; we've chosen this word to
+represent Honu's byte serialization format for data storage; encoding in a serial or
+single skyward direction and decoding in a landward direction.
+*/
+package lani
+
+// Specifies the interface for objects that can be encoded using lani.
+type Encodable interface {
+	Size() int
+	Encode(*Encoder) (int, error)
+}
+
+// Specifies the interface for objects that can be decoded using lani.
+type Decodable interface {
+	Decode(*Decoder) error
+}
+
+// Marshal an encodable type into a byte slice for storage or serialization.
+func Marshal(v Encodable) (_ []byte, err error) {
+	encoder := &Encoder{}
+	encoder.Grow(v.Size())
+	if _, err = v.Encode(encoder); err != nil {
+		return nil, err
+	}
+	return encoder.Bytes(), nil
+}
+
+// Unmarshal a decodable type from a byte slice for deserialization.
+func Unmarshal(data []byte, v Decodable) (err error) {
+	decoder := NewDecoder(data)
+	return v.Decode(decoder)
+}

pkg/store/mock.go → pkg/store/lani/mock.go

@@ -1,4 +1,4 @@
-package store
+package lani
 
 import "encoding/binary"
 

pkg/store/metadata/acls.go

@@ -0,0 +1,50 @@
+package metadata
+
+import (
+	"github.com/oklog/ulid/v2"
+	"github.com/rotationalio/honu/pkg/store/lani"
+)
+
+//===========================================================================
+// Access Control List
+//===========================================================================
+
+type AccessControl struct {
+	ClientID    ulid.ULID `json:"client_id" msg:"client_id"`
+	Permissions uint8     `json:"permissions" msg:"permissions"`
+}
+
+var _ lani.Encodable = &AccessControl{}
+var _ lani.Decodable = &AccessControl{}
+
+func (o *AccessControl) Size() int {
+	// ULID + 1 byte for the permissions
+	return 17
+}
+
+func (o *AccessControl) Encode(e *lani.Encoder) (n int, err error) {
+	var m int
+	if m, err = e.EncodeULID(o.ClientID); err != nil {
+		return n + m, err
+	}
+	n += m
+
+	if m, err = e.EncodeUint8(o.Permissions); err != nil {
+		return n + m, err
+	}
+	n += m
+
+	return
+}
+
+func (o *AccessControl) Decode(d *lani.Decoder) (err error) {
+	if o.ClientID, err = d.DecodeULID(); err != nil {
+		return err
+	}
+
+	if o.Permissions, err = d.DecodeUint8(); err != nil {
+		return err
+	}
+
+	return nil
+}