diff --git a/decode.go b/decode.go index 5df40e5..ea645aa 100644 --- a/decode.go +++ b/decode.go @@ -14,14 +14,16 @@ import ( ) const ( - looseInterfaceDecodingFlag uint32 = 1 << iota - disallowUnknownFieldsFlag + bytesAllocLimit = 1 << 20 // 1mb + sliceAllocLimit = 1e6 // 1m elements + maxMapSize = 1e6 // 1m elements ) const ( - bytesAllocLimit = 1e6 // 1mb - sliceAllocLimit = 1e4 - maxMapSize = 1e6 + looseInterfaceDecodingFlag uint32 = 1 << iota + disallowUnknownFieldsFlag + usePreallocateValues + disableAllocLimitFlag ) type bufReader interface { @@ -53,7 +55,7 @@ func PutDecoder(dec *Decoder) { // in the value pointed to by v. func Unmarshal(data []byte, v interface{}) error { dec := GetDecoder() - + dec.UsePreallocateValues(true) dec.Reset(bytes.NewReader(data)) err := dec.Decode(v) @@ -64,16 +66,14 @@ func Unmarshal(data []byte, v interface{}) error { // A Decoder reads and decodes MessagePack values from an input stream. type Decoder struct { - r io.Reader - s io.ByteScanner - buf []byte - - rec []byte // accumulates read data if not nil - + r io.Reader + s io.ByteScanner + mapDecoder func(*Decoder) (interface{}, error) + structTag string + buf []byte + rec []byte dict []string flags uint32 - structTag string - mapDecoder func(*Decoder) (interface{}, error) } // NewDecoder returns a new decoder that reads from r. @@ -95,10 +95,9 @@ func (d *Decoder) Reset(r io.Reader) { // ResetDict is like Reset, but also resets the dict. func (d *Decoder) ResetDict(r io.Reader, dict []string) { - d.resetReader(r) + d.ResetReader(r) d.flags = 0 d.structTag = "" - d.mapDecoder = nil d.dict = dict } @@ -110,10 +109,16 @@ func (d *Decoder) WithDict(dict []string, fn func(*Decoder) error) error { return err } -func (d *Decoder) resetReader(r io.Reader) { +func (d *Decoder) ResetReader(r io.Reader) { + d.mapDecoder = nil + d.dict = nil + if br, ok := r.(bufReader); ok { d.r = br d.s = br + } else if r == nil { + d.r = nil + d.s = nil } else { br := bufio.NewReader(r) d.r = br @@ -161,6 +166,24 @@ func (d *Decoder) UseInternedStrings(on bool) { } } +// UsePreallocateValues enables preallocating values in chunks +func (d *Decoder) UsePreallocateValues(on bool) { + if on { + d.flags |= usePreallocateValues + } else { + d.flags &= ^usePreallocateValues + } +} + +// DisableAllocLimit enables fully allocating slices/maps when the size is known +func (d *Decoder) DisableAllocLimit(on bool) { + if on { + d.flags |= disableAllocLimitFlag + } else { + d.flags &= ^disableAllocLimitFlag + } +} + // Buffered returns a reader of the data remaining in the Decoder's buffer. // The reader is valid until the next call to Decode. func (d *Decoder) Buffered() io.Reader { @@ -603,7 +626,11 @@ func (d *Decoder) readFull(b []byte) error { func (d *Decoder) readN(n int) ([]byte, error) { var err error - d.buf, err = readN(d.r, d.buf, n) + if d.flags&disableAllocLimitFlag != 0 { + d.buf, err = readN(d.r, d.buf, n) + } else { + d.buf, err = readNGrow(d.r, d.buf, n) + } if err != nil { return nil, err } @@ -615,6 +642,24 @@ func (d *Decoder) readN(n int) ([]byte, error) { } func readN(r io.Reader, b []byte, n int) ([]byte, error) { + if b == nil { + if n == 0 { + return make([]byte, 0), nil + } + b = make([]byte, 0, n) + } + + if n > cap(b) { + b = append(b, make([]byte, n-len(b))...) + } else if n <= cap(b) { + b = b[:n] + } + + _, err := io.ReadFull(r, b) + return b, err +} + +func readNGrow(r io.Reader, b []byte, n int) ([]byte, error) { if b == nil { if n == 0 { return make([]byte, 0), nil diff --git a/decode_map.go b/decode_map.go index d310000..c54dae3 100644 --- a/decode_map.go +++ b/decode_map.go @@ -35,7 +35,11 @@ func decodeMapValue(d *Decoder, v reflect.Value) error { } if v.IsNil() { - v.Set(reflect.MakeMap(typ)) + ln := n + if d.flags&disableAllocLimitFlag == 0 { + ln = min(ln, maxMapSize) + } + v.Set(reflect.MakeMapWithSize(typ, ln)) } if n == 0 { return nil @@ -106,7 +110,11 @@ func (d *Decoder) decodeMapStringStringPtr(ptr *map[string]string) error { m := *ptr if m == nil { - *ptr = make(map[string]string, min(size, maxMapSize)) + ln := size + if d.flags&disableAllocLimitFlag == 0 { + ln = min(size, maxMapSize) + } + *ptr = make(map[string]string, ln) m = *ptr } @@ -149,7 +157,7 @@ func (d *Decoder) DecodeMap() (map[string]interface{}, error) { return nil, nil } - m := make(map[string]interface{}, min(n, maxMapSize)) + m := make(map[string]interface{}, n) for i := 0; i < n; i++ { mk, err := d.DecodeString() @@ -176,7 +184,7 @@ func (d *Decoder) DecodeUntypedMap() (map[interface{}]interface{}, error) { return nil, nil } - m := make(map[interface{}]interface{}, min(n, maxMapSize)) + m := make(map[interface{}]interface{}, n) for i := 0; i < n; i++ { mk, err := d.decodeInterfaceCond() @@ -224,7 +232,13 @@ func (d *Decoder) DecodeTypedMap() (interface{}, error) { } mapType := reflect.MapOf(keyType, valueType) - mapValue := reflect.MakeMap(mapType) + + ln := n + if d.flags&disableAllocLimitFlag == 0 { + ln = min(ln, maxMapSize) + } + + mapValue := reflect.MakeMapWithSize(mapType, ln) mapValue.SetMapIndex(reflect.ValueOf(key), reflect.ValueOf(value)) n-- @@ -236,17 +250,18 @@ func (d *Decoder) DecodeTypedMap() (interface{}, error) { } func (d *Decoder) decodeTypedMapValue(v reflect.Value, n int) error { - typ := v.Type() - keyType := typ.Key() - valueType := typ.Elem() - + var ( + typ = v.Type() + keyType = typ.Key() + valueType = typ.Elem() + ) for i := 0; i < n; i++ { - mk := reflect.New(keyType).Elem() + mk := d.newValue(keyType).Elem() if err := d.DecodeValue(mk); err != nil { return err } - mv := reflect.New(valueType).Elem() + mv := d.newValue(valueType).Elem() if err := d.DecodeValue(mv); err != nil { return err } diff --git a/decode_query.go b/decode_query.go index c302ed1..4dce0fe 100644 --- a/decode_query.go +++ b/decode_query.go @@ -11,9 +11,8 @@ import ( type queryResult struct { query string key string + values []interface{} hasAsterisk bool - - values []interface{} } func (q *queryResult) nextKey() { diff --git a/decode_slice.go b/decode_slice.go index db6f7c5..9c155f2 100644 --- a/decode_slice.go +++ b/decode_slice.go @@ -49,7 +49,7 @@ func (d *Decoder) decodeStringSlicePtr(ptr *[]string) error { return nil } - ss := makeStrings(*ptr, n) + ss := makeStrings(*ptr, n, d.flags&disableAllocLimitFlag != 0) for i := 0; i < n; i++ { s, err := d.DecodeString() if err != nil { @@ -62,8 +62,8 @@ func (d *Decoder) decodeStringSlicePtr(ptr *[]string) error { return nil } -func makeStrings(s []string, n int) []string { - if n > sliceAllocLimit { +func makeStrings(s []string, n int, noLimit bool) []string { + if !noLimit && n > sliceAllocLimit { n = sliceAllocLimit } @@ -101,10 +101,17 @@ func decodeSliceValue(d *Decoder, v reflect.Value) error { v.Set(v.Slice(0, v.Cap())) } + noLimit := d.flags&disableAllocLimitFlag != 1 + + if noLimit && n > v.Len() { + v.Set(growSliceValue(v, n, noLimit)) + } + for i := 0; i < n; i++ { - if i >= v.Len() { - v.Set(growSliceValue(v, n)) + if !noLimit && i >= v.Len() { + v.Set(growSliceValue(v, n, noLimit)) } + elem := v.Index(i) if err := d.DecodeValue(elem); err != nil { return err @@ -114,9 +121,9 @@ func decodeSliceValue(d *Decoder, v reflect.Value) error { return nil } -func growSliceValue(v reflect.Value, n int) reflect.Value { +func growSliceValue(v reflect.Value, n int, noLimit bool) reflect.Value { diff := n - v.Len() - if diff > sliceAllocLimit { + if !noLimit && diff > sliceAllocLimit { diff = sliceAllocLimit } v = reflect.AppendSlice(v, reflect.MakeSlice(v.Type(), diff, diff)) @@ -163,7 +170,7 @@ func (d *Decoder) decodeSlice(c byte) ([]interface{}, error) { return nil, nil } - s := make([]interface{}, 0, min(n, sliceAllocLimit)) + s := make([]interface{}, 0, n) for i := 0; i < n; i++ { v, err := d.decodeInterfaceCond() if err != nil { diff --git a/decode_typgen.go b/decode_typgen.go new file mode 100644 index 0000000..0b4c1d0 --- /dev/null +++ b/decode_typgen.go @@ -0,0 +1,46 @@ +package msgpack + +import ( + "reflect" + "sync" +) + +var cachedValues struct { + m map[reflect.Type]chan reflect.Value + sync.RWMutex +} + +func cachedValue(t reflect.Type) reflect.Value { + cachedValues.RLock() + ch := cachedValues.m[t] + cachedValues.RUnlock() + if ch != nil { + return <-ch + } + + cachedValues.Lock() + defer cachedValues.Unlock() + if ch = cachedValues.m[t]; ch != nil { + return <-ch + } + + ch = make(chan reflect.Value, 256) + go func() { + for { + ch <- reflect.New(t) + } + }() + if cachedValues.m == nil { + cachedValues.m = make(map[reflect.Type]chan reflect.Value, 8) + } + cachedValues.m[t] = ch + return <-ch +} + +func (d *Decoder) newValue(t reflect.Type) reflect.Value { + if d.flags&usePreallocateValues == 0 { + return reflect.New(t) + } + + return cachedValue(t) +} diff --git a/decode_value.go b/decode_value.go index a593ea5..556c6ac 100644 --- a/decode_value.go +++ b/decode_value.go @@ -128,12 +128,12 @@ func ptrValueDecoder(typ reflect.Type) decoderFunc { return func(d *Decoder, v reflect.Value) error { if d.hasNilCode() { if !v.IsNil() { - v.Set(reflect.Zero(v.Type())) + v.Set(d.newValue(typ)) } return d.DecodeNil() } if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) + v.Set(d.newValue(typ.Elem())) } return decoder(d, v.Elem()) } @@ -155,7 +155,7 @@ func nilAwareDecoder(typ reflect.Type, fn decoderFunc) decoderFunc { return d.decodeNilValue(v) } if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) + v.Set(d.newValue(typ.Elem())) } return fn(d, v) } diff --git a/encode.go b/encode.go index 64ca87c..135adc8 100644 --- a/encode.go +++ b/encode.go @@ -75,15 +75,12 @@ func Marshal(v interface{}) ([]byte, error) { } type Encoder struct { - w writer - - buf []byte - timeBuf []byte - - dict map[string]int - - flags uint32 + w writer + dict map[string]int structTag string + buf []byte + timeBuf []byte + flags uint32 } // NewEncoder returns a new encoder that writes to w. @@ -107,7 +104,7 @@ func (e *Encoder) Reset(w io.Writer) { // ResetDict is like Reset, but also resets the dict. func (e *Encoder) ResetDict(w io.Writer, dict map[string]int) { - e.resetWriter(w) + e.ResetWriter(w) e.flags = 0 e.structTag = "" e.dict = dict @@ -121,9 +118,12 @@ func (e *Encoder) WithDict(dict map[string]int, fn func(*Encoder) error) error { return err } -func (e *Encoder) resetWriter(w io.Writer) { +func (e *Encoder) ResetWriter(w io.Writer) { + e.dict = nil if bw, ok := w.(writer); ok { e.w = bw + } else if w == nil { + e.w = nil } else { e.w = newByteWriter(w) } diff --git a/encode_value.go b/encode_value.go index 7433c88..1d6303a 100644 --- a/encode_value.go +++ b/encode_value.go @@ -200,6 +200,13 @@ func nilable(kind reflect.Kind) bool { return false } +func nilableType(t reflect.Type) bool { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + return nilable(t.Kind()) +} + //------------------------------------------------------------------------------ func marshalBinaryValueAddr(e *Encoder, v reflect.Value) error { diff --git a/ext.go b/ext.go index 65bfc19..354b9d9 100644 --- a/ext.go +++ b/ext.go @@ -254,9 +254,9 @@ func (d *Decoder) decodeInterfaceExt(c byte) (interface{}, error) { return nil, fmt.Errorf("msgpack: unknown ext id=%d", extID) } - v := reflect.New(info.Type).Elem() + v := d.newValue(info.Type).Elem() if nilable(v.Kind()) && v.IsNil() { - v.Set(reflect.New(info.Type.Elem())) + v.Set(d.newValue(info.Type.Elem())) } if err := info.Decoder(d, v, extLen); err != nil { diff --git a/go.mod b/go.mod index f630a54..f9d6cda 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,14 @@ module github.com/vmihailenco/msgpack/v5 -go 1.11 +go 1.19 require ( github.com/stretchr/testify v1.6.1 github.com/vmihailenco/tagparser/v2 v2.0.0 ) + +require ( + github.com/davecgh/go-spew v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect +) diff --git a/msgpack.go b/msgpack.go index 4db2fa2..4fa000b 100644 --- a/msgpack.go +++ b/msgpack.go @@ -43,8 +43,8 @@ func (m *RawMessage) DecodeMsgpack(dec *Decoder) error { //------------------------------------------------------------------------------ type unexpectedCodeError struct { - code byte hint string + code byte } func (err unexpectedCodeError) Error() string { diff --git a/msgpack_test.go b/msgpack_test.go index 88791fe..21a1cf6 100644 --- a/msgpack_test.go +++ b/msgpack_test.go @@ -49,7 +49,6 @@ func (t *MsgpackTest) TestTime() { t.Nil(t.dec.Decode(&out)) t.True(out.Equal(zero)) t.True(out.IsZero()) - } func (t *MsgpackTest) TestLargeBytes() { @@ -509,14 +508,15 @@ func ExampleMarshal_ignore_simple_zero_structs_when_tagged_with_omitempty() { // msgpack_test.T{I:msgpack_test.NullInt{Valid:true, Int:42}, J:msgpack_test.NullInt{Valid:true, Int:0}, S:msgpack_test.Secretive{Visible:false, hidden:false}} } -type Value interface{} -type Wrapper struct { - Value Value `msgpack:"v,omitempty"` -} +type ( + Value interface{} + Wrapper struct { + Value Value `msgpack:"v,omitempty"` + } +) func TestEncodeWrappedValue(t *testing.T) { - var v Value - v = (*time.Time)(nil) + v := (*time.Time)(nil) c := &Wrapper{ Value: v, } diff --git a/types.go b/types.go index 51851fe..d212e09 100644 --- a/types.go +++ b/types.go @@ -66,8 +66,8 @@ type structCache struct { } type structCacheKey struct { - tag string typ reflect.Type + tag string } func newStructCache() *structCache { @@ -90,11 +90,11 @@ func (m *structCache) Fields(typ reflect.Type, tag string) *fields { //------------------------------------------------------------------------------ type field struct { + encoder encoderFunc + decoder decoderFunc name string index []int omitEmpty bool - encoder encoderFunc - decoder decoderFunc } func (f *field) Omit(e *Encoder, strct reflect.Value) bool { @@ -405,7 +405,7 @@ func indirectNil(v reflect.Value) (reflect.Value, bool) { if elemType.Kind() != reflect.Struct { return v, false } - v.Set(reflect.New(elemType)) + v.Set(cachedValue(elemType)) } v = v.Elem() } diff --git a/types_test.go b/types_test.go index fbc1ad3..f8bfda1 100644 --- a/types_test.go +++ b/types_test.go @@ -778,8 +778,7 @@ func TestStringsBin(t *testing.T) { require.Nil(t, err) require.Equal(t, v.(string), test.in) - var dst interface{} - dst = "" + var dst interface{} = "" err = msgpack.Unmarshal(b, &dst) require.EqualError(t, err, "msgpack: Decode(non-pointer string)") } @@ -846,8 +845,7 @@ func TestBin(t *testing.T) { t.Fatalf("%x != %x", v, test.in) } - var dst interface{} - dst = make([]byte, 0) + var dst interface{} = make([]byte, 0) err = msgpack.Unmarshal(b, &dst) if err.Error() != "msgpack: Decode(non-pointer []uint8)" { t.Fatal(err) @@ -910,8 +908,7 @@ func TestUint64(t *testing.T) { t.Fatalf("%d != %d", out2, int64(test.in)) } - var out3 interface{} - out3 = uint64(0) + var out3 interface{} = uint64(0) err = msgpack.Unmarshal(buf.Bytes(), &out3) if err.Error() != "msgpack: Decode(non-pointer uint64)" { t.Fatal(err) @@ -998,8 +995,7 @@ func TestInt64(t *testing.T) { t.Fatalf("%d != %d", out2, uint64(test.in)) } - var out3 interface{} - out3 = int64(0) + var out3 interface{} = int64(0) err = msgpack.Unmarshal(buf.Bytes(), &out3) if err.Error() != "msgpack: Decode(non-pointer int64)" { t.Fatal(err) @@ -1068,8 +1064,7 @@ func TestFloat32(t *testing.T) { t.Fatalf("%f != %f", v, test.in) } - var dst interface{} - dst = float32(0) + var dst interface{} = float32(0) err = msgpack.Unmarshal(b, &dst) if err.Error() != "msgpack: Decode(non-pointer float32)" { t.Fatal(err) @@ -1134,8 +1129,7 @@ func TestFloat64(t *testing.T) { t.Fatalf("%f != %f", v, test.in) } - var dst interface{} - dst = float64(0) + var dst interface{} = float64(0) err = msgpack.Unmarshal(b, &dst) if err.Error() != "msgpack: Decode(non-pointer float64)" { t.Fatal(err)