test: add gzip test on shuffled kvs (#130)

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/influxdata/line-protocol/v2 v2.2.1
 	github.com/ip2location/ip2location-go v8.3.0+incompatible
 	github.com/itchyny/timefmt-go v0.1.5
+	github.com/klauspost/compress v1.16.7
 	github.com/mssola/user_agent v0.6.0
 	github.com/oschwald/geoip2-golang v1.9.0
 	github.com/pierrec/lz4/v4 v4.1.18

go.sum

@@ -563,6 +563,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
 github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=

point/kvs.go

@@ -7,8 +7,10 @@ package point
 
 import (
 	"fmt"
+	"math/rand"
 	"sort"
 	"strings"
+	"time"
 
 	influxm "github.com/influxdata/influxdb1-client/models"
 )
@@ -462,6 +464,16 @@ func (x KVs) Keys() *Keys {
 	return &Keys{arr: arr}
 }
 
+func (x KVs) shuffle() KVs {
+	rand.Seed(time.Now().UnixNano())
+	n := len(x)
+	for i := 0; i < n; i++ {
+		j := rand.Intn(n)
+		x[i], x[j] = x[j], x[i]
+	}
+	return x
+}
+
 func KVKey(kv *Field) *Key {
 	t := PBType(kv.Val)
 

point/kvs_test.go

@@ -6,10 +6,12 @@
 package point
 
 import (
+	"bytes"
 	"math"
 	"sort"
 	T "testing"
 
+	gzip "github.com/klauspost/compress/gzip"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -586,3 +588,91 @@ func BenchmarkKVsDel(b *T.B) {
 		}
 	})
 }
+
+func Test_shuffle(t *T.T) {
+	var kvs KVs
+	kvs = kvs.Add(`f1`, false, false, false)
+	kvs = kvs.Add(`f2`, 123, false, false)
+	kvs = kvs.Add(`f3`, "some string", false, false)
+	kvs = kvs.Add(`f4`, []byte("hello world"), false, false)
+	kvs = kvs.Add(`f5`, 3.14, false, false)
+	kvs = kvs.Add(`f6`, uint(8), false, false)
+
+	kvs = kvs.shuffle()
+
+	p := NewPointV2(t.Name(), kvs)
+
+	t.Logf(p.Pretty())
+}
+
+func TestShuffleGzip(t *T.T) {
+	var pts []*Point
+	for i := 0; i < 1000; i++ {
+		var kvs KVs
+		kvs = kvs.Add(`f1`, false, false, false)
+		kvs = kvs.Add(`f2`, 123, false, false)
+		kvs = kvs.Add(`f3`, "some string", false, false)
+		kvs = kvs.Add(`f4`, []byte("hello world"), false, false)
+		kvs = kvs.Add(`f5`, 3.14, false, false)
+		kvs = kvs.Add(`f6`, uint(8), false, false)
+		pts = append(pts, NewPointV2(t.Name(), kvs))
+	}
+
+	enc := GetEncoder(WithEncEncoding(Protobuf))
+	defer PutEncoder(enc)
+
+	arr, err := enc.Encode(pts)
+	assert.NoError(t, err)
+	raw1 := arr[0]
+
+	for _, pt := range pts {
+		pt.pt.Fields = KVs(pt.pt.Fields).shuffle()
+	}
+
+	// make sure kvs are shuffled.
+	t.Logf("shuffled point: %s\n%s", pts[0].Pretty(), pts[len(pts)-1].Pretty())
+
+	arr, err = enc.Encode(pts)
+	assert.NoError(t, err)
+	raw2 := arr[0]
+
+	t.Logf("#raw1: %d, #raw2: %d", len(raw1), len(raw2))
+
+	// new gzip handler
+	buf := &bytes.Buffer{}
+	w := gzip.NewWriter(buf)
+
+	_, err = w.Write(raw1)
+	assert.NoError(t, err)
+	w.Flush()
+	w.Close()
+	zlen1 := buf.Len()
+
+	// clear & reset gzip
+	buf.Reset()
+	w.Reset(buf)
+
+	// gzip on shuffled body
+	_, err = w.Write(raw2)
+	assert.NoError(t, err)
+	w.Flush()
+	w.Close()
+	zlen2 := buf.Len()
+	t.Logf("#zraw1: %d, #zraw2: %d", zlen1, zlen2)
+}
+
+func BenchmarkShuffle(b *T.B) {
+	b.Run(`basic`, func(b *T.B) {
+		var kvs KVs
+		kvs = kvs.Add(`f1`, false, false, false)
+		kvs = kvs.Add(`f2`, 123, false, false)
+		kvs = kvs.Add(`f3`, "some string", false, false)
+		kvs = kvs.Add(`f4`, []byte("hello world"), false, false)
+		kvs = kvs.Add(`f5`, 3.14, false, false)
+		kvs = kvs.Add(`f6`, uint(8), false, false)
+
+		for i := 0; i < b.N; i++ {
+			kvs = kvs.shuffle()
+		}
+	})
+}