From 63a2452a32bf627dbbd4726d9c241e7cfa241876 Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Tue, 19 Apr 2022 18:22:30 +0300 Subject: [PATCH 1/2] fix(proto): set low cardinality index properly during encoding --- proto/_golden/col_arr_of_low_cord_str.hex | 4 ++ proto/_golden/col_arr_of_low_cord_str.raw | Bin 0 -> 59 bytes ..._col_arr_of_str.hex => col_arr_of_str.hex} | 0 ..._col_arr_of_str.raw => col_arr_of_str.raw} | Bin proto/_golden/col_low_cardinality_of_str.hex | 2 +- proto/_golden/col_low_cardinality_of_str.raw | Bin 41 -> 41 bytes proto/col_arr_of.go | 10 +++++ proto/col_arr_of_test.go | 36 +++++++++++++++++- proto/col_low_cardinality_of.go | 1 + query_test.go | 29 ++++++++++++++ 10 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 proto/_golden/col_arr_of_low_cord_str.hex create mode 100644 proto/_golden/col_arr_of_low_cord_str.raw rename proto/_golden/{col_col_arr_of_str.hex => col_arr_of_str.hex} (100%) rename proto/_golden/{col_col_arr_of_str.raw => col_arr_of_str.raw} (100%) diff --git a/proto/_golden/col_arr_of_low_cord_str.hex b/proto/_golden/col_arr_of_low_cord_str.hex new file mode 100644 index 00000000..eee99a3b --- /dev/null +++ b/proto/_golden/col_arr_of_low_cord_str.hex @@ -0,0 +1,4 @@ +00000000 05 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 |................| +00000010 00 06 00 00 00 00 00 00 03 00 00 00 00 00 00 00 |................| +00000020 03 66 6f 6f 03 62 61 72 03 62 61 7a 07 00 00 00 |.foo.bar.baz....| +00000030 00 00 00 00 00 01 00 00 02 00 02 |...........| diff --git a/proto/_golden/col_arr_of_low_cord_str.raw b/proto/_golden/col_arr_of_low_cord_str.raw new file mode 100644 index 0000000000000000000000000000000000000000..98f0801bd384666341cffe935d86450ae1d25661 GIT binary patch literal 59 mcmZQ&fB<$V&AFhlvwY5DogNr^>3Tm@Id$iTqFzyttOO9SBm literal 0 HcmV?d00001 diff --git a/proto/_golden/col_col_arr_of_str.hex b/proto/_golden/col_arr_of_str.hex similarity index 100% rename from proto/_golden/col_col_arr_of_str.hex rename to proto/_golden/col_arr_of_str.hex diff --git a/proto/_golden/col_col_arr_of_str.raw b/proto/_golden/col_arr_of_str.raw similarity index 100% rename from proto/_golden/col_col_arr_of_str.raw rename to proto/_golden/col_arr_of_str.raw diff --git a/proto/_golden/col_low_cardinality_of_str.hex b/proto/_golden/col_low_cardinality_of_str.hex index 19141581..3758ac0d 100644 --- a/proto/_golden/col_low_cardinality_of_str.hex +++ b/proto/_golden/col_low_cardinality_of_str.hex @@ -1,3 +1,3 @@ 00000000 00 06 00 00 00 00 00 00 03 00 00 00 00 00 00 00 |................| 00000010 03 66 6f 6f 03 62 61 72 03 62 61 7a 05 00 00 00 |.foo.bar.baz....| -00000020 00 00 00 00 00 00 00 00 00 |.........| +00000020 00 00 00 00 00 01 00 00 02 |.........| diff --git a/proto/_golden/col_low_cardinality_of_str.raw b/proto/_golden/col_low_cardinality_of_str.raw index c6093d425c6f4d97db13f8608d8b9400d35607ab..c4ca2d99bd6e092757581fe2cea6d92727eb274d 100644 GIT binary patch delta 9 QcmdPYoS@3W$iTn^00*i84*&oF delta 9 OcmdPYoS@3W00aOBqyY>7 diff --git a/proto/col_arr_of.go b/proto/col_arr_of.go index c38055bc..2c31224f 100644 --- a/proto/col_arr_of.go +++ b/proto/col_arr_of.go @@ -59,6 +59,16 @@ func (c *ColArrOf[T]) EncodeState(b *Buffer) { } } +// Prepare ensures Preparable column propagation. +func (c *ColArrOf[T]) Prepare() error { + if v, ok := c.Data.(Preparable); ok { + if err := v.Prepare(); err != nil { + return errors.Wrap(err, "prepare data") + } + } + return nil +} + // RowAppend appends i-th row to target and returns it. func (c ColArrOf[T]) RowAppend(i int, target []T) []T { var start int diff --git a/proto/col_arr_of_test.go b/proto/col_arr_of_test.go index 7aa1c267..2aac42d7 100644 --- a/proto/col_arr_of_test.go +++ b/proto/col_arr_of_test.go @@ -62,7 +62,7 @@ func TestColArrOfStr(t *testing.T) { var buf Buffer col.EncodeColumn(&buf) t.Run("Golden", func(t *testing.T) { - gold.Bytes(t, buf.Buf, "col_col_arr_of_str") + gold.Bytes(t, buf.Buf, "col_arr_of_str") }) t.Run("Ok", func(t *testing.T) { br := bytes.NewReader(buf.Buf) @@ -85,3 +85,37 @@ func TestColArrOfStr(t *testing.T) { requireNoShortRead(t, buf.Buf, colAware(dec, col.Rows())) }) } + +func TestArrOfLowCordStr(t *testing.T) { + col := ArrayOf[string](new(ColStr).LowCardinality()) + col.Append([]string{"foo", "bar", "foo", "foo", "baz"}) + col.Append([]string{"foo", "baz"}) + + require.NoError(t, col.Prepare()) + + var buf Buffer + col.EncodeColumn(&buf) + t.Run("Golden", func(t *testing.T) { + gold.Bytes(t, buf.Buf, "col_arr_of_low_cord_str") + }) + t.Run("Ok", func(t *testing.T) { + br := bytes.NewReader(buf.Buf) + r := NewReader(br) + dec := ArrayOf[string](new(ColStr).LowCardinality()) + + require.NoError(t, dec.DecodeColumn(r, col.Rows())) + require.Equal(t, col.Rows(), dec.Rows()) + require.Equal(t, ColumnType("Array(LowCardinality(String))"), dec.Type()) + require.Equal(t, []string{"foo", "bar", "foo", "foo", "baz"}, dec.Row(0)) + require.Equal(t, []string{"foo", "baz"}, dec.Row(1)) + }) + t.Run("ErrUnexpectedEOF", func(t *testing.T) { + r := NewReader(bytes.NewReader(nil)) + dec := ArrayOf[string](new(ColStr).LowCardinality()) + require.ErrorIs(t, dec.DecodeColumn(r, col.Rows()), io.ErrUnexpectedEOF) + }) + t.Run("NoShortRead", func(t *testing.T) { + dec := ArrayOf[string](new(ColStr).LowCardinality()) + requireNoShortRead(t, buf.Buf, colAware(dec, col.Rows())) + }) +} diff --git a/proto/col_low_cardinality_of.go b/proto/col_low_cardinality_of.go index d66285f4..21ea7bdb 100644 --- a/proto/col_low_cardinality_of.go +++ b/proto/col_low_cardinality_of.go @@ -248,6 +248,7 @@ func (c *ColLowCardinalityOf[T]) Prepare() error { if !ok { c.index.Append(v) c.kv[v] = last + idx = last last++ } c.keys[i] = idx diff --git a/query_test.go b/query_test.go index 631fe206..b879b06d 100644 --- a/query_test.go +++ b/query_test.go @@ -478,6 +478,35 @@ func TestClient_Query(t *testing.T) { require.Len(t, data, 1) require.Equal(t, data, gotData) }) + t.Run("ArrayLowCardinality", func(t *testing.T) { + t.Parallel() + conn := Conn(t) + require.NoError(t, conn.Do(ctx, Query{ + Body: "CREATE TABLE test_table (v Array(LowCardinality(String))) ENGINE = Memory", + }), "create table") + + v := proto.ArrayOf[string](new(proto.ColStr).LowCardinality()) + v.Append([]string{"foo", "bar"}) + v.Append([]string{"baz"}) + + require.NoError(t, conn.Do(ctx, Query{ + Body: "INSERT INTO test_table VALUES", + Input: []proto.InputColumn{ + {Name: "v", Data: v}, + }, + }), "insert") + + gotData := proto.ArrayOf[string](new(proto.ColStr).LowCardinality()) + require.NoError(t, conn.Do(ctx, Query{ + Body: "SELECT * FROM test_table", + Result: proto.Results{ + {Name: "v", Data: gotData}, + }, + }), "select") + + assert.Equal(t, []string{"foo", "bar"}, gotData.Row(0)) + assert.Equal(t, []string{"baz"}, gotData.Row(1)) + }) t.Run("InsertLowCardinalityString", func(t *testing.T) { t.Parallel() conn := Conn(t) From 5bbcb8b95fefc9698cad9c2b4e50aba36d2490be Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Tue, 19 Apr 2022 18:25:46 +0300 Subject: [PATCH 2/2] test(proto): improve LowCardinalityOf test --- proto/col_low_cardinality_of_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/proto/col_low_cardinality_of_test.go b/proto/col_low_cardinality_of_test.go index 5a6e1a74..79c15206 100644 --- a/proto/col_low_cardinality_of_test.go +++ b/proto/col_low_cardinality_of_test.go @@ -5,6 +5,7 @@ import ( "io" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/go-faster/ch/internal/gold" @@ -19,7 +20,8 @@ func TestLowCardinalityOf(t *testing.T) { func TestLowCardinalityOfStr(t *testing.T) { col := (&ColStr{}).LowCardinality() - col.AppendArr([]string{"foo", "bar", "foo", "foo", "baz"}) + v := []string{"foo", "bar", "foo", "foo", "baz"} + col.AppendArr(v) require.NoError(t, col.Prepare()) @@ -35,7 +37,10 @@ func TestLowCardinalityOfStr(t *testing.T) { require.NoError(t, dec.DecodeColumn(r, col.Rows())) require.Equal(t, col.Rows(), dec.Rows()) - require.Equal(t, ColumnType("LowCardinality(String)"), dec.Type()) + for i, s := range v { + assert.Equal(t, s, col.Row(i)) + } + assert.Equal(t, ColumnType("LowCardinality(String)"), dec.Type()) }) t.Run("ErrUnexpectedEOF", func(t *testing.T) { r := NewReader(bytes.NewReader(nil))