Merge pull request #131 from ClickHouse/feat/datetime-rework

feat(proto): total Date, DateTime refactor

block_fuzz_test.go

@@ -188,9 +188,9 @@ func FuzzDecodeBlockAuto(f *testing.F) {
 			{},
 			{100},
 		}),
-		proto.ColDateTime64Auto{
+		proto.ColDateTime64{
 			Precision: 9,
-			ColDateTime64: proto.ColDateTime64{
+			Data: []proto.DateTime64{
 				1, 2, 3,
 			},
 		},

cht/cht_test.go

@@ -270,7 +270,9 @@ ENGINE = Distributed('nexus', default, hits, rand())`,
 						{
 							Name: "EventDate",
 							Data: proto.ColDateTime{
-								proto.ToDateTime(time.Now()),
+								Data: []proto.DateTime{
+									proto.ToDateTime(time.Now()),
+								},
 							},
 						},
 						{

example_test.go

@@ -10,25 +10,25 @@ import (
 func ExampleQuery_multipleInputColumns() {
 	var (
 		body      proto.ColStr
-		timestamp proto.ColDateTime64
 		name      proto.ColStr
 		sevText   proto.ColEnum
 		sevNumber proto.ColUInt8
 
+		ts  = new(proto.ColDateTime64).WithPrecision(proto.PrecisionNano)
 		arr = new(proto.ColStr).Array() // Array(String)
 		now = time.Date(2010, 1, 1, 10, 22, 33, 345678, time.UTC)
 	)
 	// Append 10 rows.
 	for i := 0; i < 10; i++ {
 		body.AppendBytes([]byte("Hello"))
-		timestamp = append(timestamp, proto.ToDateTime64(now, proto.PrecisionNano))
+		ts.Append(now)
 		name.Append("name")
 		sevText.Values = append(sevText.Values, "INFO")
 		sevNumber = append(sevNumber, 10)
 		arr.Append([]string{"foo", "bar", "baz"})
 	}
 	input := proto.Input{
-		{Name: "timestamp", Data: timestamp.Wrap(proto.PrecisionNano)},
+		{Name: "ts", Data: ts},
 		{Name: "severity_text", Data: &sevText},
 		{Name: "severity_number", Data: sevNumber},
 		{Name: "body", Data: body},
@@ -38,5 +38,5 @@ func ExampleQuery_multipleInputColumns() {
 	fmt.Println(input.Into("logs"))
 
 	// Output:
-	// INSERT INTO "logs" ("timestamp","severity_text","severity_number","body","name","arr") VALUES
+	// INSERT INTO "logs" ("ts","severity_text","severity_number","body","name","arr") VALUES
 }

internal/cmd/ch-insert-lag/main.go

@@ -38,11 +38,13 @@ func main() {
 				return errors.Wrap(err, "create")
 			}
 			close(ready)
-			data := make(proto.ColDateTime64, 50_000)
+			data := proto.ColDateTime64{
+				Data: make([]proto.DateTime64, 50_000),
+			}
 			fill := func() {
 				now := proto.ToDateTime64(time.Now(), precision)
-				for i := range data {
-					data[i] = now
+				for i := range data.Data {
+					data.Data[i] = now
 				}
 			}
 			fill()
@@ -89,15 +91,14 @@ func main() {
 				}); err != nil {
 					return errors.Wrap(err, "select")
 				}
-				if len(data) == 0 {
+				if data.Rows() == 0 {
 					continue
 				}
-				v := data[0]
-				if v == 0 {
+				v := data.Row(0)
+				if v.IsZero() {
 					continue
 				}
-				latest := v.Time(precision)
-				lag := time.Since(latest)
+				lag := time.Since(v)
 				fmt.Println(lag.Round(time.Millisecond))
 			}
 			return nil

otel_test.go

@@ -91,7 +91,7 @@ type OTELRow struct {
 
 func (t *OTEL) Append(row OTELRow) {
 	t.Body.AppendBytes(row.Body)
-	t.Timestamp.Append(proto.DateTime64(row.Timestamp))
+	t.Timestamp.Append(proto.DateTime64(row.Timestamp).Time(proto.PrecisionNano))
 	t.SevNumber.Append(row.SeverityNumber)
 	t.SevText.Append(row.SeverityText)
 

proto/cmd/ch-gen-col/main.go

@@ -146,6 +146,27 @@ func (v Variant) ElemLower() string {
 	return strings.ToLower(v.ElemType())
 }
 
+func (v Variant) Complex() bool {
+	return v.Time()
+}
+
+func (v Variant) Time() bool {
+	switch v.Kind {
+	case KindDate, KindDateTime:
+		return true
+	default:
+		return false
+	}
+}
+
+func (v Variant) Date() bool {
+	return v.Kind == KindDate
+}
+
+func (v Variant) DateTime() bool {
+	return v.Kind == KindDateTime
+}
+
 func (v Variant) ElemType() string {
 	if v.Kind == KindEnum {
 		return fmt.Sprintf("Enum%d", v.Bits)
@@ -193,20 +214,18 @@ func (v Variant) ElemType() string {
 	return b.String()
 }
 
-//go:embed main.tpl
-var mainTemplate string
-
-//go:embed test.tpl
-var testTemplate string
-
-//go:embed infer.tpl
-var inferTemplate string
-
-//go:embed safe.tpl
-var safeTemplate string
-
-//go:embed unsafe.tpl
-var unsafeTemplate string
+var (
+	//go:embed main.go.tmpl
+	mainTemplate string
+	//go:embed test.go.tmpl
+	testTemplate string
+	//go:embed infer.go.tmpl
+	inferTemplate string
+	//go:embed safe.go.tmpl
+	safeTemplate string
+	//go:embed unsafe.go.tmpl
+	unsafeTemplate string
+)
 
 func write(name string, v interface{}, t *template.Template) error {
 	out := new(bytes.Buffer)
@@ -322,8 +341,10 @@ func run() error {
 			v.GenerateUnsafe = true
 		}
 		base := "col_" + v.ElemLower()
-		if err := write(base+"_gen", v, tpl); err != nil {
-			return errors.Wrap(err, "write")
+		if !v.DateTime() {
+			if err := write(base+"_gen", v, tpl); err != nil {
+				return errors.Wrap(err, "write")
+			}
 		}
 		if err := write(base+"_safe_gen", v, tplSafe); err != nil {
 			return errors.Wrap(err, "write")

proto/cmd/ch-gen-col/main.tpl → proto/cmd/ch-gen-col/main.go.tmpl

@@ -13,16 +13,22 @@ var (
 	_ Column    = (*{{ .Type }})(nil)
 )
 
-// Type returns ColumnType of {{ .Name }}.
-func ({{ .Type }}) Type() ColumnType {
-	return {{ .ColumnType }}
-}
-
 // Rows returns count of rows in column.
 func (c {{ .Type }}) Rows() int {
 	return len(c)
 }
 
+// Reset resets data in row, preserving capacity for efficiency.
+func (c *{{ .Type }}) Reset() {
+	*c = (*c)[:0]
+}
+
+// Type returns ColumnType of {{ .Name }}.
+func ({{ .Type }}) Type() ColumnType {
+	return {{ .ColumnType }}
+}
+
+{{ if not .Time }}
 // Row returns i-th row of column.
 func (c {{ .Type }}) Row(i int) {{ .ElemType }} {
 	return c[i]
@@ -33,10 +39,6 @@ func (c *{{ .Type }}) Append(v {{ .ElemType }})  {
 	*c = append(*c, v)
 }
 
-// Reset resets data in row, preserving capacity for efficiency.
-func (c *{{ .Type }}) Reset() {
-	*c = (*c)[:0]
-}
 
 // LowCardinality returns LowCardinality for {{ .Name }} .
 func (c *{{ .Type }}) LowCardinality() *ColLowCardinality[{{ .ElemType }}] {
@@ -65,3 +67,5 @@ func NewArr{{ .Name }}() *ColArr[{{ .ElemType }}] {
 		Data: new({{ .Type }}),
 	}
 }
+
+{{ end }}

proto/cmd/ch-gen-col/safe.tpl → proto/cmd/ch-gen-col/safe.go.tmpl

@@ -40,7 +40,11 @@ func (c *{{ .Type }}) DecodeColumn(r *Reader, rows int) error {
 	}
 	*c = v
 	{{- else }}
+	{{- if .DateTime }}
+	v := c.Data
+	{{- else }}
 	v := *c
+	{{- end }}
 	// Move bound check out of loop.
 	//
 	// See https://github.com/golang/go/issues/30945.
@@ -54,39 +58,48 @@ func (c *{{ .Type }}) DecodeColumn(r *Reader, rows int) error {
 		{{- else }}
 			{{ .BinGet }}(data[i : i+size]),
 		{{- end }}
-	)
+		)
 	}
+	{{- if .DateTime }}
+	c.Data = v
+	{{- else }}
 	*c = v
 	{{- end }}
+	{{- end }}
 	return nil
 }
 
 // EncodeColumn encodes {{ .Name }} rows to *Buffer.
 func (c {{ .Type }}) EncodeColumn(b *Buffer) {
-	if len(c) == 0 {
+	{{- if .DateTime }}
+	v := c.Data
+	{{- else }}
+	v := c
+	{{- end }}
+	if len(v) == 0 {
 		return
 	}
 	{{- if .Byte }}
-	b.Buf = append(b.Buf, c...)
+	b.Buf = append(b.Buf, v...)
 	{{- else if .SingleByte }}
 	start := len(b.Buf)
-	b.Buf = append(b.Buf, make([]byte, len(c))...)
-	for i := range c {
-		b.Buf[i+start] = {{ .UnsignedType }}(c[i])
+	b.Buf = append(b.Buf, make([]byte, len(v))...)
+	for i := range v {
+		b.Buf[i+start] = {{ .UnsignedType }}(v[i])
 	}
 	{{- else }}
 	const size = {{ .Bits }} / 8
 	offset := len(b.Buf)
-	b.Buf = append(b.Buf, make([]byte, size*len(c))...)
-	for _, v := range c {
+	b.Buf = append(b.Buf, make([]byte, size*len(v))...)
+	for _, vv := range v {
 		{{ .BinPut }}(
 			b.Buf[offset : offset+size],
 		{{- if .IsFloat }}
-			math.{{ .Name }}bits(v),
+			math.{{ .Name }}bits(vv),
 		{{- else if .Cast }}
-			{{ .UnsignedType }}(v),
+			{{ .UnsignedType }}(vv),
 		{{- else }}
-			v,
+			vv,
 		{{- end }}
 	)
 		offset += size

proto/cmd/ch-gen-col/test.tpl → proto/cmd/ch-gen-col/test.go.tmpl

@@ -18,9 +18,15 @@ func Test{{ .Type }}_DecodeColumn(t *testing.T) {
 	const rows = 50
 	var data {{ .Type }}
 	for i := 0; i < rows; i++ {
+		{{- if .DateTime }}
+		data.Data = append(data.Data, {{ .New }}(i))
+		{{- else if .Date }}
+		data = append(data, {{ .New }}(i))
+		{{- else }}
 		v := {{ .New }}(i)
 		data.Append(v)
 		require.Equal(t, v, data.Row(i))
+		{{- end }}
 	}
 
 	var buf Buffer
@@ -39,7 +45,9 @@ func Test{{ .Type }}_DecodeColumn(t *testing.T) {
 		require.Equal(t, rows, dec.Rows())
 		dec.Reset()
 		require.Equal(t, 0, dec.Rows())
+		{{ if not .Time }}
 		require.Equal(t, {{ .ColumnType }}, dec.Type())
+		{{ end }}
 	})
 	t.Run("ZeroRows", func(t *testing.T) {
 		r := NewReader(bytes.NewReader(nil))
@@ -63,6 +71,7 @@ func Test{{ .Type }}_DecodeColumn(t *testing.T) {
 	})
 }
 
+{{- if not .Time }}
 func Test{{ .Type }}Array(t *testing.T) {
 	const rows = 50
 	data := NewArr{{ .Name }}()
@@ -98,12 +107,17 @@ func Test{{ .Type }}Array(t *testing.T) {
 		require.ErrorIs(t, dec.DecodeColumn(r, rows), io.ErrUnexpectedEOF)
 	})
 }
+{{ end }}
 
 func Benchmark{{ .Type }}_DecodeColumn(b *testing.B) {
 	const rows = 1_000
 	var data {{ .Type }}
 	for i := 0; i < rows; i++ {
+		{{- if .DateTime }}
+		data.Data = append(data.Data, {{ .New }}(i))
+		{{- else -}}
 		data = append(data, {{ .New }}(i))
+		{{- end -}}
 	}
 
 	var buf Buffer
@@ -135,7 +149,11 @@ func Benchmark{{ .Type }}_EncodeColumn(b *testing.B) {
 	const rows = 1_000
 	var data {{ .Type }}
 	for i := 0; i < rows; i++ {
+		{{- if .DateTime }}
+		data.Data = append(data.Data, {{ .New }}(i))
+		{{- else -}}
 		data = append(data, {{ .New }}(i))
+		{{- end -}}
 	}
 
 	var buf Buffer