Skip to content

Commit

Permalink
feat(proto): total Date, DateTime refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
ernado committed Jun 18, 2022
1 parent 764f00a commit dc5c270
Show file tree
Hide file tree
Showing 124 changed files with 896 additions and 937 deletions.
4 changes: 2 additions & 2 deletions block_fuzz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,9 +188,9 @@ func FuzzDecodeBlockAuto(f *testing.F) {
{},
{100},
}),
proto.ColDateTime64Auto{
proto.ColDateTime64{
Precision: 9,
ColDateTime64: proto.ColDateTime64{
Data: []proto.DateTime64{
1, 2, 3,
},
},
Expand Down
4 changes: 3 additions & 1 deletion cht/cht_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,9 @@ ENGINE = Distributed('nexus', default, hits, rand())`,
{
Name: "EventDate",
Data: proto.ColDateTime{
proto.ToDateTime(time.Now()),
Data: []proto.DateTime{
proto.ToDateTime(time.Now()),
},
},
},
{
Expand Down
8 changes: 4 additions & 4 deletions example_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,25 @@ import (
func ExampleQuery_multipleInputColumns() {
var (
body proto.ColStr
timestamp proto.ColDateTime64
name proto.ColStr
sevText proto.ColEnum
sevNumber proto.ColUInt8

ts = new(proto.ColDateTime64).WithPrecision(proto.PrecisionNano)
arr = new(proto.ColStr).Array() // Array(String)
now = time.Date(2010, 1, 1, 10, 22, 33, 345678, time.UTC)
)
// Append 10 rows.
for i := 0; i < 10; i++ {
body.AppendBytes([]byte("Hello"))
timestamp = append(timestamp, proto.ToDateTime64(now, proto.PrecisionNano))
ts.Append(now)
name.Append("name")
sevText.Values = append(sevText.Values, "INFO")
sevNumber = append(sevNumber, 10)
arr.Append([]string{"foo", "bar", "baz"})
}
input := proto.Input{
{Name: "timestamp", Data: timestamp.Wrap(proto.PrecisionNano)},
{Name: "ts", Data: ts},
{Name: "severity_text", Data: &sevText},
{Name: "severity_number", Data: sevNumber},
{Name: "body", Data: body},
Expand All @@ -38,5 +38,5 @@ func ExampleQuery_multipleInputColumns() {
fmt.Println(input.Into("logs"))

// Output:
// INSERT INTO "logs" ("timestamp","severity_text","severity_number","body","name","arr") VALUES
// INSERT INTO "logs" ("ts","severity_text","severity_number","body","name","arr") VALUES
}
17 changes: 9 additions & 8 deletions internal/cmd/ch-insert-lag/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,13 @@ func main() {
return errors.Wrap(err, "create")
}
close(ready)
data := make(proto.ColDateTime64, 50_000)
data := proto.ColDateTime64{
Data: make([]proto.DateTime64, 50_000),
}
fill := func() {
now := proto.ToDateTime64(time.Now(), precision)
for i := range data {
data[i] = now
for i := range data.Data {
data.Data[i] = now
}
}
fill()
Expand Down Expand Up @@ -89,15 +91,14 @@ func main() {
}); err != nil {
return errors.Wrap(err, "select")
}
if len(data) == 0 {
if data.Rows() == 0 {
continue
}
v := data[0]
if v == 0 {
v := data.Row(0)
if v.IsZero() {
continue
}
latest := v.Time(precision)
lag := time.Since(latest)
lag := time.Since(v)
fmt.Println(lag.Round(time.Millisecond))
}
return nil
Expand Down
2 changes: 1 addition & 1 deletion otel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ type OTELRow struct {

func (t *OTEL) Append(row OTELRow) {
t.Body.AppendBytes(row.Body)
t.Timestamp.Append(proto.DateTime64(row.Timestamp))
t.Timestamp.Append(proto.DateTime64(row.Timestamp).Time(proto.PrecisionNano))
t.SevNumber.Append(row.SeverityNumber)
t.SevText.Append(row.SeverityText)

Expand Down
File renamed without changes.
53 changes: 37 additions & 16 deletions proto/cmd/ch-gen-col/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,27 @@ func (v Variant) ElemLower() string {
return strings.ToLower(v.ElemType())
}

func (v Variant) Complex() bool {
return v.Time()
}

func (v Variant) Time() bool {
switch v.Kind {
case KindDate, KindDateTime:
return true
default:
return false
}
}

func (v Variant) Date() bool {
return v.Kind == KindDate
}

func (v Variant) DateTime() bool {
return v.Kind == KindDateTime
}

func (v Variant) ElemType() string {
if v.Kind == KindEnum {
return fmt.Sprintf("Enum%d", v.Bits)
Expand Down Expand Up @@ -193,20 +214,18 @@ func (v Variant) ElemType() string {
return b.String()
}

//go:embed main.tpl
var mainTemplate string

//go:embed test.tpl
var testTemplate string

//go:embed infer.tpl
var inferTemplate string

//go:embed safe.tpl
var safeTemplate string

//go:embed unsafe.tpl
var unsafeTemplate string
var (
//go:embed main.go.tmpl
mainTemplate string
//go:embed test.go.tmpl
testTemplate string
//go:embed infer.go.tmpl
inferTemplate string
//go:embed safe.go.tmpl
safeTemplate string
//go:embed unsafe.go.tmpl
unsafeTemplate string
)

func write(name string, v interface{}, t *template.Template) error {
out := new(bytes.Buffer)
Expand Down Expand Up @@ -322,8 +341,10 @@ func run() error {
v.GenerateUnsafe = true
}
base := "col_" + v.ElemLower()
if err := write(base+"_gen", v, tpl); err != nil {
return errors.Wrap(err, "write")
if !v.DateTime() {
if err := write(base+"_gen", v, tpl); err != nil {
return errors.Wrap(err, "write")
}
}
if err := write(base+"_safe_gen", v, tplSafe); err != nil {
return errors.Wrap(err, "write")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,22 @@ var (
_ Column = (*{{ .Type }})(nil)
)

// Type returns ColumnType of {{ .Name }}.
func ({{ .Type }}) Type() ColumnType {
return {{ .ColumnType }}
}

// Rows returns count of rows in column.
func (c {{ .Type }}) Rows() int {
return len(c)
}

// Reset resets data in row, preserving capacity for efficiency.
func (c *{{ .Type }}) Reset() {
*c = (*c)[:0]
}

// Type returns ColumnType of {{ .Name }}.
func ({{ .Type }}) Type() ColumnType {
return {{ .ColumnType }}
}

{{ if not .Time }}
// Row returns i-th row of column.
func (c {{ .Type }}) Row(i int) {{ .ElemType }} {
return c[i]
Expand All @@ -33,10 +39,6 @@ func (c *{{ .Type }}) Append(v {{ .ElemType }}) {
*c = append(*c, v)
}

// Reset resets data in row, preserving capacity for efficiency.
func (c *{{ .Type }}) Reset() {
*c = (*c)[:0]
}

// LowCardinality returns LowCardinality for {{ .Name }} .
func (c *{{ .Type }}) LowCardinality() *ColLowCardinality[{{ .ElemType }}] {
Expand Down Expand Up @@ -65,3 +67,5 @@ func NewArr{{ .Name }}() *ColArr[{{ .ElemType }}] {
Data: new({{ .Type }}),
}
}

{{ end }}
35 changes: 24 additions & 11 deletions proto/cmd/ch-gen-col/safe.tpl → proto/cmd/ch-gen-col/safe.go.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,11 @@ func (c *{{ .Type }}) DecodeColumn(r *Reader, rows int) error {
}
*c = v
{{- else }}
{{- if .DateTime }}
v := c.Data
{{- else }}
v := *c
{{- end }}
// Move bound check out of loop.
//
// See https://github.com/golang/go/issues/30945.
Expand All @@ -54,39 +58,48 @@ func (c *{{ .Type }}) DecodeColumn(r *Reader, rows int) error {
{{- else }}
{{ .BinGet }}(data[i : i+size]),
{{- end }}
)
)
}
{{- if .DateTime }}
c.Data = v
{{- else }}
*c = v
{{- end }}
{{- end }}
return nil
}

// EncodeColumn encodes {{ .Name }} rows to *Buffer.
func (c {{ .Type }}) EncodeColumn(b *Buffer) {
if len(c) == 0 {
{{- if .DateTime }}
v := c.Data
{{- else }}
v := c
{{- end }}
if len(v) == 0 {
return
}
{{- if .Byte }}
b.Buf = append(b.Buf, c...)
b.Buf = append(b.Buf, v...)
{{- else if .SingleByte }}
start := len(b.Buf)
b.Buf = append(b.Buf, make([]byte, len(c))...)
for i := range c {
b.Buf[i+start] = {{ .UnsignedType }}(c[i])
b.Buf = append(b.Buf, make([]byte, len(v))...)
for i := range v {
b.Buf[i+start] = {{ .UnsignedType }}(v[i])
}
{{- else }}
const size = {{ .Bits }} / 8
offset := len(b.Buf)
b.Buf = append(b.Buf, make([]byte, size*len(c))...)
for _, v := range c {
b.Buf = append(b.Buf, make([]byte, size*len(v))...)
for _, vv := range v {
{{ .BinPut }}(
b.Buf[offset : offset+size],
{{- if .IsFloat }}
math.{{ .Name }}bits(v),
math.{{ .Name }}bits(vv),
{{- else if .Cast }}
{{ .UnsignedType }}(v),
{{ .UnsignedType }}(vv),
{{- else }}
v,
vv,
{{- end }}
)
offset += size
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,15 @@ func Test{{ .Type }}_DecodeColumn(t *testing.T) {
const rows = 50
var data {{ .Type }}
for i := 0; i < rows; i++ {
{{- if .DateTime }}
data.Data = append(data.Data, {{ .New }}(i))
{{- else if .Date }}
data = append(data, {{ .New }}(i))
{{- else }}
v := {{ .New }}(i)
data.Append(v)
require.Equal(t, v, data.Row(i))
{{- end }}
}

var buf Buffer
Expand All @@ -39,7 +45,9 @@ func Test{{ .Type }}_DecodeColumn(t *testing.T) {
require.Equal(t, rows, dec.Rows())
dec.Reset()
require.Equal(t, 0, dec.Rows())
{{ if not .Time }}
require.Equal(t, {{ .ColumnType }}, dec.Type())
{{ end }}
})
t.Run("ZeroRows", func(t *testing.T) {
r := NewReader(bytes.NewReader(nil))
Expand All @@ -63,6 +71,7 @@ func Test{{ .Type }}_DecodeColumn(t *testing.T) {
})
}

{{- if not .Time }}
func Test{{ .Type }}Array(t *testing.T) {
const rows = 50
data := NewArr{{ .Name }}()
Expand Down Expand Up @@ -98,12 +107,17 @@ func Test{{ .Type }}Array(t *testing.T) {
require.ErrorIs(t, dec.DecodeColumn(r, rows), io.ErrUnexpectedEOF)
})
}
{{ end }}

func Benchmark{{ .Type }}_DecodeColumn(b *testing.B) {
const rows = 1_000
var data {{ .Type }}
for i := 0; i < rows; i++ {
{{- if .DateTime }}
data.Data = append(data.Data, {{ .New }}(i))
{{- else -}}
data = append(data, {{ .New }}(i))
{{- end -}}
}

var buf Buffer
Expand Down Expand Up @@ -135,7 +149,11 @@ func Benchmark{{ .Type }}_EncodeColumn(b *testing.B) {
const rows = 1_000
var data {{ .Type }}
for i := 0; i < rows; i++ {
{{- if .DateTime }}
data.Data = append(data.Data, {{ .New }}(i))
{{- else -}}
data = append(data, {{ .New }}(i))
{{- end -}}
}

var buf Buffer
Expand Down
Loading

0 comments on commit dc5c270

Please sign in to comment.