diff --git a/go.mod b/go.mod index 492f2096a..d92ccf3ad 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/goccy/go-json v0.10.0 github.com/goccy/go-yaml v1.9.5 github.com/goccy/go-zetasql v0.5.5 - github.com/goccy/go-zetasqlite v0.19.0 + github.com/goccy/go-zetasqlite v0.19.1 github.com/google/go-cmp v0.5.9 github.com/googleapis/gax-go/v2 v2.11.0 github.com/gorilla/mux v1.8.0 @@ -75,11 +75,11 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.16.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index b5e3e93b4..327d1dcc3 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0= github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/goccy/go-zetasql v0.5.5 h1:3JpXt3p2533rnZMu09upCcI7YJ2KfjKgdo2Lu0xo+fU= github.com/goccy/go-zetasql v0.5.5/go.mod h1:xvvooX2RG404vnbdFZuAM8bTFksYwVUlqeIUrUNuo40= -github.com/goccy/go-zetasqlite v0.19.0 h1:pvWFw/QThPP5a3UMkgirIsb9/5tlfUq25DggFfhI86o= -github.com/goccy/go-zetasqlite v0.19.0/go.mod h1:ThavIQcmI6a/sVTpvJtj+idUui32gwbnXe1jeb7pISY= +github.com/goccy/go-zetasqlite v0.19.1 h1:yebtcyHgSNnNywan6FbI93uxJ6GXDpVMoqTlfT/Bdm4= +github.com/goccy/go-zetasqlite v0.19.1/go.mod h1:ThavIQcmI6a/sVTpvJtj+idUui32gwbnXe1jeb7pISY= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -250,8 +250,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= @@ -278,8 +278,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -314,8 +314,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/contentdata/repository.go b/internal/contentdata/repository.go index f737f949b..7be8c3bc7 100644 --- a/internal/contentdata/repository.go +++ b/internal/contentdata/repository.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "reflect" - "sort" "strings" "github.com/goccy/go-zetasqlite" @@ -371,34 +370,58 @@ func (r *Repository) AddTableData(ctx context.Context, tx *connection.Tx, projec _ = tx.MetadataRepoMode() }() - var columns []string + var columns []*types.Column for _, col := range table.Columns { - columns = append(columns, col.Name) - } - sort.Strings(columns) - rows := make([]string, 0, len(table.Data)) - values := make([]interface{}, 0, len(table.Data)*len(columns)) - for _, data := range table.Data { - placeholders := make([]string, 0, len(data)) - for _, col := range columns { - values = append(values, data[col]) - placeholders = append(placeholders, "?") - } - rows = append(rows, fmt.Sprintf("(%s)", strings.Join(placeholders, ","))) + columns = append(columns, col) } + + placeholders := make([]string, 0, len(columns)) columnsWithEscape := make([]string, 0, len(columns)) for _, col := range columns { - columnsWithEscape = append(columnsWithEscape, fmt.Sprintf("`%s`", col)) + placeholders = append(placeholders, "?") + columnsWithEscape = append(columnsWithEscape, fmt.Sprintf("`%s`", col.Name)) } + query := fmt.Sprintf( - "INSERT `%s` (%s) VALUES %s", + "INSERT `%s` (%s) VALUES (%s)", r.tablePath(projectID, datasetID, table.ID), strings.Join(columnsWithEscape, ","), - strings.Join(rows, ","), + strings.Join(placeholders, ","), ) - if _, err := tx.Tx().ExecContext(ctx, query, values...); err != nil { + + stmt, err := tx.Tx().PrepareContext(ctx, query) + if err != nil { return err } + + for _, data := range table.Data { + values := make([]interface{}, 0, len(table.Columns)) + + for _, column := range columns { + if value, found := data[column.Name]; found { + isTimestampColumn := column.Type == types.TIMESTAMP + inputString, isInputString := value.(string) + + if isInputString && isTimestampColumn { + parsedTimestamp, err := zetasqlite.TimeFromTimestampValue(inputString) + // If we could parse the timestamp, use it when inserting, otherwise fallback to the supplied value + if err == nil { + values = append(values, parsedTimestamp) + continue + } + } + + values = append(values, value) + } else { + values = append(values, nil) + } + } + + if _, err := stmt.ExecContext(ctx, values...); err != nil { + return err + } + } + return nil } diff --git a/internal/types/types.go b/internal/types/types.go index 471442920..ba8bef6cc 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -192,7 +192,7 @@ func Format(schema *bigqueryv2.TableSchema, rows []*TableRow, useInt64Timestamp for _, row := range rows { cells := make([]*TableCell, 0, len(row.F)) for colIdx, cell := range row.F { - if schema.Fields[colIdx].Type == "TIMESTAMP" { + if schema.Fields[colIdx].Type == "TIMESTAMP" && cell.V != nil { t, _ := zetasqlite.TimeFromTimestampValue(cell.V.(string)) microsec := t.UnixNano() / int64(time.Microsecond) cells = append(cells, &TableCell{