From edce9eb90a0fdf446d1bf62848d93cf1748b4a65 Mon Sep 17 00:00:00 2001 From: Aris Tzoumas Date: Tue, 20 Feb 2024 18:01:33 +0200 Subject: [PATCH] fixup! feat: sqlconnect library --- sqlconnect/internal/bigquery/mappings.go | 4 +- .../testdata/column-mapping-test-columns.json | 36 +++++ .../testdata/column-mapping-test-rows.json | 27 ++++ .../testdata/column-mapping-test-seed.sql | 17 ++- sqlconnect/internal/databricks/db.go | 2 +- sqlconnect/internal/databricks/mappings.go | 71 +++++++-- .../testdata/column-mapping-test-columns.json | 98 +++++++++++++ .../testdata/column-mapping-test-rows.json | 83 +++++++++++ .../testdata/column-mapping-test-seed.sql | 33 +++++ .../db_integration_test_scenario.go | 4 - sqlconnect/internal/mysql/mappings.go | 136 ++++++++---------- .../testdata/column-mapping-test-columns.json | 130 +++++++++++++++++ .../testdata/column-mapping-test-rows.json | 104 ++++++++++++++ .../testdata/column-mapping-test-seed.sql | 41 ++++++ sqlconnect/internal/postgres/mappings.go | 13 +- .../testdata/column-mapping-test-columns.json | 114 +++++++++++++++ .../testdata/column-mapping-test-rows.json | 96 +++++++++++++ .../testdata/column-mapping-test-seed.sql | 37 +++++ sqlconnect/internal/redshift/db.go | 1 + sqlconnect/internal/redshift/mappings.go | 26 ++++ .../testdata/column-mapping-test-columns.json | 106 ++++++++++++++ .../testdata/column-mapping-test-rows.json | 86 +++++++++++ .../testdata/column-mapping-test-seed.sql | 35 +++++ sqlconnect/internal/snowflake/mappings.go | 56 ++++---- .../testdata/column-mapping-test-columns.json | 134 +++++++++++++++++ .../testdata/column-mapping-test-rows.json | 107 ++++++++++++++ .../testdata/column-mapping-test-seed.sql | 51 +++++++ sqlconnect/internal/trino/mappings.go | 54 ++++--- .../testdata/column-mapping-test-columns.json | 70 +++++++++ .../testdata/column-mapping-test-rows.json | 69 +++++++++ .../testdata/column-mapping-test-seed.sql | 26 ++++ 31 files changed, 1725 insertions(+), 142 deletions(-) create mode 100644 sqlconnect/internal/databricks/testdata/column-mapping-test-columns.json create mode 100644 sqlconnect/internal/databricks/testdata/column-mapping-test-rows.json create mode 100644 sqlconnect/internal/databricks/testdata/column-mapping-test-seed.sql create mode 100644 sqlconnect/internal/mysql/testdata/column-mapping-test-columns.json create mode 100644 sqlconnect/internal/mysql/testdata/column-mapping-test-rows.json create mode 100644 sqlconnect/internal/mysql/testdata/column-mapping-test-seed.sql create mode 100644 sqlconnect/internal/postgres/testdata/column-mapping-test-columns.json create mode 100644 sqlconnect/internal/postgres/testdata/column-mapping-test-rows.json create mode 100644 sqlconnect/internal/postgres/testdata/column-mapping-test-seed.sql create mode 100644 sqlconnect/internal/redshift/testdata/column-mapping-test-columns.json create mode 100644 sqlconnect/internal/redshift/testdata/column-mapping-test-rows.json create mode 100644 sqlconnect/internal/redshift/testdata/column-mapping-test-seed.sql create mode 100644 sqlconnect/internal/snowflake/testdata/column-mapping-test-columns.json create mode 100644 sqlconnect/internal/snowflake/testdata/column-mapping-test-rows.json create mode 100644 sqlconnect/internal/snowflake/testdata/column-mapping-test-seed.sql create mode 100644 sqlconnect/internal/trino/testdata/column-mapping-test-columns.json create mode 100644 sqlconnect/internal/trino/testdata/column-mapping-test-rows.json create mode 100644 sqlconnect/internal/trino/testdata/column-mapping-test-seed.sql diff --git a/sqlconnect/internal/bigquery/mappings.go b/sqlconnect/internal/bigquery/mappings.go index abe6bc7..f491e00 100644 --- a/sqlconnect/internal/bigquery/mappings.go +++ b/sqlconnect/internal/bigquery/mappings.go @@ -7,6 +7,7 @@ import ( "strings" "cloud.google.com/go/bigquery" + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" ) @@ -49,7 +50,8 @@ var columnTypeMappings = map[string]string{ "RECORD": "json", } -var re = regexp.MustCompile(`(\([^)]+\)|<[^>]+>)`) +// var re = regexp.MustCompile(`(\([^)]+\)|<[^>]+>)`) // remove type parameters [<>] and size constraints [()] +var re = regexp.MustCompile(`(\(.+\)|<.+>)`) // remove type parameters [<>] and size constraints [()] func columnTypeMapper(columnType base.ColumnType) string { databaseTypeName := strings.ToUpper(re.ReplaceAllString(columnType.DatabaseTypeName(), "")) diff --git a/sqlconnect/internal/bigquery/testdata/column-mapping-test-columns.json b/sqlconnect/internal/bigquery/testdata/column-mapping-test-columns.json index e4d3d3f..d7637b7 100644 --- a/sqlconnect/internal/bigquery/testdata/column-mapping-test-columns.json +++ b/sqlconnect/internal/bigquery/testdata/column-mapping-test-columns.json @@ -15,10 +15,18 @@ "name": "_bignumericnoscale", "type": "float" }, + { + "name": "_bigdecimal", + "type": "float" + }, { "name": "_bool", "type": "boolean" }, + { + "name": "_boolean", + "type": "boolean" + }, { "name": "_bytes", "type": "string" @@ -43,6 +51,30 @@ "name": "_int64", "type": "int" }, + { + "name": "_int", + "type": "int" + }, + { + "name": "_smallint", + "type": "int" + }, + { + "name": "_integer", + "type": "int" + }, + { + "name": "_bigint", + "type": "int" + }, + { + "name": "_tinyint", + "type": "int" + }, + { + "name": "_byteint", + "type": "int" + }, { "name": "_interval", "type": "int" @@ -55,6 +87,10 @@ "name": "_numeric", "type": "float" }, + { + "name": "_decimal", + "type": "float" + }, { "name": "_string", "type": "string" diff --git a/sqlconnect/internal/bigquery/testdata/column-mapping-test-rows.json b/sqlconnect/internal/bigquery/testdata/column-mapping-test-rows.json index cbe76d1..eab80bb 100644 --- a/sqlconnect/internal/bigquery/testdata/column-mapping-test-rows.json +++ b/sqlconnect/internal/bigquery/testdata/column-mapping-test-rows.json @@ -4,16 +4,25 @@ "_array": ["ONE"], "_bignumeric": 1.1, "_bignumericnoscale": 1, + "_bigdecimal": 1, "_bool": true, + "_boolean": true, "_bytes": "abc", "_date": "2014-09-27", "_datetime": "2014-09-27T12:30:00.450000000", "_float64": 1.1, "_geo": "POINT(32 90)", "_int64": 1, + "_int": 1, + "_smallint": 1, + "_integer": 1, + "_bigint": 1, + "_tinyint": 1, + "_byteint": 1, "_interval": 31104000000000000, "_json": {"key": "value"}, "_numeric": 1, + "_decimal": 1, "_string": "string", "_struct": ["string", 1], "_time": "12:30:00.450000000", @@ -24,16 +33,25 @@ "_array": null, "_bignumeric": 0, "_bignumericnoscale": 0, + "_bigdecimal": 0, "_bool": false, + "_boolean": false, "_bytes": "", "_date": "2014-09-27", "_datetime": "2014-09-27T12:30:00.450000000", "_float64": 0, "_geo": "GEOMETRYCOLLECTION EMPTY", "_int64": 0, + "_int": 0, + "_smallint": 0, + "_integer": 0, + "_bigint": 0, + "_tinyint": 0, + "_byteint": 0, "_interval": 31104000000000000, "_json": {}, "_numeric": 0, + "_decimal": 0, "_string": "", "_struct": ["",0], "_time": "12:30:00.450000000", @@ -44,16 +62,25 @@ "_array": null, "_bignumeric": null, "_bignumericnoscale": null, + "_bigdecimal": null, "_bool": null, + "_boolean": null, "_bytes": null, "_date": null, "_datetime": null, "_float64": null, "_geo": null, "_int64": null, + "_int": null, + "_smallint": null, + "_integer": null, + "_bigint": null, + "_tinyint": null, + "_byteint": null, "_interval": null, "_json": null, "_numeric": null, + "_decimal": null, "_string": null, "_struct": null, "_time": null, diff --git a/sqlconnect/internal/bigquery/testdata/column-mapping-test-seed.sql b/sqlconnect/internal/bigquery/testdata/column-mapping-test-seed.sql index 27f9532..44a0479 100644 --- a/sqlconnect/internal/bigquery/testdata/column-mapping-test-seed.sql +++ b/sqlconnect/internal/bigquery/testdata/column-mapping-test-seed.sql @@ -3,16 +3,25 @@ CREATE TABLE `{{.schema}}`.`column_mappings_test` ( _array ARRAY, _bignumeric BIGNUMERIC(2,1), _bignumericnoscale BIGNUMERIC(1,0), + _bigdecimal BIGDECIMAL, _bool BOOL, + _boolean BOOLEAN, _bytes BYTES, _date DATE, _datetime DATETIME, _float64 FLOAT64, _geo GEOGRAPHY, _int64 INT64, + _int INT, + _smallint SMALLINT, + _integer INTEGER, + _bigint BIGINT, + _tinyint TINYINT, + _byteint BYTEINT, _interval INTERVAL, _json JSON, _numeric NUMERIC, + _decimal NUMERIC, _string STRING(10), _struct STRUCT, _time TIME, @@ -20,8 +29,8 @@ CREATE TABLE `{{.schema}}`.`column_mappings_test` ( ); INSERT INTO `{{.schema}}`.`column_mappings_test` - (_order, _array, _bignumeric, _bignumericnoscale, _bool, _bytes, _date, _datetime, _float64, _geo, _int64, _interval, _json, _numeric, _string, _struct, _time, _timestamp) + (_order, _array, _bignumeric, _bignumericnoscale, _bigdecimal, _bool, _boolean, _bytes, _date, _datetime, _float64, _geo, _int64, _int, _smallint, _integer, _bigint, _tinyint, _byteint, _interval, _json, _numeric, _decimal, _string, _struct, _time, _timestamp) VALUES - (1, ['ONE'], 1.1, 1, TRUE, B"abc", '2014-09-27', '2014-09-27 12:30:00.45', 1.1, ST_GEOGFROMTEXT('POINT(32 90)'), 1, INTERVAL 1 YEAR, JSON '{"key": "value"}', 1, 'string', ('string', 1), '12:30:00.45', '2014-09-27 12:30:00.45-08'), - (2, [], 0.0, 0, FALSE, B"", '2014-09-27', '2014-09-27 12:30:00.45', 0.0, ST_GEOGFROMTEXT('POINT EMPTY'), 0, INTERVAL 1 YEAR, JSON '{}', 0, '', ('', 0), '12:30:00.45', '2014-09-27 12:30:00.45-08'), - (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); \ No newline at end of file + (1, ['ONE'], 1.1, 1, 1, TRUE, TRUE, B"abc", '2014-09-27', '2014-09-27 12:30:00.45', 1.1, ST_GEOGFROMTEXT('POINT(32 90)'), 1, 1, 1, 1, 1, 1, 1, INTERVAL 1 YEAR, JSON '{"key": "value"}', 1, 1, 'string', ('string', 1), '12:30:00.45', '2014-09-27 12:30:00.45-08'), + (2, [], 0.0, 0, 0, FALSE, FALSE, B"", '2014-09-27', '2014-09-27 12:30:00.45', 0.0, ST_GEOGFROMTEXT('POINT EMPTY'), 0, 0, 0, 0, 0, 0, 0, INTERVAL 1 YEAR, JSON '{}', 0, 0, '', ('', 0), '12:30:00.45', '2014-09-27 12:30:00.45-08'), + (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); \ No newline at end of file diff --git a/sqlconnect/internal/databricks/db.go b/sqlconnect/internal/databricks/db.go index d685bd5..7681428 100644 --- a/sqlconnect/internal/databricks/db.go +++ b/sqlconnect/internal/databricks/db.go @@ -53,7 +53,7 @@ func NewDB(configJson json.RawMessage) (*DB, error) { db, lo.Ternary(config.RudderSchema != "", config.RudderSchema, defaultRudderSchema), base.WithDialect(dialect{}), - base.WithColumnTypeMappings(columnTypeMappings), + base.WithColumnTypeMapper(columnTypeMapper), base.WithJsonRowMapper(jsonRowMapper), base.WithSQLCommandsOverride(func(cmds base.SQLCommands) base.SQLCommands { cmds.ListSchemas = func() (string, string) { return "SHOW SCHEMAS", "schema_name" } diff --git a/sqlconnect/internal/databricks/mappings.go b/sqlconnect/internal/databricks/mappings.go index 6681d24..11047bb 100644 --- a/sqlconnect/internal/databricks/mappings.go +++ b/sqlconnect/internal/databricks/mappings.go @@ -1,15 +1,35 @@ package databricks +import ( + "encoding/json" + "regexp" + "strconv" + "strings" + + "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" +) + // mapping of database column types to rudder types var columnTypeMappings = map[string]string{ - "DECIMAL": "int", - "NUMERIC": "int", - "DEC": "int", - "INT": "int", - "BIGINT": "int", - "SMALLINT": "int", - "TINYINT": "int", - "FLOAT": "float", + "DECIMAL": "int", // DECIMAL and aliases + "NUMERIC": "int", + "DEC": "int", + + "INT": "int", // INT and aliases + "INTEGER": "int", + + "BIGINT": "int", // BIGINT and aliases + "LONG": "int", + + "SMALLINT": "int", // SMALLINT and aliases + "SHORT": "int", + + "TINYINT": "int", // TINYINT and aliases + "BYTE": "int", + + "FLOAT": "float", // FLOAT and aliases + "REAL": "float", + "DOUBLE": "float", "BOOLEAN": "boolean", "STRING": "string", @@ -19,9 +39,21 @@ var columnTypeMappings = map[string]string{ "VOID": "string", "TIMESTAMP": "datetime", "TIMESTAMP_NTZ": "datetime", - "ARRAY": "json", - "MAP": "json", - "STRUCT": "json", + + "ARRAY": "json", + "MAP": "json", + "STRUCT": "json", +} + +// var re = regexp.MustCompile(`(\([^)]+\)|<[^>]+>)`) // remove type parameters [<>] and size constraints [()] +var re = regexp.MustCompile(`(\(.+\)|<.+>)`) // remove type parameters [<>] and size constraints [()] + +func columnTypeMapper(columnType base.ColumnType) string { + databaseTypeName := strings.ToUpper(re.ReplaceAllString(columnType.DatabaseTypeName(), "")) + if mappedType, ok := columnTypeMappings[strings.ToUpper(databaseTypeName)]; ok { + return mappedType + } + return databaseTypeName } // jsonRowMapper maps a row's scanned column to a json object's field @@ -29,6 +61,23 @@ func jsonRowMapper(databaseTypeName string, value any) any { switch v := value.(type) { case []uint8: return string(v) + case string: + switch databaseTypeName { + case "DECIMAL": + // convert to float + f, err := strconv.ParseFloat(v, 64) + if err != nil { + return v + } + return f + case "ARRAY", "STRUCT", "MAP": // convert string to json + var j any + err := json.Unmarshal([]byte(v), &j) + if err != nil { + return v + } + return j + } } return value } diff --git a/sqlconnect/internal/databricks/testdata/column-mapping-test-columns.json b/sqlconnect/internal/databricks/testdata/column-mapping-test-columns.json new file mode 100644 index 0000000..56354e7 --- /dev/null +++ b/sqlconnect/internal/databricks/testdata/column-mapping-test-columns.json @@ -0,0 +1,98 @@ +[ + { + "name": "_order", + "type": "int" + }, + { + "name": "_decimal", + "type": "int" + }, + { + "name": "_numeric", + "type": "int" + }, + { + "name": "_dec", + "type": "int" + }, + { + "name": "_int", + "type": "int" + }, + { + "name": "_integer", + "type": "int" + }, + { + "name": "_bigint", + "type": "int" + }, + { + "name": "_long", + "type": "int" + }, + { + "name": "_smallint", + "type": "int" + }, + { + "name": "_short", + "type": "int" + }, + { + "name": "_tinyint", + "type": "int" + }, + { + "name": "_byte", + "type": "int" + }, + { + "name": "_float", + "type": "float" + }, + { + "name": "_real", + "type": "float" + }, + { + "name": "_double", + "type": "float" + }, + { + "name": "_boolean", + "type": "boolean" + }, + { + "name": "_string", + "type": "string" + }, + { + "name": "_binary", + "type": "string" + }, + { + "name": "_date", + "type": "datetime" + }, + { + "name": "_timestamp", + "type": "datetime" + }, + { + "name": "_timestampntz", + "type": "datetime" + }, + { + "name": "_array", + "type": "json" + }, + { + "name": "_map", + "type": "json" + }, + { + "name": "_struct", + "type": "json" + } +] \ No newline at end of file diff --git a/sqlconnect/internal/databricks/testdata/column-mapping-test-rows.json b/sqlconnect/internal/databricks/testdata/column-mapping-test-rows.json new file mode 100644 index 0000000..308489e --- /dev/null +++ b/sqlconnect/internal/databricks/testdata/column-mapping-test-rows.json @@ -0,0 +1,83 @@ +[ + { + "_order": 1, + "_decimal": 1.1, + "_numeric": 1, + "_dec": 1, + "_int": 1, + "_integer": 1, + "_bigint": 1, + "_long": 1, + "_smallint": 1, + "_short": 1, + "_tinyint": 1, + "_byte": 1, + "_float": 1.1, + "_real": 1.1, + "_double": 1.1, + "_boolean": true, + "_string": "s", + "_binary": "\u0001", + "_date": "2020-12-31T00:00:00Z", + "_timestamp": "2021-07-01T05:43:28Z", + "_timestampntz": "2021-07-01T08:43:28.123456Z", + "_array": [1,2,3], + "_map": {"key":"value"}, + "_struct": {"col1": "val1", "col2": 1} + }, + { + "_order": 2, + "_decimal": 0, + "_numeric": 0, + "_dec": 0, + "_int": 0, + "_integer": 0, + "_bigint": 0, + "_long": 0, + "_smallint": 0, + "_short": 0, + "_tinyint": 0, + "_byte": 0, + "_float": 0, + "_real": 0, + "_double": 0, + "_boolean": false, + "_string": "", + "_binary": "", + "_date": "2020-12-31T00:00:00Z", + "_timestamp": "2021-07-01T05:43:28Z", + "_timestampntz": "2021-07-01T08:43:28.123456Z", + "_array": [], + "_map": {"": ""}, + "_struct": {"col1": "", "col2": 0} + }, + { + "_order": 3, + "_decimal": null, + "_numeric": null, + "_dec": null, + "_int": null, + "_integer": null, + "_bigint": null, + "_long": null, + "_smallint": null, + "_short": null, + "_tinyint": null, + "_byte": null, + "_float": null, + "_real": null, + "_double": null, + "_boolean": null, + "_string": null, + "_binary": null, + "_date": null, + "_timestamp": null, + "_timestampntz": null, + "_array": null, + "_map": null, + "_struct": { + "col1": null, + "col2": null + } + } +] \ No newline at end of file diff --git a/sqlconnect/internal/databricks/testdata/column-mapping-test-seed.sql b/sqlconnect/internal/databricks/testdata/column-mapping-test-seed.sql new file mode 100644 index 0000000..9635a70 --- /dev/null +++ b/sqlconnect/internal/databricks/testdata/column-mapping-test-seed.sql @@ -0,0 +1,33 @@ +CREATE TABLE `{{.schema}}`.`column_mappings_test` ( + _order INT, + _decimal DECIMAL(2,1), + _numeric NUMERIC, + _dec DEC, + _int INT, + _integer INTEGER, + _bigint BIGINT, + _long LONG, + _smallint SMALLINT, + _short SHORT, + _tinyint TINYINT, + _byte BYTE, + _float FLOAT, + _real REAL, + _double DOUBLE, + _boolean BOOLEAN, + _string STRING, + _binary BINARY, + _date DATE, + _timestamp TIMESTAMP, + _timestampntz TIMESTAMP_NTZ, + _array ARRAY, + _map MAP, + _struct STRUCT +); + +INSERT INTO `{{.schema}}`.`column_mappings_test` + (_order, _decimal, _numeric, _dec, _int, _integer, _bigint, _long, _smallint, _short, _tinyint, _byte, _float, _real, _double, _boolean, _string, _binary, _date, _timestamp, _timestampntz, _array, _map, _struct) +VALUES + (1, 1.1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1, true, 's', X'1', CAST('2020-12-31' AS DATE), '2021-7-1T8:43:28UTC+3', '2021-7-1T8:43:28.123456', ARRAY(1,2,3), map('key', 'value'), struct('val1', 1)), + (2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, false, '', X'', CAST('2020-12-31' AS DATE), '2021-7-1T8:43:28UTC+3', '2021-7-1T8:43:28.123456', ARRAY(), map('',''), struct('', 0) ), + (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ); \ No newline at end of file diff --git a/sqlconnect/internal/integration_test/db_integration_test_scenario.go b/sqlconnect/internal/integration_test/db_integration_test_scenario.go index cdf5cea..da5c412 100644 --- a/sqlconnect/internal/integration_test/db_integration_test_scenario.go +++ b/sqlconnect/internal/integration_test/db_integration_test_scenario.go @@ -73,7 +73,6 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe require.NoError(t, err, "it shouldn't fail if the schema already exists") }) }) - t.Run("exists", func(t *testing.T) { exists, err := db.SchemaExists(ctx, schema) require.NoError(t, err, "it should be able to check if a schema exists") @@ -276,7 +275,6 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) t.Run("column mapping", func(t *testing.T) { - t.Skip("column mapping tests are skipped until they are implemented for all warehouses") table := sqlconnect.NewRelationRef(formatfn("column_mappings_test"), sqlconnect.WithSchema(schema.Name)) ExecuteStatements(t, db, schema.Name, "testdata/column-mapping-test-seed.sql") @@ -306,7 +304,6 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe }) t.Run("json mapper", func(t *testing.T) { - expectedRowsJSON, err := os.ReadFile("testdata/column-mapping-test-rows.json") require.NoErrorf(t, err, "it should be able to read the rows json file") @@ -325,7 +322,6 @@ func TestDatabaseScenarios(t *testing.T, warehouse string, configJSON json.RawMe require.JSONEq(t, string(expectedRowsJSON), string(actualRowsJSON), "it should return the correct rows: "+string(actualRowsJSON)) }) - }) } diff --git a/sqlconnect/internal/mysql/mappings.go b/sqlconnect/internal/mysql/mappings.go index 7411467..f97cd7c 100644 --- a/sqlconnect/internal/mysql/mappings.go +++ b/sqlconnect/internal/mysql/mappings.go @@ -1,95 +1,85 @@ package mysql import ( + "encoding/binary" "encoding/json" "strconv" "time" ) -const ( - charvar = "CHAR" - varchar = "VARCHAR" - blob = "BLOB" - text = "TEXT" - tinyblob = "TINYBLOB" - tinytext = "TINYTEXT" - mediumblob = "MEDIUMBLOB" - mediumtext = "MEDIUMTEXT" - longblob = "LONGBLOB" - longtext = "LONGTEXT" - enumvar = "ENUM" - datevar = "DATE" - datetimevar = "DATETIME" - timestampvar = "TIMESTAMP" - timevar = "TIME" - year = "YEAR" - floatvar = "FLOAT" - doublevar = "DOUBLE" - decimalvar = "DECIMAL" - integer = "INT" - tinyint = "TINYINT" - smallint = "SMALLINt" - mediumint = "MEDIUMINT" - bigint = "BIGINT" -) - // mapping of database column types to rudder types var columnTypeMappings = map[string]string{ - varchar: "string", - integer: "int", + "INTEGER": "int", + "INT": "int", + "TINYINT": "int", + "SMALLINT": "int", + "MEDIUMINT": "int", + "BIGINT": "int", + "DECIMAL": "float", + "NUMERIC": "float", + "FLOAT": "float", + "DOUBLE": "float", + "BIT": "int", + "CHAR": "string", + "VARCHAR": "string", + "BINARY": "string", + "VARBINARY": "string", + "BLOB": "string", + "TINYBLOB": "string", + "MEDIUMBLOB": "string", + "LONGBLOB": "string", + "TEXT": "string", + "TINYTEXT": "string", + "MEDIUMTEXT": "string", + "LONGTEXT": "string", + "ENUM": "string", + "SET": "string", + "DATE": "datetime", + "DATETIME": "datetime", + "TIMESTAMP": "datetime", + "TIME": "datetime", + "YEAR": "datetime", + "JSON": "json", } func jsonRowMapper(databaseTypeName string, value interface{}) interface{} { if value == nil { return nil } - switch databaseTypeName { - case charvar, varchar, blob, text, tinyblob, tinytext, mediumblob, mediumtext, longblob, longtext, enumvar: - switch v := value.(type) { - case []uint8: - return string(v) - default: - return json.RawMessage(value.(string)) - } - case datevar, datetimevar, timestampvar, timevar, year: - switch v := value.(type) { - case []uint8: - return string(v) - default: - return value.(time.Time) - } - case floatvar, doublevar, decimalvar: - switch v := value.(type) { - case []uint8: - n, err := strconv.ParseFloat(string(v), 64) - if err != nil { - panic(err) - } - return n - default: - n, err := strconv.ParseFloat(value.(string), 64) - if err != nil { - panic(err) - } - return n + var stringValue string + switch v := value.(type) { + case []uint8: + stringValue = string(v) + case time.Time: + stringValue = v.String() + case string: + stringValue = v + } + + switch databaseTypeName { + case "CHAR", "VARCHAR", "BLOB", "TEXT", "TINYBLOB", "TINYTEXT", "MEDIUMBLOB", "MEDIUMTEXT", "LONGBLOB", "LONGTEXT", + "ENUM", "BINARY", "VARBINARY", + "SET", + "DATE", "DATETIME", "TIMESTAMP", "TIME", "YEAR": + return stringValue + case "JSON": + return json.RawMessage(stringValue) + case "FLOAT", "DOUBLE", "DECIMAL": + n, err := strconv.ParseFloat(stringValue, 64) + if err != nil { + panic(err) } - case integer, tinyint, smallint, mediumint, bigint: - switch v := value.(type) { - case []uint8: - n, err := strconv.ParseInt(string(v), 10, 64) - if err != nil { - panic(err) - } - return n - default: - n, err := strconv.ParseInt(value.(string), 10, 64) - if err != nil { - panic(err) - } - return n + return n + case "INT", "TINYINT", "SMALLINT", "MEDIUMINT", "BIGINT": + n, err := strconv.ParseInt(stringValue, 10, 64) + if err != nil { + panic(err) } + return n + case "BIT": + n := binary.BigEndian.Uint64([]byte(stringValue)) + return n } - return value } diff --git a/sqlconnect/internal/mysql/testdata/column-mapping-test-columns.json b/sqlconnect/internal/mysql/testdata/column-mapping-test-columns.json new file mode 100644 index 0000000..7499964 --- /dev/null +++ b/sqlconnect/internal/mysql/testdata/column-mapping-test-columns.json @@ -0,0 +1,130 @@ +[ + { + "name": "_order", + "type": "int" + }, + { + "name": "_integer", + "type": "int" + }, + { + "name": "_int", + "type": "int" + }, + { + "name": "_tinyint", + "type": "int" + }, + { + "name": "_smallint", + "type": "int" + }, + { + "name": "_mediumint", + "type": "int" + }, + { + "name": "_bigint", + "type": "int" + }, + { + "name": "_decimal", + "type": "float" + }, + { + "name": "_numeric", + "type": "float" + }, + { + "name": "_float", + "type": "float" + }, + { + "name": "_double", + "type": "float" + }, + { + "name": "_bit", + "type": "int" + }, + { + "name": "_char", + "type": "string" + }, + { + "name": "_varchar", + "type": "string" + }, + { + "name": "_bin", + "type": "string" + }, + { + "name": "_varbinary", + "type": "string" + }, + { + "name": "_blob", + "type": "string" + }, + { + "name": "_tinyblob", + "type": "string" + }, + { + "name": "_mediumblob", + "type": "string" + }, + { + "name": "_longblob", + "type": "string" + }, + { + "name": "_text", + "type": "string" + }, + { + "name": "_tinytext", + "type": "string" + }, + { + "name": "_mediumtext", + "type": "string" + }, + { + "name": "_longtext", + "type": "string" + }, + { + "name": "_enum", + "type": "string" + }, + { + "name": "_set", + "type": "string" + }, + { + "name": "_date", + "type": "datetime" + }, + { + "name": "_datetime", + "type": "datetime" + }, + { + "name": "_timestamp", + "type": "datetime" + }, + { + "name": "_time", + "type": "datetime" + }, + { + "name": "_year", + "type": "datetime" + }, + { + "name": "_json", + "type": "json" + } +] \ No newline at end of file diff --git a/sqlconnect/internal/mysql/testdata/column-mapping-test-rows.json b/sqlconnect/internal/mysql/testdata/column-mapping-test-rows.json new file mode 100644 index 0000000..48acacd --- /dev/null +++ b/sqlconnect/internal/mysql/testdata/column-mapping-test-rows.json @@ -0,0 +1,104 @@ +[ + { + "_order": 1, + "_integer": 1, + "_int": 1, + "_tinyint": 1, + "_smallint": 1, + "_mediumint": 1, + "_bigint": 1, + "_decimal": 1.1, + "_numeric": 1.1, + "_float": 1.1, + "_double": 1.1, + "_bit": 3047, + "_char": "a", + "_varchar": "abc", + "_bin": "a\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "_varbinary": "a", + "_blob": "b", + "_tinyblob": "b", + "_mediumblob": "b", + "_longblob": "b", + "_text": "t", + "_tinytext": "t", + "_mediumtext": "t", + "_longtext": "t", + "_enum": "1", + "_set": "one,two", + "_date": "2020-01-01", + "_datetime": "2020-01-01 15:10:10", + "_timestamp": "2020-01-01 15:10:10", + "_time": "10:45:15", + "_year": "2020", + "_json": {"key": "value"} + }, + { + "_order": 2, + "_integer": 0, + "_int": 0, + "_tinyint": 0, + "_smallint": 0, + "_mediumint": 0, + "_bigint": 0, + "_decimal": 0, + "_numeric": 0, + "_float": 0, + "_double": 0, + "_bit": 0, + "_char": "", + "_varchar": "", + "_bin": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "_varbinary": "", + "_blob": "", + "_tinyblob": "", + "_mediumblob": "", + "_longblob": "", + "_text": "", + "_tinytext": "", + "_mediumtext": "", + "_longtext": "", + "_enum": "1", + "_set": "", + "_date": "2020-01-01", + "_datetime": "2020-01-01 15:10:10", + "_timestamp": "2020-01-01 15:10:10", + "_time": "10:45:15", + "_year": "2020", + "_json": {} + }, + { + "_order": 3, + "_integer": null, + "_int": null, + "_tinyint": null, + "_smallint": null, + "_mediumint": null, + "_bigint": null, + "_decimal": null, + "_numeric": null, + "_float": null, + "_double": null, + "_bit": null, + "_char": null, + "_varchar": null, + "_bin": null, + "_varbinary": null, + "_blob": null, + "_tinyblob": null, + "_mediumblob": null, + "_longblob": null, + "_text": null, + "_tinytext": null, + "_mediumtext": null, + "_longtext": null, + "_enum": null, + "_set": null, + "_date": null, + "_datetime": null, + "_timestamp": null, + "_time": null, + "_year": null, + "_json": null + } +] \ No newline at end of file diff --git a/sqlconnect/internal/mysql/testdata/column-mapping-test-seed.sql b/sqlconnect/internal/mysql/testdata/column-mapping-test-seed.sql new file mode 100644 index 0000000..8cf40f5 --- /dev/null +++ b/sqlconnect/internal/mysql/testdata/column-mapping-test-seed.sql @@ -0,0 +1,41 @@ +CREATE TABLE `{{.schema}}`.`column_mappings_test` ( + _order INT, + _integer INTEGER, + _int INT, + _tinyint TINYINT, + _smallint SMALLINT, + _mediumint MEDIUMINT, + _bigint BIGINT, + _decimal DECIMAL(5,2), + _numeric NUMERIC(5,2), + _float FLOAT, + _double DOUBLE, + _bit BIT(64), + _char CHAR(1), + _varchar VARCHAR(10), + _bin BINARY(8), + _varbinary VARBINARY(100), + _blob BLOB, + _tinyblob TINYBLOB, + _mediumblob MEDIUMBLOB, + _longblob LONGBLOB, + _text TEXT(10), + _tinytext TINYTEXT, + _mediumtext MEDIUMTEXT, + _longtext LONGTEXT, + _enum ENUM('1', '2', '3'), + _set SET('one', 'two', 'three'), + _date DATE, + _datetime DATETIME, + _timestamp TIMESTAMP, + _time TIME, + _year YEAR, + _json JSON +); + +INSERT INTO `{{.schema}}`.`column_mappings_test` + (_order, _integer, _int, _tinyint, _smallint, _mediumint, _bigint, _decimal, _numeric, _float, _double, _bit, _char, _varchar, _bin, _varbinary, _blob, _tinyblob, _mediumblob, _longblob, _text, _tinytext, _mediumtext, _longtext, _enum, _set, _date, _datetime, _timestamp, _time, _year, _json) +VALUES + (1, 1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1, 1.1, b'101111100111', 'a', 'abc', 'a', 'a', 'b', 'b', 'b', 'b', 't', 't', 't', 't', '1', 'one,two', '2020-01-01', '2020-01-01 15:10:10', '2020-01-01 15:10:10', '10:45:15', '2020', '{"key": "value"}'), + (2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', '1', '', '2020-01-01', '2020-01-01 15:10:10', '2020-01-01 15:10:10', '10:45:15', '2020', '{}' ), + (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ); \ No newline at end of file diff --git a/sqlconnect/internal/postgres/mappings.go b/sqlconnect/internal/postgres/mappings.go index 7c3d50b..1aff175 100644 --- a/sqlconnect/internal/postgres/mappings.go +++ b/sqlconnect/internal/postgres/mappings.go @@ -1,6 +1,9 @@ package postgres -import "encoding/json" +import ( + "encoding/json" + "strconv" +) // mapping of database column types to rudder types var columnTypeMappings = map[string]string{ @@ -31,6 +34,7 @@ var columnTypeMappings = map[string]string{ "timestamp": "datetime", "boolean": "boolean", "bool": "boolean", + "json": "json", "jsonb": "json", } @@ -44,6 +48,13 @@ func jsonRowMapper(databaseTypeName string, value any) any { case string: return json.RawMessage(v) } + case "NUMERIC": + switch v := value.(type) { + case []byte: + if n, err := strconv.ParseFloat(string(v), 64); err == nil { + return n + } + } default: switch v := value.(type) { case []byte: diff --git a/sqlconnect/internal/postgres/testdata/column-mapping-test-columns.json b/sqlconnect/internal/postgres/testdata/column-mapping-test-columns.json new file mode 100644 index 0000000..39157cd --- /dev/null +++ b/sqlconnect/internal/postgres/testdata/column-mapping-test-columns.json @@ -0,0 +1,114 @@ +[ + { + "name": "_order", + "type": "int" + }, + { + "name": "_int", + "type": "int" + }, + { + "name": "_int2", + "type": "int" + }, + { + "name": "_int4", + "type": "int" + }, + { + "name": "_int8", + "type": "int" + }, + { + "name": "_integer", + "type": "int" + }, + { + "name": "_smallint", + "type": "int" + }, + { + "name": "_bigint", + "type": "int" + }, + { + "name": "_real", + "type": "float" + }, + { + "name": "_float", + "type": "float" + }, + { + "name": "_float4", + "type": "float" + }, + { + "name": "_float8", + "type": "float" + }, + { + "name": "_numeric", + "type": "float" + }, + { + "name": "_double", + "type": "float" + }, + { + "name": "_text", + "type": "string" + }, + { + "name": "_varchar", + "type": "string" + }, + { + "name": "_charvar", + "type": "string" + }, + { + "name": "_nchar", + "type": "string" + }, + { + "name": "_bpchar", + "type": "string" + }, + { + "name": "_character", + "type": "string" + }, + { + "name": "_timestamptz", + "type": "datetime" + }, + { + "name": "_timestampntz", + "type": "datetime" + }, + { + "name": "_timestampwtz", + "type": "datetime" + }, + { + "name": "_timestamp", + "type": "datetime" + }, + { + "name": "_boolean", + "type": "boolean" + }, + { + "name": "_bool", + "type": "boolean" + }, + { + "name": "_json", + "type": "json" + }, + { + "name": "_jsonb", + "type": "json" + } +] \ No newline at end of file diff --git a/sqlconnect/internal/postgres/testdata/column-mapping-test-rows.json b/sqlconnect/internal/postgres/testdata/column-mapping-test-rows.json new file mode 100644 index 0000000..14bc4fd --- /dev/null +++ b/sqlconnect/internal/postgres/testdata/column-mapping-test-rows.json @@ -0,0 +1,96 @@ +[ + { + "_order": 1, + "_int": 1, + "_int2": 1, + "_int4": 1, + "_int8": 1, + "_integer": 1, + "_smallint": 1, + "_bigint": 1, + "_real": 1.1, + "_float": 1.1, + "_float4": 1.1, + "_float8": 1.1, + "_numeric": 1.1, + "_double": 1.1, + "_text": "abc", + "_varchar": "abc", + "_charvar": "abc", + "_nchar": "abc ", + "_bpchar": "abc", + "_character": "abc ", + "_timestamptz": "2004-10-19T08:23:54Z", + "_timestampntz": "2004-10-19T10:23:54Z", + "_timestampwtz": "2004-10-19T08:23:54Z", + "_timestamp": "2004-10-19T10:23:54Z", + "_boolean": true, + "_bool": true, + "_json": { + "a": 1 + }, + "_jsonb": { + "a": 1 + } + }, + { + "_order": 2, + "_int": 0, + "_int2": 0, + "_int4": 0, + "_int8": 0, + "_integer": 0, + "_smallint": 0, + "_bigint": 0, + "_real": 0, + "_float": 0, + "_float4": 0, + "_float8": 0, + "_numeric": 0, + "_double": 0, + "_text": "", + "_varchar": "", + "_charvar": "", + "_nchar": " ", + "_bpchar": "", + "_character": " ", + "_timestamptz": "2004-10-19T08:23:54Z", + "_timestampntz": "2004-10-19T10:23:54Z", + "_timestampwtz": "2004-10-19T08:23:54Z", + "_timestamp": "2004-10-19T10:23:54Z", + "_boolean": false, + "_bool": false, + "_json": {}, + "_jsonb": {} + }, + { + "_order": 3, + "_int": null, + "_int2": null, + "_int4": null, + "_int8": null, + "_integer": null, + "_smallint": null, + "_bigint": null, + "_real": null, + "_float": null, + "_float4": null, + "_float8": null, + "_numeric": null, + "_double": null, + "_text": null, + "_varchar": null, + "_charvar": null, + "_nchar": null, + "_bpchar": null, + "_character": null, + "_timestamptz": null, + "_timestampntz": null, + "_timestampwtz": null, + "_timestamp": null, + "_boolean": null, + "_bool": null, + "_json": null, + "_jsonb": null + } +] \ No newline at end of file diff --git a/sqlconnect/internal/postgres/testdata/column-mapping-test-seed.sql b/sqlconnect/internal/postgres/testdata/column-mapping-test-seed.sql new file mode 100644 index 0000000..3c88142 --- /dev/null +++ b/sqlconnect/internal/postgres/testdata/column-mapping-test-seed.sql @@ -0,0 +1,37 @@ +CREATE TABLE "{{.schema}}"."column_mappings_test" ( + _order INT, + _int INT, + _int2 INT2, + _int4 INT4, + _int8 INT8, + _integer INTEGER, + _smallint SMALLINT, + _bigint BIGINT, + _real REAL, + _float FLOAT, + _float4 FLOAT4, + _float8 FLOAT8, + _numeric NUMERIC(10,2), + _double DOUBLE PRECISION, + _text TEXT, + _varchar VARCHAR(10), + _charvar CHARACTER VARYING, + _nchar NCHAR(10), + _bpchar BPCHAR, + _character CHARACTER(10), + _timestamptz TIMESTAMPTZ, + _timestampntz TIMESTAMP WITHOUT TIME ZONE, + _timestampwtz TIMESTAMP WITH TIME ZONE, + _timestamp TIMESTAMP, + _boolean BOOLEAN, + _bool BOOL, + _json JSON, + _jsonb JSONB +); + +INSERT INTO "{{.schema}}"."column_mappings_test" + (_order, _int, _int2, _int4, _int8, _integer, _smallint, _bigint, _real, _float, _float4, _float8, _numeric, _double, _text, _varchar, _charvar, _nchar, _bpchar, _character, _timestamptz, _timestampntz, _timestampwtz, _timestamp, _boolean, _bool, _json, _jsonb) +VALUES + (1, 1, 1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 'abc', 'abc', 'abc', 'abc', 'abc', 'abc', '2004-10-19 10:23:54+02', '2004-10-19 10:23:54', '2004-10-19 10:23:54+02', '2004-10-19 10:23:54+02', true, true, '{"a": 1}', '{"a": 1}'), + (2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', '', '', '', '', '2004-10-19 10:23:54+02', '2004-10-19 10:23:54', '2004-10-19 10:23:54+02', '2004-10-19 10:23:54+02', false, false, '{}', '{}' ), + (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ); \ No newline at end of file diff --git a/sqlconnect/internal/redshift/db.go b/sqlconnect/internal/redshift/db.go index 1bf3ab7..aabab29 100644 --- a/sqlconnect/internal/redshift/db.go +++ b/sqlconnect/internal/redshift/db.go @@ -34,6 +34,7 @@ func NewDB(credentialsJSON json.RawMessage) (*DB, error) { db, lo.Ternary(config.RudderSchema != "", config.RudderSchema, defaultRudderSchema), base.WithColumnTypeMappings(columnTypeMappings), + base.WithJsonRowMapper(jsonRowMapper), base.WithSQLCommandsOverride(func(cmds base.SQLCommands) base.SQLCommands { cmds.ListSchemas = func() (string, string) { return "SELECT schema_name FROM svv_redshift_schemas", "schema_name" diff --git a/sqlconnect/internal/redshift/mappings.go b/sqlconnect/internal/redshift/mappings.go index c0a5799..fbf6bdb 100644 --- a/sqlconnect/internal/redshift/mappings.go +++ b/sqlconnect/internal/redshift/mappings.go @@ -1,5 +1,9 @@ package redshift +import ( + "strconv" +) + var columnTypeMappings = map[string]string{ "int": "int", "int2": "int", @@ -25,6 +29,28 @@ var columnTypeMappings = map[string]string{ "nvarchar": "string", "string": "string", "date": "datetime", + "timestamptz": "datetime", "timestamp without time zone": "datetime", "timestamp with time zone": "datetime", + "timestamp": "datetime", +} + +// jsonRowMapper maps a row's scanned column to a json object's field +func jsonRowMapper(databaseTypeName string, value any) any { + switch databaseTypeName { + case "NUMERIC": + switch v := value.(type) { + case []byte: + if n, err := strconv.ParseFloat(string(v), 64); err == nil { + return n + } + } + default: + switch v := value.(type) { + case []byte: + return string(v) + } + } + + return value } diff --git a/sqlconnect/internal/redshift/testdata/column-mapping-test-columns.json b/sqlconnect/internal/redshift/testdata/column-mapping-test-columns.json new file mode 100644 index 0000000..b7870f7 --- /dev/null +++ b/sqlconnect/internal/redshift/testdata/column-mapping-test-columns.json @@ -0,0 +1,106 @@ +[ + { + "name": "_order", + "type": "int" + }, + { + "name": "_int", + "type": "int" + }, + { + "name": "_int2", + "type": "int" + }, + { + "name": "_int4", + "type": "int" + }, + { + "name": "_int8", + "type": "int" + }, + { + "name": "_integer", + "type": "int" + }, + { + "name": "_smallint", + "type": "int" + }, + { + "name": "_bigint", + "type": "int" + }, + { + "name": "_real", + "type": "float" + }, + { + "name": "_float", + "type": "float" + }, + { + "name": "_float4", + "type": "float" + }, + { + "name": "_float8", + "type": "float" + }, + { + "name": "_numeric", + "type": "float" + }, + { + "name": "_double", + "type": "float" + }, + { + "name": "_text", + "type": "string" + }, + { + "name": "_varchar", + "type": "string" + }, + { + "name": "_charvar", + "type": "string" + }, + { + "name": "_nchar", + "type": "string" + }, + { + "name": "_bpchar", + "type": "string" + }, + { + "name": "_character", + "type": "string" + }, + { + "name": "_timestamptz", + "type": "datetime" + }, + { + "name": "_timestampntz", + "type": "datetime" + }, + { + "name": "_timestampwtz", + "type": "datetime" + }, + { + "name": "_timestamp", + "type": "datetime" + }, + { + "name": "_boolean", + "type": "boolean" + }, + { + "name": "_bool", + "type": "boolean" + } +] \ No newline at end of file diff --git a/sqlconnect/internal/redshift/testdata/column-mapping-test-rows.json b/sqlconnect/internal/redshift/testdata/column-mapping-test-rows.json new file mode 100644 index 0000000..e4eaa73 --- /dev/null +++ b/sqlconnect/internal/redshift/testdata/column-mapping-test-rows.json @@ -0,0 +1,86 @@ +[ + { + "_order": 1, + "_int": 1, + "_int2": 1, + "_int4": 1, + "_int8": 1, + "_integer": 1, + "_smallint": 1, + "_bigint": 1, + "_real": 1.1, + "_float": 1.1, + "_float4": 1.1, + "_float8": 1.1, + "_numeric": 1.1, + "_double": 1.1, + "_text": "abc", + "_varchar": "abc", + "_charvar": "abc", + "_nchar": "abc ", + "_bpchar": "abc ", + "_character": "abc ", + "_timestamptz": "2004-10-19T08:23:54Z", + "_timestampntz": "2004-10-19T10:23:54Z", + "_timestampwtz": "2004-10-19T08:23:54Z", + "_timestamp": "2004-10-19T10:23:54Z", + "_boolean": true, + "_bool": true + }, + { + "_order": 2, + "_int": 0, + "_int2": 0, + "_int4": 0, + "_int8": 0, + "_integer": 0, + "_smallint": 0, + "_bigint": 0, + "_real": 0, + "_float": 0, + "_float4": 0, + "_float8": 0, + "_numeric": 0, + "_double": 0, + "_text": "", + "_varchar": "", + "_charvar": "", + "_nchar": " ", + "_bpchar": " ", + "_character": " ", + "_timestamptz": "2004-10-19T08:23:54Z", + "_timestampntz": "2004-10-19T10:23:54Z", + "_timestampwtz": "2004-10-19T08:23:54Z", + "_timestamp": "2004-10-19T10:23:54Z", + "_boolean": false, + "_bool": false + }, + { + "_order": 3, + "_int": null, + "_int2": null, + "_int4": null, + "_int8": null, + "_integer": null, + "_smallint": null, + "_bigint": null, + "_real": null, + "_float": null, + "_float4": null, + "_float8": null, + "_numeric": null, + "_double": null, + "_text": null, + "_varchar": null, + "_charvar": null, + "_nchar": null, + "_bpchar": null, + "_character": null, + "_timestamptz": null, + "_timestampntz": null, + "_timestampwtz": null, + "_timestamp": null, + "_boolean": null, + "_bool": null + } +] \ No newline at end of file diff --git a/sqlconnect/internal/redshift/testdata/column-mapping-test-seed.sql b/sqlconnect/internal/redshift/testdata/column-mapping-test-seed.sql new file mode 100644 index 0000000..57c29aa --- /dev/null +++ b/sqlconnect/internal/redshift/testdata/column-mapping-test-seed.sql @@ -0,0 +1,35 @@ +CREATE TABLE "{{.schema}}"."column_mappings_test" ( + _order INT, + _int INT, + _int2 INT2, + _int4 INT4, + _int8 INT8, + _integer INTEGER, + _smallint SMALLINT, + _bigint BIGINT, + _real REAL, + _float FLOAT, + _float4 FLOAT4, + _float8 FLOAT8, + _numeric NUMERIC(10,2), + _double DOUBLE PRECISION, + _text TEXT, + _varchar VARCHAR(10), + _charvar CHARACTER VARYING, + _nchar NCHAR(10), + _bpchar BPCHAR, + _character CHARACTER(10), + _timestamptz TIMESTAMPTZ, + _timestampntz TIMESTAMP WITHOUT TIME ZONE, + _timestampwtz TIMESTAMP WITH TIME ZONE, + _timestamp TIMESTAMP, + _boolean BOOLEAN, + _bool BOOL +); + +INSERT INTO "{{.schema}}"."column_mappings_test" + (_order, _int, _int2, _int4, _int8, _integer, _smallint, _bigint, _real, _float, _float4, _float8, _numeric, _double, _text, _varchar, _charvar, _nchar, _bpchar, _character, _timestamptz, _timestampntz, _timestampwtz, _timestamp, _boolean, _bool) +VALUES + (1, 1, 1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 'abc', 'abc', 'abc', 'abc', 'abc', 'abc', '2004-10-19 10:23:54+02', '2004-10-19 10:23:54', '2004-10-19 10:23:54+02', '2004-10-19 10:23:54+02', true, true ), + (2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', '', '', '', '', '2004-10-19 10:23:54+02', '2004-10-19 10:23:54', '2004-10-19 10:23:54+02', '2004-10-19 10:23:54+02', false, false), + (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ); \ No newline at end of file diff --git a/sqlconnect/internal/snowflake/mappings.go b/sqlconnect/internal/snowflake/mappings.go index 000b537..7f6fd73 100644 --- a/sqlconnect/internal/snowflake/mappings.go +++ b/sqlconnect/internal/snowflake/mappings.go @@ -13,10 +13,9 @@ import ( // mapping of database column types to rudder types var columnTypeMappings = map[string]string{ - "NUMBER": "int", - "DECIMAL": "int", - "NUMERIC": "int", "INT": "int", + "DECIMAL": "float", + "NUMERIC": "float", "INTEGER": "int", "BIGINT": "int", "SMALLINT": "int", @@ -43,17 +42,33 @@ var columnTypeMappings = map[string]string{ "TIMESTAMP_LTZ": "datetime", "TIMESTAMP_TZ": "datetime", "VARIANT": "json", + "OBJECT": "json", + "ARRAY": "json", } -var re = regexp.MustCompile(`([^(]+) ?\(.*`) +// var re = regexp.MustCompile(`(\([^)]+\)|<[^>]+>)`) // remove type parameters [<>] and size constraints [()] +var ( + re = regexp.MustCompile(`(\(.+\)|<.+>)`) // remove type parameters [<>] and size constraints [()] + numberPrecision = regexp.MustCompile(`NUMBER\((?P\d+),(?P\d+)\)`) +) func columnTypeMapper(columnType base.ColumnType) string { - databaseTypeName := strings.ToUpper(re.ReplaceAllString(columnType.DatabaseTypeName(), "$1")) + databaseTypeName := strings.ToUpper(re.ReplaceAllString(columnType.DatabaseTypeName(), "")) if mappedType, ok := columnTypeMappings[strings.ToUpper(databaseTypeName)]; ok { return mappedType } - if databaseTypeName == "FIXED" { - if _, decimals, ok := columnType.DecimalSize(); ok && decimals > 0 { + + if databaseTypeName == "NUMBER" { // [DESCRIBE TABLE] returns [NUMBER(precision,scale)] for various numeric types, including [INT] types + if matches := numberPrecision.FindStringSubmatch(columnType.DatabaseTypeName()); len(matches) > 0 { + precisionIndex := numberPrecision.SubexpIndex("precision") + if precision, err := strconv.ParseInt(matches[precisionIndex+1], 10, 64); err == nil && precision == 0 { + return "int" + } + } + return "float" + } + if databaseTypeName == "FIXED" { // When finding column types of a query, for most numeric types the driver returns [FIXED] + if precision, decimals, ok := columnType.DecimalSize(); ok && precision > 0 && decimals > 0 { return "float" } return "int" @@ -66,14 +81,11 @@ func jsonRowMapper(databaseTypeName string, value interface{}) interface{} { if value == nil { return nil } - switch databaseTypeName { - // in case of NOT string, the function returns the value itself case "BOOLEAN": if s, ok := value.(string); ok { return s == "1" } - case "FIXED": switch v := value.(type) { case float64: @@ -91,30 +103,14 @@ func jsonRowMapper(databaseTypeName string, value interface{}) interface{} { default: panic(fmt.Errorf("unsupported type for FIXED:%t", v)) } - case "OBJECT": return json.RawMessage(value.(string)) - - case "ARRAY": - return json.RawMessage(value.(string)) - case "VARIANT": - return value.(string) - - case "DATE": - return value.(time.Time) - - case "TIME": - return value.(time.Time) - - case "TIMESTAMP_LTZ": - return value.(time.Time) - - case "TIMESTAMP_NTZ": - return value.(time.Time) - - case "TIMESTAMP_TZ": + return json.RawMessage(value.(string)) + case "DATE", "TIME", "TIMESTAMP", "TIMESTAMP_LTZ", "TIMESTAMP_NTZ", "TIMESTAMP_TZ": return value.(time.Time) + case "BINARY", "VARBINARY": + return string(value.([]byte)) } return value diff --git a/sqlconnect/internal/snowflake/testdata/column-mapping-test-columns.json b/sqlconnect/internal/snowflake/testdata/column-mapping-test-columns.json new file mode 100644 index 0000000..7ee829d --- /dev/null +++ b/sqlconnect/internal/snowflake/testdata/column-mapping-test-columns.json @@ -0,0 +1,134 @@ +[ + { + "name": "_ORDER", + "type": "int" + }, + { + "name": "_INT", + "type": "int" + }, + { + "name": "_NUMBER", + "type": "float" + }, + { + "name": "_DECIMAL", + "type": "float" + }, + { + "name": "_NUMERIC", + "type": "float" + }, + { + "name": "_INTEGER", + "type": "int" + }, + { + "name": "_BIGINT", + "type": "int" + }, + { + "name": "_SMALLINT", + "type": "int" + }, + { + "name": "_TINYINT", + "type": "int" + }, + { + "name": "_FLOAT", + "type": "float" + }, + { + "name": "_FLOAT4", + "type": "float" + }, + { + "name": "_FLOAT8", + "type": "float" + }, + { + "name": "_DOUBLE", + "type": "float" + }, + { + "name": "_REAL", + "type": "float" + }, + { + "name": "_DOUBLE_PRECISION", + "type": "float" + }, + { + "name": "_BOOLEAN", + "type": "boolean" + }, + { + "name": "_TEXT", + "type": "string" + }, + { + "name": "_VARCHAR", + "type": "string" + }, + { + "name": "_CHAR", + "type": "string" + }, + { + "name": "_CHARACTER", + "type": "string" + }, + { + "name": "_STRING", + "type": "string" + }, + { + "name": "_BINARY", + "type": "string" + }, + { + "name": "_VARBINARY", + "type": "string" + }, + { + "name": "_DATE", + "type": "datetime" + }, + { + "name": "_DATETIME", + "type": "datetime" + }, + { + "name": "_TIME", + "type": "datetime" + }, + { + "name": "_TIMESTAMP", + "type": "datetime" + }, + { + "name": "_TIMESTAMPNTZ", + "type": "datetime" + }, + { + "name": "_TIMESTAMPLTZ", + "type": "datetime" + }, + { + "name": "_TIMESTAMPTZ", + "type": "datetime" + }, + { + "name": "_VARIANT", + "type": "json" + }, + { + "name": "_OBJECT", + "type": "json" + }, + { + "name": "_ARRAY", + "type": "json" + } +] \ No newline at end of file diff --git a/sqlconnect/internal/snowflake/testdata/column-mapping-test-rows.json b/sqlconnect/internal/snowflake/testdata/column-mapping-test-rows.json new file mode 100644 index 0000000..5ad04f5 --- /dev/null +++ b/sqlconnect/internal/snowflake/testdata/column-mapping-test-rows.json @@ -0,0 +1,107 @@ +[ + { + "_ORDER": 1, + "_INT": 1, + "_NUMBER": 1.1, + "_DECIMAL": 1.1, + "_NUMERIC": 1.1, + "_INTEGER": 1, + "_BIGINT": 1, + "_SMALLINT": 1, + "_TINYINT": 1, + "_FLOAT": 1.1, + "_FLOAT4": 1.1, + "_FLOAT8": 1.1, + "_DOUBLE": 1.1, + "_REAL": 1.1, + "_DOUBLE_PRECISION": 1.1, + "_BOOLEAN": true, + "_TEXT": "t", + "_VARCHAR": "vc", + "_CHAR": "c", + "_CHARACTER": "c", + "_STRING": "s", + "_BINARY": "bin", + "_VARBINARY": "vbin", + "_DATE": "2021-07-01T00:00:00Z", + "_DATETIME": "2017-01-01T12:00:00Z", + "_TIME": "0001-01-01T12:00:00Z", + "_TIMESTAMP": "2014-01-01T16:00:00Z", + "_TIMESTAMPNTZ": "2014-01-01T16:00:00Z", + "_TIMESTAMPLTZ": "2014-01-01T16:00:00-08:00", + "_TIMESTAMPTZ": "2014-01-01T16:00:00-08:00", + "_VARIANT": {"key": "value"}, + "_OBJECT": { "key": "value"}, + "_ARRAY": [1,2,3] + }, + { + "_ORDER": 2, + "_INT": 0, + "_NUMBER": 0, + "_DECIMAL": 0, + "_NUMERIC": 0, + "_INTEGER": 0, + "_BIGINT": 0, + "_SMALLINT": 0, + "_TINYINT": 0, + "_FLOAT": 0, + "_FLOAT4": 0, + "_FLOAT8": 0, + "_DOUBLE": 0, + "_REAL": 0, + "_DOUBLE_PRECISION": 0, + "_BOOLEAN": false, + "_TEXT": "", + "_VARCHAR": "", + "_CHAR": "", + "_CHARACTER": "", + "_STRING": "", + "_BINARY": "", + "_VARBINARY": "", + "_DATE": "2021-07-01T00:00:00Z", + "_DATETIME": "2017-01-01T12:00:00Z", + "_TIME": "0001-01-01T12:00:00Z", + "_TIMESTAMP": "2014-01-01T16:00:00Z", + "_TIMESTAMPNTZ": "2014-01-01T16:00:00Z", + "_TIMESTAMPLTZ": "2014-01-01T16:00:00-08:00", + "_TIMESTAMPTZ": "2014-01-01T16:00:00-08:00", + "_VARIANT": "string", + "_OBJECT": {}, + "_ARRAY": [] + }, + { + "_ORDER": 3, + "_INT": null, + "_NUMBER": null, + "_DECIMAL": null, + "_NUMERIC": null, + "_INTEGER": null, + "_BIGINT": null, + "_SMALLINT": null, + "_TINYINT": null, + "_FLOAT": null, + "_FLOAT4": null, + "_FLOAT8": null, + "_DOUBLE": null, + "_REAL": null, + "_DOUBLE_PRECISION": null, + "_BOOLEAN": null, + "_TEXT": null, + "_VARCHAR": null, + "_CHAR": null, + "_CHARACTER": null, + "_STRING": null, + "_BINARY": null, + "_VARBINARY": null, + "_DATE": null, + "_DATETIME": null, + "_TIME": null, + "_TIMESTAMP": null, + "_TIMESTAMPNTZ": null, + "_TIMESTAMPLTZ": null, + "_TIMESTAMPTZ": null, + "_VARIANT": null, + "_OBJECT": null, + "_ARRAY": null + } +] \ No newline at end of file diff --git a/sqlconnect/internal/snowflake/testdata/column-mapping-test-seed.sql b/sqlconnect/internal/snowflake/testdata/column-mapping-test-seed.sql new file mode 100644 index 0000000..f2634ea --- /dev/null +++ b/sqlconnect/internal/snowflake/testdata/column-mapping-test-seed.sql @@ -0,0 +1,51 @@ +CREATE TABLE "{{.schema}}"."COLUMN_MAPPINGS_TEST" ( + _order INT, + _int INT, + _number NUMBER(10,2), + _decimal DECIMAL(10,2), + _numeric NUMERIC(10,2), + _integer INTEGER, + _bigint BIGINT, + _smallint SMALLINT, + _tinyint TINYINT, + _float FLOAT, + _float4 FLOAT4, + _float8 FLOAT8, + _double DOUBLE, + _real REAL, + _double_precision DOUBLE PRECISION, + _boolean BOOLEAN, + _text TEXT, + _varchar VARCHAR, + _char CHAR, + _character CHARACTER, + _string STRING, + _binary BINARY, + _varbinary VARBINARY, + _date DATE, + _datetime DATETIME, + _time TIME, + _timestamp TIMESTAMP, + _timestampntz TIMESTAMP_NTZ, + _timestampltz TIMESTAMP_LTZ, + _timestamptz TIMESTAMP_TZ, + _variant VARIANT, + _object OBJECT, + _array VARIANT +); + +INSERT INTO "{{.schema}}"."COLUMN_MAPPINGS_TEST" + (_order, _int, _number, _decimal, _numeric, _integer, _bigint, _smallint, _tinyint, _float, _float4, _float8, _double, _real, _double_precision, _boolean, _text, _varchar, _char, _character, _string, _binary, _varbinary, _date, _datetime, _time, _timestamp, _timestampntz, _timestampltz, _timestamptz, _variant, _object, _array) +SELECT + 1, 1, 1.1, 1.1, 1.1, 1, 1, 1, 1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, true, 't', 'vc', 'c', 'c', 's', TO_BINARY('bin', 'UTF-8'), TO_BINARY('vbin', 'UTF-8'), '2021-7-1', '2017-01-01 12:00:00', '12:00:00', '2014-01-01 16:00:00', '2014-01-01 16:00:00', '2014-01-01 16:00:00', '2014-01-01 16:00:00', object_construct('key', 'value'), object_construct('key', 'value'), array_construct(1,2,3); + +INSERT INTO "{{.schema}}"."COLUMN_MAPPINGS_TEST" + (_order, _int, _number, _decimal, _numeric, _integer, _bigint, _smallint, _tinyint, _float, _float4, _float8, _double, _real, _double_precision, _boolean, _text, _varchar, _char, _character, _string, _binary, _varbinary, _date, _datetime, _time, _timestamp, _timestampntz, _timestampltz, _timestamptz, _variant, _object, _array) +SELECT + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, false, '', '', '', '', '', '', '', '2021-7-1', '2017-01-01 12:00:00', '12:00:00', '2014-01-01 16:00:00', '2014-01-01 16:00:00', '2014-01-01 16:00:00', '2014-01-01 16:00:00', 'string'::VARIANT, object_construct(), array_construct(); + + +INSERT INTO "{{.schema}}"."COLUMN_MAPPINGS_TEST" + (_order, _int, _number, _decimal, _numeric, _integer, _bigint, _smallint, _tinyint, _float, _float4, _float8, _double, _real, _double_precision, _boolean, _text, _varchar, _char, _character, _string, _binary, _varbinary, _date, _datetime, _time, _timestamp, _timestampntz, _timestampltz, _timestamptz, _variant, _object, _array) +SELECT + 3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL; \ No newline at end of file diff --git a/sqlconnect/internal/trino/mappings.go b/sqlconnect/internal/trino/mappings.go index 575f8c0..1be7b30 100644 --- a/sqlconnect/internal/trino/mappings.go +++ b/sqlconnect/internal/trino/mappings.go @@ -1,6 +1,8 @@ package trino import ( + "regexp" + "strconv" "strings" "github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base" @@ -8,30 +10,41 @@ import ( // mapping of database column types to rudder types var columnTypeMappings = map[string]string{ - "BOOLEAN": "boolean", - "INT": "int", - "INTEGER": "int", - "BIGINT": "int", - "SMALLINT": "int", - "TINYINT": "int", - "DECIMAL": "float", - "DOUBLE": "float", - "REAL": "float", - "TEXT": "string", + "BOOLEAN": "boolean", + "TINYINT": "int", + "SMALLINT": "int", + "INTEGER": "int", + "INT": "int", + "BIGINT": "int", + + "REAL": "float", + "DOUBLE": "float", + "DECIMAL": "float", + "VARCHAR": "string", "CHAR": "string", - "JSON": "json", - "DATE": "datetime", - "DATETIME": "datetime", - "TIME": "datetime", - "TIMESTAMP": "datetime", + "VARBINARY": "string", + + "DATE": "datetime", + "TIME": "datetime", + "TIMESTAMP": "datetime", + "TIME WITH TIME ZONE": "datetime", + "TIMESTAMP WITH TIME ZONE": "datetime", + + "JSON": "json", + "ARRAY": "json", + "MAP": "json", } +var re = regexp.MustCompile(`(\(.+\)|<.+>)`) // remove type parameters [<>] and size constraints [()] + func columnTypeMapper(columnType base.ColumnType) string { - databaseTypeName := strings.ToUpper(columnType.DatabaseTypeName()) - if mappedType, ok := columnTypeMappings[databaseTypeName]; ok { + databaseTypeName := strings.ToUpper(re.ReplaceAllString(columnType.DatabaseTypeName(), "")) + if mappedType, ok := columnTypeMappings[strings.ToUpper(databaseTypeName)]; ok { return mappedType } + + // TODO: is this still needed? if strings.Contains(databaseTypeName, "CHAR") || strings.Contains(databaseTypeName, "VARCHAR") { return "string" } else if strings.Contains(databaseTypeName, "TIMESTAMP") { @@ -45,6 +58,13 @@ func columnTypeMapper(columnType base.ColumnType) string { // jsonRowMapper maps a row's scanned column to a json object's field func jsonRowMapper(databaseTypeName string, value any) any { switch databaseTypeName { + case "DECIMAL": + switch v := value.(type) { + case string: + if value, err := strconv.ParseFloat(v, 64); err == nil { + return value + } + } default: switch v := value.(type) { case []byte: diff --git a/sqlconnect/internal/trino/testdata/column-mapping-test-columns.json b/sqlconnect/internal/trino/testdata/column-mapping-test-columns.json new file mode 100644 index 0000000..1cba59d --- /dev/null +++ b/sqlconnect/internal/trino/testdata/column-mapping-test-columns.json @@ -0,0 +1,70 @@ +[ + { + "name": "_order", + "type": "int" + }, + { + "name": "_int", + "type": "int" + }, + { + "name": "_tinyint", + "type": "int" + }, + { + "name": "_smallint", + "type": "int" + }, + { + "name": "_integer", + "type": "int" + }, + { + "name": "_bigint", + "type": "int" + }, + { + "name": "_real", + "type": "float" + }, + { + "name": "_double", + "type": "float" + }, + { + "name": "_decimal", + "type": "float" + }, + { + "name": "_varchar", + "type": "string" + }, + { + "name": "_char", + "type": "string" + }, + { + "name": "_varbinary", + "type": "string" + }, + { + "name": "_boolean", + "type": "boolean" + }, + { + "name": "_date", + "type": "datetime" + }, + { + "name": "_timestamp", + "type": "datetime" + }, + { + "name": "_array", + "type": "json" + }, + { + "name": "_map", + "type": "json" + } +] \ No newline at end of file diff --git a/sqlconnect/internal/trino/testdata/column-mapping-test-rows.json b/sqlconnect/internal/trino/testdata/column-mapping-test-rows.json new file mode 100644 index 0000000..9d7d707 --- /dev/null +++ b/sqlconnect/internal/trino/testdata/column-mapping-test-rows.json @@ -0,0 +1,69 @@ +[ + { + "_order": 1, + "_int": 1, + "_tinyint": 1, + "_smallint": 1, + "_integer": 1, + "_bigint": 1, + "_real": 1.1, + "_double": 1.1, + "_decimal": 1.1, + "_varchar": "abc", + "_char": "abc", + "_varbinary": "YWJj", + "_boolean": true, + "_date": "2004-10-19T00:00:00+03:00", + "_timestamp": "2004-10-19T10:23:54+03:00", + "_array": [ + 1, + 2, + 3 + ], + "_map": { + "bar": 2, + "foo": 1 + } + }, + { + "_order": 2, + "_int": 0, + "_tinyint": 0, + "_smallint": 0, + "_integer": 0, + "_bigint": 0, + "_real": 0, + "_double": 0, + "_decimal": 0, + "_varchar": "", + "_char": " ", + "_varbinary": "", + "_boolean": false, + "_date": "2004-10-19T00:00:00+03:00", + "_timestamp": "2004-10-19T10:23:54+03:00", + "_array": [], + "_map": { + "bar": 2, + "foo": 1 + } + }, + { + "_order": 3, + "_int": null, + "_tinyint": null, + "_smallint": null, + "_integer": null, + "_bigint": null, + "_real": null, + "_double": null, + "_decimal": null, + "_varchar": null, + "_char": null, + "_varbinary": null, + "_boolean": null, + "_date": null, + "_timestamp": null, + "_array": null, + "_map": null + } +] \ No newline at end of file diff --git a/sqlconnect/internal/trino/testdata/column-mapping-test-seed.sql b/sqlconnect/internal/trino/testdata/column-mapping-test-seed.sql new file mode 100644 index 0000000..cc47bf8 --- /dev/null +++ b/sqlconnect/internal/trino/testdata/column-mapping-test-seed.sql @@ -0,0 +1,26 @@ +CREATE TABLE "{{.schema}}"."column_mappings_test" ( + _order INT, + _int INT, + _tinyint TINYINT, + _smallint SMALLINT, + _integer INTEGER, + _bigint BIGINT, + _real REAL, + _double DOUBLE, + _decimal DECIMAL(2,1), + _varchar VARCHAR(3), + _char CHAR(3), + _varbinary VARBINARY, + _boolean BOOLEAN, + _date DATE, + _timestamp TIMESTAMP, + _array ARRAY, + _map MAP +); + +INSERT INTO "{{.schema}}"."column_mappings_test" + (_order, _int, _tinyint, _smallint, _integer, _bigint, _real, _double, _decimal, _varchar, _char, _varbinary, _boolean, _date, _timestamp, _array, _map) +VALUES + (1, 1, TINYINT '1', SMALLINT '1', 1, BIGINT '1', REAL '1.1', DOUBLE '1.1', DECIMAL '1.1', 'abc', CHAR 'abc', VARBINARY 'abc', true, DATE '2004-10-19', TIMESTAMP '2004-10-19 10:23:54', ARRAY[1, 2, 3], MAP(ARRAY['foo', 'bar'], ARRAY[1, 2]) ), + (2, 0, TINYINT '0', SMALLINT '0', 0, BIGINT '0', REAL '0', DOUBLE '0', DECIMAL '0', '', CHAR '', VARBINARY '', false, DATE '2004-10-19', TIMESTAMP '2004-10-19 10:23:54', ARRAY[], MAP(ARRAY['foo', 'bar'], ARRAY[1, 2]) ), + (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ); \ No newline at end of file