Skip to content

Commit

Permalink
fixup! feat: sqlconnect library
Browse files Browse the repository at this point in the history
  • Loading branch information
@atzoum
atzoum committed Feb 20, 2024
1 parent 96abec3 commit edce9eb
Show file tree
Hide file tree
Showing 31 changed files with 1,725 additions and 142 deletions.
4 changes: 3 additions & 1 deletion sqlconnect/internal/bigquery/mappings.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strings"

"cloud.google.com/go/bigquery"

"github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base"
)

Expand Down Expand Up @@ -49,7 +50,8 @@ var columnTypeMappings = map[string]string{
"RECORD": "json",
}

var re = regexp.MustCompile(`(\([^)]+\)|<[^>]+>)`)
// var re = regexp.MustCompile(`(\([^)]+\)|<[^>]+>)`) // remove type parameters [<>] and size constraints [()]
var re = regexp.MustCompile(`(\(.+\)|<.+>)`) // remove type parameters [<>] and size constraints [()]

func columnTypeMapper(columnType base.ColumnType) string {
databaseTypeName := strings.ToUpper(re.ReplaceAllString(columnType.DatabaseTypeName(), ""))
Expand Down
36 changes: 36 additions & 0 deletions sqlconnect/internal/bigquery/testdata/column-mapping-test-columns.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,18 @@
"name": "_bignumericnoscale",
"type": "float"
},
{
"name": "_bigdecimal",
"type": "float"
},
{
"name": "_bool",
"type": "boolean"
},
{
"name": "_boolean",
"type": "boolean"
},
{
"name": "_bytes",
"type": "string"
Expand All @@ -43,6 +51,30 @@
"name": "_int64",
"type": "int"
},
{
"name": "_int",
"type": "int"
},
{
"name": "_smallint",
"type": "int"
},
{
"name": "_integer",
"type": "int"
},
{
"name": "_bigint",
"type": "int"
},
{
"name": "_tinyint",
"type": "int"
},
{
"name": "_byteint",
"type": "int"
},
{
"name": "_interval",
"type": "int"
Expand All @@ -55,6 +87,10 @@
"name": "_numeric",
"type": "float"
},
{
"name": "_decimal",
"type": "float"
},
{
"name": "_string",
"type": "string"
Expand Down
27 changes: 27 additions & 0 deletions sqlconnect/internal/bigquery/testdata/column-mapping-test-rows.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,25 @@
"_array": ["ONE"],
"_bignumeric": 1.1,
"_bignumericnoscale": 1,
"_bigdecimal": 1,
"_bool": true,
"_boolean": true,
"_bytes": "abc",
"_date": "2014-09-27",
"_datetime": "2014-09-27T12:30:00.450000000",
"_float64": 1.1,
"_geo": "POINT(32 90)",
"_int64": 1,
"_int": 1,
"_smallint": 1,
"_integer": 1,
"_bigint": 1,
"_tinyint": 1,
"_byteint": 1,
"_interval": 31104000000000000,
"_json": {"key": "value"},
"_numeric": 1,
"_decimal": 1,
"_string": "string",
"_struct": ["string", 1],
"_time": "12:30:00.450000000",
Expand All @@ -24,16 +33,25 @@
"_array": null,
"_bignumeric": 0,
"_bignumericnoscale": 0,
"_bigdecimal": 0,
"_bool": false,
"_boolean": false,
"_bytes": "",
"_date": "2014-09-27",
"_datetime": "2014-09-27T12:30:00.450000000",
"_float64": 0,
"_geo": "GEOMETRYCOLLECTION EMPTY",
"_int64": 0,
"_int": 0,
"_smallint": 0,
"_integer": 0,
"_bigint": 0,
"_tinyint": 0,
"_byteint": 0,
"_interval": 31104000000000000,
"_json": {},
"_numeric": 0,
"_decimal": 0,
"_string": "",
"_struct": ["",0],
"_time": "12:30:00.450000000",
Expand All @@ -44,16 +62,25 @@
"_array": null,
"_bignumeric": null,
"_bignumericnoscale": null,
"_bigdecimal": null,
"_bool": null,
"_boolean": null,
"_bytes": null,
"_date": null,
"_datetime": null,
"_float64": null,
"_geo": null,
"_int64": null,
"_int": null,
"_smallint": null,
"_integer": null,
"_bigint": null,
"_tinyint": null,
"_byteint": null,
"_interval": null,
"_json": null,
"_numeric": null,
"_decimal": null,
"_string": null,
"_struct": null,
"_time": null,
Expand Down
17 changes: 13 additions & 4 deletions sqlconnect/internal/bigquery/testdata/column-mapping-test-seed.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,34 @@ CREATE TABLE `{{.schema}}`.`column_mappings_test` (
_array ARRAY<STRING>,
_bignumeric BIGNUMERIC(2,1),
_bignumericnoscale BIGNUMERIC(1,0),
_bigdecimal BIGDECIMAL,
_bool BOOL,
_boolean BOOLEAN,
_bytes BYTES,
_date DATE,
_datetime DATETIME,
_float64 FLOAT64,
_geo GEOGRAPHY,
_int64 INT64,
_int INT,
_smallint SMALLINT,
_integer INTEGER,
_bigint BIGINT,
_tinyint TINYINT,
_byteint BYTEINT,
_interval INTERVAL,
_json JSON,
_numeric NUMERIC,
_decimal NUMERIC,
_string STRING(10),
_struct STRUCT<a STRING, b INT64>,
_time TIME,
_timestamp TIMESTAMP,
);

INSERT INTO `{{.schema}}`.`column_mappings_test`
(_order, _array, _bignumeric, _bignumericnoscale, _bool, _bytes, _date, _datetime, _float64, _geo, _int64, _interval, _json, _numeric, _string, _struct, _time, _timestamp)
(_order, _array, _bignumeric, _bignumericnoscale, _bigdecimal, _bool, _boolean, _bytes, _date, _datetime, _float64, _geo, _int64, _int, _smallint, _integer, _bigint, _tinyint, _byteint, _interval, _json, _numeric, _decimal, _string, _struct, _time, _timestamp)
VALUES
(1, ['ONE'], 1.1, 1, TRUE, B"abc", '2014-09-27', '2014-09-27 12:30:00.45', 1.1, ST_GEOGFROMTEXT('POINT(32 90)'), 1, INTERVAL 1 YEAR, JSON '{"key": "value"}', 1, 'string', ('string', 1), '12:30:00.45', '2014-09-27 12:30:00.45-08'),
(2, [], 0.0, 0, FALSE, B"", '2014-09-27', '2014-09-27 12:30:00.45', 0.0, ST_GEOGFROMTEXT('POINT EMPTY'), 0, INTERVAL 1 YEAR, JSON '{}', 0, '', ('', 0), '12:30:00.45', '2014-09-27 12:30:00.45-08'),
(3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
(1, ['ONE'], 1.1, 1, 1, TRUE, TRUE, B"abc", '2014-09-27', '2014-09-27 12:30:00.45', 1.1, ST_GEOGFROMTEXT('POINT(32 90)'), 1, 1, 1, 1, 1, 1, 1, INTERVAL 1 YEAR, JSON '{"key": "value"}', 1, 1, 'string', ('string', 1), '12:30:00.45', '2014-09-27 12:30:00.45-08'),
(2, [], 0.0, 0, 0, FALSE, FALSE, B"", '2014-09-27', '2014-09-27 12:30:00.45', 0.0, ST_GEOGFROMTEXT('POINT EMPTY'), 0, 0, 0, 0, 0, 0, 0, INTERVAL 1 YEAR, JSON '{}', 0, 0, '', ('', 0), '12:30:00.45', '2014-09-27 12:30:00.45-08'),
(3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
2 changes: 1 addition & 1 deletion sqlconnect/internal/databricks/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func NewDB(configJson json.RawMessage) (*DB, error) {
db,
lo.Ternary(config.RudderSchema != "", config.RudderSchema, defaultRudderSchema),
base.WithDialect(dialect{}),
base.WithColumnTypeMappings(columnTypeMappings),
base.WithColumnTypeMapper(columnTypeMapper),
base.WithJsonRowMapper(jsonRowMapper),
base.WithSQLCommandsOverride(func(cmds base.SQLCommands) base.SQLCommands {
cmds.ListSchemas = func() (string, string) { return "SHOW SCHEMAS", "schema_name" }
Expand Down
71 changes: 60 additions & 11 deletions sqlconnect/internal/databricks/mappings.go
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,35 @@
package databricks

import (
"encoding/json"
"regexp"
"strconv"
"strings"

"github.com/rudderlabs/sqlconnect-go/sqlconnect/internal/base"
)

// mapping of database column types to rudder types
var columnTypeMappings = map[string]string{
"DECIMAL": "int",
"NUMERIC": "int",
"DEC": "int",
"INT": "int",
"BIGINT": "int",
"SMALLINT": "int",
"TINYINT": "int",
"FLOAT": "float",
"DECIMAL": "int", // DECIMAL and aliases
"NUMERIC": "int",
"DEC": "int",

"INT": "int", // INT and aliases
"INTEGER": "int",

"BIGINT": "int", // BIGINT and aliases
"LONG": "int",

"SMALLINT": "int", // SMALLINT and aliases
"SHORT": "int",

"TINYINT": "int", // TINYINT and aliases
"BYTE": "int",

"FLOAT": "float", // FLOAT and aliases
"REAL": "float",

"DOUBLE": "float",
"BOOLEAN": "boolean",
"STRING": "string",
Expand All @@ -19,16 +39,45 @@ var columnTypeMappings = map[string]string{
"VOID": "string",
"TIMESTAMP": "datetime",
"TIMESTAMP_NTZ": "datetime",
"ARRAY": "json",
"MAP": "json",
"STRUCT": "json",

"ARRAY": "json",
"MAP": "json",
"STRUCT": "json",
}

// var re = regexp.MustCompile(`(\([^)]+\)|<[^>]+>)`) // remove type parameters [<>] and size constraints [()]
var re = regexp.MustCompile(`(\(.+\)|<.+>)`) // remove type parameters [<>] and size constraints [()]

func columnTypeMapper(columnType base.ColumnType) string {
databaseTypeName := strings.ToUpper(re.ReplaceAllString(columnType.DatabaseTypeName(), ""))
if mappedType, ok := columnTypeMappings[strings.ToUpper(databaseTypeName)]; ok {
return mappedType
}
return databaseTypeName
}

// jsonRowMapper maps a row's scanned column to a json object's field
func jsonRowMapper(databaseTypeName string, value any) any {
switch v := value.(type) {
case []uint8:
return string(v)
case string:
switch databaseTypeName {
case "DECIMAL":
// convert to float
f, err := strconv.ParseFloat(v, 64)
if err != nil {
return v
}
return f
case "ARRAY", "STRUCT", "MAP": // convert string to json
var j any
err := json.Unmarshal([]byte(v), &j)
if err != nil {
return v
}
return j
}
}
return value
}
98 changes: 98 additions & 0 deletions sqlconnect/internal/databricks/testdata/column-mapping-test-columns.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
[
{
"name": "_order",
"type": "int"
},
{
"name": "_decimal",
"type": "int"
},
{
"name": "_numeric",
"type": "int"
},
{
"name": "_dec",
"type": "int"
},
{
"name": "_int",
"type": "int"
},
{
"name": "_integer",
"type": "int"
},
{
"name": "_bigint",
"type": "int"
},
{
"name": "_long",
"type": "int"
},
{
"name": "_smallint",
"type": "int"
},
{
"name": "_short",
"type": "int"
},
{
"name": "_tinyint",
"type": "int"
},
{
"name": "_byte",
"type": "int"
},
{
"name": "_float",
"type": "float"
},
{
"name": "_real",
"type": "float"
},
{
"name": "_double",
"type": "float"
},
{
"name": "_boolean",
"type": "boolean"
},
{
"name": "_string",
"type": "string"
},
{
"name": "_binary",
"type": "string"
},
{
"name": "_date",
"type": "datetime"
},
{
"name": "_timestamp",
"type": "datetime"
},
{
"name": "_timestampntz",
"type": "datetime"
},
{
"name": "_array",
"type": "json"
},
{
"name": "_map",
"type": "json"
},
{
"name": "_struct",
"type": "json"
}
]
Loading

0 comments on commit edce9eb

Please sign in to comment.