Support for "unknown" types (#121)

If table has a column with not supported type entire table was skipped. 

Change. Resolve not supported types as "UnknownType". Kibana renders
these fields quite good.


There is a new index `type_logs`. Index will be used for type testings.


<img width="1719" alt="Screenshot 2024-05-15 at 13 52 58"
src="https://github.com/QuesmaOrg/quesma/assets/1474/dad66018-d8c3-402e-a5d2-ebfe81c53cd1">

<img width="1363" alt="Screenshot 2024-05-15 at 14 00 15"
src="https://github.com/QuesmaOrg/quesma/assets/1474/b52e1a54-91f5-4243-a551-7307c04eb9ad">


<img width="1715" alt="Screenshot 2024-05-15 at 13 53 24"
src="https://github.com/QuesmaOrg/quesma/assets/1474/8d845050-dcaf-4b9d-95ee-5ecdfd604547">

docker/clean-clickhouse/schema/03-type_logs.sql

@@ -0,0 +1,25 @@
+
+CREATE TABLE IF NOT EXISTS "type_logs"
+(
+    "attributes_string_key" Array(String),
+    "attributes_string_value" Array(String),
+
+    "@timestamp" DateTime64 DEFAULT now64(),
+
+    `map_string_string` Map(String, Nullable(String)),
+    `point` Point,
+    `ipv4`  IPv4,
+    `ipv6`  IPv6,
+)
+ENGINE = MergeTree
+ORDER BY ("@timestamp")
+COMMENT 'Table for type testing. Created by clean-clickhouse.'
+;
+
+
+
+
+
+
+
+

docker/clean-clickhouse/schema/04-type-logs-insert.sql

@@ -0,0 +1,4 @@
+insert into type_logs(map_string_string, point,ipv4,ipv6) values
+ ({'key1': 'key2', 'key3': 'value3'},  (0,0), '127.0.0.1', '2a02:aa08:e000:3100::2')
+ ({'foo': 'bar'}, (1,1), '10.10.10.10', '2001:44c8:129:2632:33:0:252:2')
+;

docker/clean-clickhouse/schema/README.md

@@ -1,2 +1,3 @@
 This directory contains scripts (.sql) to initialize the Clickhouse database on startup.
 
+Clickhouse doesn't support multiple queries. So one file per query.

docker/kibana/add_sample_data.sh

@@ -132,6 +132,18 @@ do_silent_http_post "api/data_views/data_view" '{
     "override": true
 }'
 
+echo -n "Adding data view Type Logs... "
+do_silent_http_post "api/data_views/data_view" '{
+    "data_view": {
+       "name": "Type Logs",
+       "title": "type_logs",
+       "id": "type_logs",
+       "timeFieldName": "@timestamp",
+       "allowNoIndex": true
+    },
+    "override": true
+}'
+
 echo ""
 
 echo ""

docker/quesma/config/local-dev.yaml

@@ -54,4 +54,6 @@ indexes:
   opensearch_dashboards_sample_data_flights:
     enabled: true
   opensearch_dashboards_sample_data_logs:
-    enabled: true
+    enabled: true
+  type_logs:
+    enabled: true

http_requests/search-type-logs.http

@@ -0,0 +1,53 @@
+POST http://localhost:8080/type_logs/_search
+Content-Type: application/json
+
+
+{
+	"_source": false,
+	"fields": [
+		{
+			"field": "*",
+			"include_unmapped": "true"
+		},
+		{
+			"field": "@timestamp",
+			"format": "strict_date_optional_time"
+		}
+	],
+	"highlight": {
+
+	},
+	"query": {
+		"bool": {
+			"filter": [
+
+			],
+			"must": [],
+			"must_not": [],
+			"should": []
+		}
+	},
+	"runtime_mappings": {},
+	"script_fields": {},
+	"size": 500,
+	"sort": [
+		{
+			"@timestamp": {
+				"format": "strict_date_optional_time",
+				"order": "desc",
+				"unmapped_type": "boolean"
+			}
+		},
+		{
+			"_doc": {
+				"order": "desc",
+				"unmapped_type": "boolean"
+			}
+		}
+	],
+	"stored_fields": [
+		"*"
+	],
+	"track_total_hits": false,
+	"version": true
+}

quesma/clickhouse/schema.go

@@ -191,6 +191,9 @@ func NewBaseType(clickHouseTypeName string) BaseType {
 	return BaseType{Name: clickHouseTypeName, goType: goType}
 }
 
+// this is catch all type for all types we do not exlicitly support
+type UnknownType struct{}
+
 func ResolveType(clickHouseTypeName string) reflect.Type {
 	switch clickHouseTypeName {
 	case "String", "LowCardinality(String)", "UUID":
@@ -209,7 +212,10 @@ func ResolveType(clickHouseTypeName string) reflect.Type {
 		return reflect.TypeOf(true)
 	case "JSON":
 		return reflect.TypeOf(map[string]interface{}{})
+	case "Unknown":
+		return reflect.TypeOf(UnknownType{})
 	}
+
 	return nil
 }
 

quesma/clickhouse/schema_loader.go

@@ -206,8 +206,16 @@ func resolveColumn(colName, colType string) *Column {
 			},
 		}
 	} else {
-		logger.Error().Msgf("unknown type: %s, in column: %s, resolving to nil", colType, colName)
-		return nil
+		logger.Warn().Msgf("unknown type for column %s, %s", colName, colType)
+		typeName := "Unknown(" + colType + ")"
+		return &Column{
+			Name: colName,
+			Type: BaseType{
+				Name:     typeName,
+				goType:   NewBaseType("Unknown").goType,
+				Nullable: isNullable,
+			},
+		}
 	}
 }