diff --git a/quesma/ab_testing/collector/diff.go b/quesma/ab_testing/collector/diff.go index d5095cc46..dfcc461db 100644 --- a/quesma/ab_testing/collector/diff.go +++ b/quesma/ab_testing/collector/diff.go @@ -4,8 +4,8 @@ package collector import ( "crypto/sha1" - "encoding/json" "fmt" + "github.com/goccy/go-json" "quesma/jsondiff" "quesma/quesma/types" ) diff --git a/quesma/ab_testing/collector/fanout.go b/quesma/ab_testing/collector/fanout.go index cfc9478f4..f5fe07240 100644 --- a/quesma/ab_testing/collector/fanout.go +++ b/quesma/ab_testing/collector/fanout.go @@ -5,8 +5,8 @@ package collector import ( "bytes" "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" "net/http" "quesma/ingest" "quesma/logger" diff --git a/quesma/ab_testing/collector/processors.go b/quesma/ab_testing/collector/processors.go index 275f98a94..676c248f4 100644 --- a/quesma/ab_testing/collector/processors.go +++ b/quesma/ab_testing/collector/processors.go @@ -3,8 +3,8 @@ package collector import ( - "encoding/json" "fmt" + "github.com/goccy/go-json" "quesma/quesma/types" "regexp" ) diff --git a/quesma/ast/query_traversal_test.go b/quesma/ast/query_traversal_test.go index fcfc1856e..57df7cd2a 100644 --- a/quesma/ast/query_traversal_test.go +++ b/quesma/ast/query_traversal_test.go @@ -3,8 +3,8 @@ package ast import ( - "encoding/json" "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "testing" diff --git a/quesma/buildinfo/build.go b/quesma/buildinfo/build.go index 1f966e27c..37d91c8e2 100644 --- a/quesma/buildinfo/build.go +++ b/quesma/buildinfo/build.go @@ -3,9 +3,9 @@ package buildinfo import ( - "encoding/json" "fmt" "github.com/coreos/go-semver/semver" + "github.com/goccy/go-json" "net/http" "time" ) diff --git a/quesma/clickhouse/table_discovery.go b/quesma/clickhouse/table_discovery.go index 313bef8e6..0f35fa624 100644 --- a/quesma/clickhouse/table_discovery.go +++ b/quesma/clickhouse/table_discovery.go @@ -5,9 +5,9 @@ package clickhouse import ( "context" "database/sql" - "encoding/json" "errors" "fmt" + "github.com/goccy/go-json" "quesma/common_table" "quesma/end_user_errors" "quesma/logger" diff --git a/quesma/clickhouse/util.go b/quesma/clickhouse/util.go index b112778f3..4fec7bd5a 100644 --- a/quesma/clickhouse/util.go +++ b/quesma/clickhouse/util.go @@ -4,8 +4,8 @@ package clickhouse import ( "bytes" - "encoding/json" "fmt" + "github.com/goccy/go-json" "quesma/logger" "quesma/model" "strings" diff --git a/quesma/elasticsearch/index_resolver.go b/quesma/elasticsearch/index_resolver.go index dd70ed1bf..b63f31d51 100644 --- a/quesma/elasticsearch/index_resolver.go +++ b/quesma/elasticsearch/index_resolver.go @@ -4,7 +4,7 @@ package elasticsearch import ( "context" - "encoding/json" + "github.com/goccy/go-json" "io" "net/http" "quesma/quesma/config" diff --git a/quesma/eql/e2e/ingest_integration_test.go b/quesma/eql/e2e/ingest_integration_test.go index 3177adc35..6cf7481ff 100644 --- a/quesma/eql/e2e/ingest_integration_test.go +++ b/quesma/eql/e2e/ingest_integration_test.go @@ -6,8 +6,8 @@ package e2e import ( "bytes" - "encoding/json" "fmt" + "github.com/goccy/go-json" "log" "net/http" "time" diff --git a/quesma/eql/e2e/query_integration_test.go b/quesma/eql/e2e/query_integration_test.go index 4aaa17c3d..d2d857930 100644 --- a/quesma/eql/e2e/query_integration_test.go +++ b/quesma/eql/e2e/query_integration_test.go @@ -6,8 +6,8 @@ package e2e import ( "bytes" - "encoding/json" "fmt" + "github.com/goccy/go-json" "io" "net/http" "quesma/jsonprocessor" diff --git a/quesma/go.mod b/quesma/go.mod index 826a78648..e874c7662 100644 --- a/quesma/go.mod +++ b/quesma/go.mod @@ -31,6 +31,7 @@ require ( require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/goccy/go-json v0.10.3 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect diff --git a/quesma/go.sum b/quesma/go.sum index 993505b17..9c539f313 100644 --- a/quesma/go.sum +++ b/quesma/go.sum @@ -32,6 +32,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= diff --git a/quesma/health/elastic.go b/quesma/health/elastic.go index a3cc43431..db3d14ba3 100644 --- a/quesma/health/elastic.go +++ b/quesma/health/elastic.go @@ -4,8 +4,8 @@ package health import ( "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" "io" "net/http" "quesma/elasticsearch" diff --git a/quesma/ingest/common_table_test.go b/quesma/ingest/common_table_test.go index 48f902f03..b872d153c 100644 --- a/quesma/ingest/common_table_test.go +++ b/quesma/ingest/common_table_test.go @@ -4,8 +4,8 @@ package ingest import ( "context" - "encoding/json" "github.com/DATA-DOG/go-sqlmock" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "quesma/clickhouse" "quesma/common_table" diff --git a/quesma/ingest/processor.go b/quesma/ingest/processor.go index c8a32dddb..5f282d596 100644 --- a/quesma/ingest/processor.go +++ b/quesma/ingest/processor.go @@ -5,9 +5,9 @@ package ingest import ( "context" "database/sql" - "encoding/json" "fmt" "github.com/ClickHouse/clickhouse-go/v2" + "github.com/goccy/go-json" chLib "quesma/clickhouse" "quesma/comment_metadata" "quesma/common_table" @@ -490,32 +490,14 @@ func (ip *IngestProcessor) GenerateIngestContent(table *chLib.Table, config *chLib.ChTableConfig, encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) ([]string, types.JSON, []NonSchemaField, error) { - jsonAsBytesSlice, err := json.Marshal(data) - - if err != nil { - return nil, nil, nil, err - } - - // we find all non-schema fields - jsonMap, err := types.ParseJSON(string(jsonAsBytesSlice)) - if err != nil { - return nil, nil, nil, err - } - if len(config.Attributes) == 0 { - return nil, jsonMap, nil, nil - } - - schemaFieldsJson, err := json.Marshal(jsonMap) - - if err != nil { - return nil, jsonMap, nil, err + return nil, data, nil, nil } - mDiff := DifferenceMap(jsonMap, table) // TODO change to DifferenceMap(m, t) + mDiff := DifferenceMap(data, table) // TODO change to DifferenceMap(m, t) - if len(mDiff) == 0 && string(schemaFieldsJson) == string(jsonAsBytesSlice) && len(inValidJson) == 0 { // no need to modify, just insert 'js' - return nil, jsonMap, nil, nil + if len(mDiff) == 0 && len(inValidJson) == 0 { // no need to modify, just insert 'js' + return nil, data, nil, nil } // check attributes precondition @@ -546,7 +528,7 @@ func (ip *IngestProcessor) GenerateIngestContent(table *chLib.Table, return nil, nil, nil, err } - onlySchemaFields := RemoveNonSchemaFields(jsonMap, table) + onlySchemaFields := RemoveNonSchemaFields(data, table) return alterCmd, onlySchemaFields, nonSchemaFields, nil } diff --git a/quesma/ingest/processor_test.go b/quesma/ingest/processor_test.go index e636ff1ec..f351635ee 100644 --- a/quesma/ingest/processor_test.go +++ b/quesma/ingest/processor_test.go @@ -3,7 +3,7 @@ package ingest import ( - "encoding/json" + "github.com/goccy/go-json" "quesma/clickhouse" "quesma/persistence" "quesma/quesma/config" diff --git a/quesma/ingest/util.go b/quesma/ingest/util.go index de2e21502..57e1eac42 100644 --- a/quesma/ingest/util.go +++ b/quesma/ingest/util.go @@ -4,8 +4,8 @@ package ingest import ( "bytes" - "encoding/json" "fmt" + "github.com/goccy/go-json" "quesma/clickhouse" "strings" ) diff --git a/quesma/jsonprocessor/json_processor_test.go b/quesma/jsonprocessor/json_processor_test.go index 56bc1076e..33b5ac860 100644 --- a/quesma/jsonprocessor/json_processor_test.go +++ b/quesma/jsonprocessor/json_processor_test.go @@ -3,7 +3,7 @@ package jsonprocessor import ( - "encoding/json" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "quesma/quesma/types" "reflect" diff --git a/quesma/licensing/license_manager.go b/quesma/licensing/license_manager.go index ff6080112..abdd1ff46 100644 --- a/quesma/licensing/license_manager.go +++ b/quesma/licensing/license_manager.go @@ -4,8 +4,8 @@ package licensing import ( "bytes" - "encoding/json" "fmt" + "github.com/goccy/go-json" "io" "net/http" "time" diff --git a/quesma/model/query_result.go b/quesma/model/query_result.go index bfb0154d9..5648e7a1c 100644 --- a/quesma/model/query_result.go +++ b/quesma/model/query_result.go @@ -4,8 +4,8 @@ package model import ( "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" "quesma/common_table" "quesma/logger" "quesma/schema" diff --git a/quesma/model/search_response.go b/quesma/model/search_response.go index dbae60928..d325fcf16 100644 --- a/quesma/model/search_response.go +++ b/quesma/model/search_response.go @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Elastic-2.0 package model -import "encoding/json" +import "github.com/goccy/go-json" type JsonMap = map[string]interface{} diff --git a/quesma/persistence/elastic.go b/quesma/persistence/elastic.go index aa12fae76..ba9e959c9 100644 --- a/quesma/persistence/elastic.go +++ b/quesma/persistence/elastic.go @@ -4,8 +4,8 @@ package persistence import ( "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" "io" "log" "net/http" diff --git a/quesma/proxy/l4_proxy_test.go b/quesma/proxy/l4_proxy_test.go index f01213840..e983a9a55 100644 --- a/quesma/proxy/l4_proxy_test.go +++ b/quesma/proxy/l4_proxy_test.go @@ -4,8 +4,8 @@ package proxy import ( "bytes" - "encoding/json" "fmt" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "log" "net" diff --git a/quesma/queryparser/dashboard_error.go b/quesma/queryparser/dashboard_error.go index 7f354a320..417b60850 100644 --- a/quesma/queryparser/dashboard_error.go +++ b/quesma/queryparser/dashboard_error.go @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Elastic-2.0 package queryparser -import "encoding/json" +import "github.com/goccy/go-json" func BadRequestParseError(err error) []byte { serialized, _ := json.Marshal(DashboardErrorResponse{ diff --git a/quesma/queryparser/query_parser.go b/quesma/queryparser/query_parser.go index 4307b3e0a..11b3415fd 100644 --- a/quesma/queryparser/query_parser.go +++ b/quesma/queryparser/query_parser.go @@ -6,8 +6,8 @@ package queryparser import ( "context" "encoding/hex" - "encoding/json" "fmt" + "github.com/goccy/go-json" "github.com/k0kubun/pp" "quesma/clickhouse" "quesma/logger" diff --git a/quesma/queryparser/query_translator_test.go b/quesma/queryparser/query_translator_test.go index 819e79d34..84c14f63e 100644 --- a/quesma/queryparser/query_translator_test.go +++ b/quesma/queryparser/query_translator_test.go @@ -4,7 +4,7 @@ package queryparser import ( "context" - "encoding/json" + "github.com/goccy/go-json" "github.com/k0kubun/pp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/quesma/quesma/config/env2json_test.go b/quesma/quesma/config/env2json_test.go index d222bcc6f..09747eb6b 100644 --- a/quesma/quesma/config/env2json_test.go +++ b/quesma/quesma/config/env2json_test.go @@ -3,7 +3,7 @@ package config import ( - "encoding/json" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "os" "testing" diff --git a/quesma/quesma/functionality/bulk/bulk.go b/quesma/quesma/functionality/bulk/bulk.go index 1b2b93f7c..4452cbd88 100644 --- a/quesma/quesma/functionality/bulk/bulk.go +++ b/quesma/quesma/functionality/bulk/bulk.go @@ -4,8 +4,8 @@ package bulk import ( "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" "io" "net/http" "quesma/clickhouse" diff --git a/quesma/quesma/functionality/bulk/bulk_test.go b/quesma/quesma/functionality/bulk/bulk_test.go index d215cb38d..c0dea5a71 100644 --- a/quesma/quesma/functionality/bulk/bulk_test.go +++ b/quesma/quesma/functionality/bulk/bulk_test.go @@ -3,7 +3,7 @@ package bulk import ( - "encoding/json" + "github.com/goccy/go-json" "github.com/stretchr/testify/require" "testing" ) diff --git a/quesma/quesma/functionality/field_capabilities/field_caps.go b/quesma/quesma/functionality/field_capabilities/field_caps.go index 02a4f6c66..d205f2505 100644 --- a/quesma/quesma/functionality/field_capabilities/field_caps.go +++ b/quesma/quesma/functionality/field_capabilities/field_caps.go @@ -4,8 +4,8 @@ package field_capabilities import ( "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" "quesma/clickhouse" "quesma/elasticsearch" "quesma/elasticsearch/elasticsearch_field_types" diff --git a/quesma/quesma/functionality/field_capabilities/field_caps_test.go b/quesma/quesma/functionality/field_capabilities/field_caps_test.go index d477d8527..ae43eaca3 100644 --- a/quesma/quesma/functionality/field_capabilities/field_caps_test.go +++ b/quesma/quesma/functionality/field_capabilities/field_caps_test.go @@ -3,7 +3,7 @@ package field_capabilities import ( - "encoding/json" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "quesma/clickhouse" "quesma/model" diff --git a/quesma/quesma/functionality/terms_enum/terms_enum.go b/quesma/quesma/functionality/terms_enum/terms_enum.go index f92ff8987..0a2569c2f 100644 --- a/quesma/quesma/functionality/terms_enum/terms_enum.go +++ b/quesma/quesma/functionality/terms_enum/terms_enum.go @@ -4,9 +4,9 @@ package terms_enum import ( "context" - "encoding/json" "errors" "fmt" + "github.com/goccy/go-json" "quesma/clickhouse" "quesma/end_user_errors" "quesma/logger" diff --git a/quesma/quesma/functionality/terms_enum/terms_enum_test.go b/quesma/quesma/functionality/terms_enum/terms_enum_test.go index a43473245..d5030fc1b 100644 --- a/quesma/quesma/functionality/terms_enum/terms_enum_test.go +++ b/quesma/quesma/functionality/terms_enum/terms_enum_test.go @@ -5,9 +5,9 @@ package terms_enum import ( "bytes" "context" - "encoding/json" "fmt" "github.com/DATA-DOG/go-sqlmock" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "quesma/clickhouse" "quesma/logger" diff --git a/quesma/quesma/highlight_test.go b/quesma/quesma/highlight_test.go index 105dba184..58d63dc60 100644 --- a/quesma/quesma/highlight_test.go +++ b/quesma/quesma/highlight_test.go @@ -4,7 +4,7 @@ package quesma import ( "context" - "encoding/json" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "quesma/clickhouse" "quesma/model" diff --git a/quesma/quesma/mappings_test.go b/quesma/quesma/mappings_test.go index 1ccaf7994..5804302da 100644 --- a/quesma/quesma/mappings_test.go +++ b/quesma/quesma/mappings_test.go @@ -3,7 +3,7 @@ package quesma import ( - "encoding/json" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "quesma/elasticsearch" diff --git a/quesma/quesma/router.go b/quesma/quesma/router.go index 169a7e1c5..b14931ce0 100644 --- a/quesma/quesma/router.go +++ b/quesma/quesma/router.go @@ -4,8 +4,8 @@ package quesma import ( "context" - "encoding/json" "errors" + "github.com/goccy/go-json" "net/http" "quesma/clickhouse" "quesma/elasticsearch" diff --git a/quesma/quesma/search_ab_testing.go b/quesma/quesma/search_ab_testing.go index 90ecf0583..198e03f44 100644 --- a/quesma/quesma/search_ab_testing.go +++ b/quesma/quesma/search_ab_testing.go @@ -4,8 +4,8 @@ package quesma import ( "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" "io" "net/http" "quesma/ab_testing" diff --git a/quesma/quesma/search_test.go b/quesma/quesma/search_test.go index f5cd10ba7..87f2627f4 100644 --- a/quesma/quesma/search_test.go +++ b/quesma/quesma/search_test.go @@ -6,9 +6,9 @@ import ( "context" "database/sql" "database/sql/driver" - "encoding/json" "fmt" "github.com/DATA-DOG/go-sqlmock" + "github.com/goccy/go-json" "github.com/k0kubun/pp" "github.com/stretchr/testify/assert" "quesma/clickhouse" diff --git a/quesma/quesma/types/json.go b/quesma/quesma/types/json.go index faa6fec45..3730cf4b6 100644 --- a/quesma/quesma/types/json.go +++ b/quesma/quesma/types/json.go @@ -3,9 +3,9 @@ package types import ( - "encoding/json" "fmt" "github.com/barkimedes/go-deepcopy" + "github.com/goccy/go-json" "github.com/tailscale/hujson" ) @@ -39,14 +39,6 @@ func (j JSON) Bytes() ([]byte, error) { return json.Marshal(j) } -func (j JSON) Remarshal(v interface{}) error { - b, err := json.Marshal(j) - if err != nil { - return err - } - return json.Unmarshal(b, v) -} - func (j JSON) ShortString() string { var asString string diff --git a/quesma/quesma/types/json_test.go b/quesma/quesma/types/json_test.go index 363f0840d..5ccd050b9 100644 --- a/quesma/quesma/types/json_test.go +++ b/quesma/quesma/types/json_test.go @@ -3,7 +3,6 @@ package types import ( - "encoding/json" "github.com/stretchr/testify/assert" "testing" ) @@ -20,36 +19,6 @@ func TestCommentedJson(t *testing.T) { assert.Equal(t, jsonStr, withoutComment) } -func TestReMarshalJSON(t *testing.T) { - - type dest struct { - Key1 string `json:"key1"` - Key2 string `json:"key2"` - } - - // given - jsonStr := `{"key1":"value1","key2":"value2"}` - - var jsonData JSON - - err := json.Unmarshal([]byte(jsonStr), &jsonData) - if err != nil { - t.Fatal(err) - } - - // when - var destData dest - err = jsonData.Remarshal(&destData) - if err != nil { - t.Fatal(err) - } - - // then - - assert.Equal(t, "value1", destData.Key1) - assert.Equal(t, "value2", destData.Key2) -} - func TestJSONClone(t *testing.T) { simpleJson := JSON{"key1": "value1", "key2": "value2"} clonedA := simpleJson.Clone() @@ -66,3 +35,39 @@ func TestJSONClone(t *testing.T) { delete(arrayJson["key2"].([]JSON)[0], "key3") assert.Equal(t, "value3", clonedC["key2"].([]JSON)[0]["key3"]) } + +func TestNDJSON_BulkForEach(t *testing.T) { + + input := `{"create":{"_index":"kibana_sample_data_flights", "_id": "1"}} +{"FlightNum":"9HY9SWR","DestCountry":"AU","OriginWeather":"Sunny","OriginCityName":"Frankfurt am Main" } +{"create":{"_index":"kibana_sample_data_flights", "_id": "2"}} +{"FlightNum":"FOO","DestCountry":"BAR","OriginWeather":"BAZ","OriginCityName":"QUIX" } +` + + ndjson, err := ParseNDJSON(input) + assert.NoError(t, err) + + err = ndjson.BulkForEach(func(entryNumber int, operationParsed BulkOperation, operation JSON, doc JSON) error { + + switch entryNumber { + + case 0: + assert.Equal(t, "create", operationParsed.GetOperation()) + assert.Equal(t, "kibana_sample_data_flights", operationParsed.GetIndex()) + assert.Equal(t, "9HY9SWR", doc["FlightNum"]) + + case 1: + assert.Equal(t, "create", operationParsed.GetOperation()) + assert.Equal(t, "kibana_sample_data_flights", operationParsed.GetIndex()) + assert.Equal(t, "FOO", doc["FlightNum"]) + + default: + t.Errorf("Unexpected entry number: %d", entryNumber) + } + + return nil + }) + + assert.NoError(t, err) + +} diff --git a/quesma/quesma/types/ndjson.go b/quesma/quesma/types/ndjson.go index d56b06cef..4f4ebd6b9 100644 --- a/quesma/quesma/types/ndjson.go +++ b/quesma/quesma/types/ndjson.go @@ -3,8 +3,8 @@ package types import ( - "encoding/json" "fmt" + "github.com/goccy/go-json" "strings" ) @@ -69,9 +69,22 @@ func (n NDJSON) BulkForEach(f func(entryNumber int, operationParsed BulkOperatio operation := n[i] // {"create":{"_index":"kibana_sample_data_flights", "_id": 1}} document := n[i+1] // {"FlightNum":"9HY9SWR","DestCountry":"AU","OriginWeather":"Sunny","OriginCityName":"Frankfurt am Main" } - var operationParsed BulkOperation // operationName (create, index, update, delete) -> DocumentTarget + operationParsed := make(map[string]DocumentTarget) - _ = operation.Remarshal(&operationParsed) // ignore error, the callback must handle it (it will see an unknown operation) + for k, v := range operation { + d := DocumentTarget{} + if op, ok := v.(map[string]interface{}); ok { + if index, ok := op["_index"].(string); ok { + d.Index = &index + } + + if id, ok := op["_id"].(string); ok { + d.Id = &id + } + } + + operationParsed[k] = d + } err := f(i/2, operationParsed, operation, document) if err != nil { diff --git a/quesma/quesma/ui/ab_testing.go b/quesma/quesma/ui/ab_testing.go index 5f2c2a511..17829d79b 100644 --- a/quesma/quesma/ui/ab_testing.go +++ b/quesma/quesma/ui/ab_testing.go @@ -4,8 +4,8 @@ package ui import ( "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" "io" "quesma/elasticsearch" "quesma/jsondiff" diff --git a/quesma/quesma/ui/console_routes.go b/quesma/quesma/ui/console_routes.go index a6a39a5fe..c1b108357 100644 --- a/quesma/quesma/ui/console_routes.go +++ b/quesma/quesma/ui/console_routes.go @@ -5,8 +5,8 @@ package ui import ( "embed" - "encoding/json" "errors" + "github.com/goccy/go-json" "github.com/gorilla/mux" "github.com/gorilla/securecookie" "github.com/gorilla/sessions" diff --git a/quesma/quesma/ui/es_auth_provider.go b/quesma/quesma/ui/es_auth_provider.go index 12b2afb59..64dbfd3d0 100644 --- a/quesma/quesma/ui/es_auth_provider.go +++ b/quesma/quesma/ui/es_auth_provider.go @@ -4,7 +4,7 @@ package ui import ( - "encoding/json" + "github.com/goccy/go-json" "github.com/markbates/goth" "golang.org/x/oauth2" ) diff --git a/quesma/quesma/ui/live_tail_drilldown.go b/quesma/quesma/ui/live_tail_drilldown.go index d6d86a4b1..02de4e743 100644 --- a/quesma/quesma/ui/live_tail_drilldown.go +++ b/quesma/quesma/ui/live_tail_drilldown.go @@ -4,8 +4,8 @@ package ui import ( "encoding/base64" - "encoding/json" "fmt" + "github.com/goccy/go-json" "gopkg.in/yaml.v3" "quesma/quesma/types" "quesma/quesma/ui/internal/builder" diff --git a/quesma/quesma/ui/telemetry.go b/quesma/quesma/ui/telemetry.go index f88a8f118..827ff3394 100644 --- a/quesma/quesma/ui/telemetry.go +++ b/quesma/quesma/ui/telemetry.go @@ -3,7 +3,7 @@ package ui import ( - "encoding/json" + "github.com/goccy/go-json" "quesma/logger" ) diff --git a/quesma/stats/ingest_statistics_test.go b/quesma/stats/ingest_statistics_test.go index 95db20415..f821d3ef8 100644 --- a/quesma/stats/ingest_statistics_test.go +++ b/quesma/stats/ingest_statistics_test.go @@ -3,7 +3,7 @@ package stats import ( - "encoding/json" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "quesma/quesma/config" "quesma/quesma/types" diff --git a/quesma/telemetry/phone_home.go b/quesma/telemetry/phone_home.go index 6e73d3f76..8d44dceb1 100644 --- a/quesma/telemetry/phone_home.go +++ b/quesma/telemetry/phone_home.go @@ -7,8 +7,8 @@ import ( "context" "crypto/tls" "database/sql" - "encoding/json" "fmt" + "github.com/goccy/go-json" "github.com/google/uuid" "github.com/shirou/gopsutil/v3/mem" "io" diff --git a/quesma/telemetry/phone_home_test.go b/quesma/telemetry/phone_home_test.go index d05bddec3..71ed80ef6 100644 --- a/quesma/telemetry/phone_home_test.go +++ b/quesma/telemetry/phone_home_test.go @@ -3,7 +3,7 @@ package telemetry import ( - "encoding/json" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "testing" ) diff --git a/quesma/util/utils.go b/quesma/util/utils.go index 8d9197873..fdb377cfa 100644 --- a/quesma/util/utils.go +++ b/quesma/util/utils.go @@ -6,9 +6,9 @@ import ( "bytes" "database/sql" "encoding/base64" - "encoding/json" "fmt" "github.com/DATA-DOG/go-sqlmock" + "github.com/goccy/go-json" "github.com/hashicorp/go-multierror" "github.com/k0kubun/pp" "io" @@ -914,7 +914,29 @@ func ExtractUsernameFromBasicAuthHeader(authHeader string) (string, error) { return pair[0], nil } +var patternCache = make(map[string]*regexp.Regexp) +var patternCacheLock = sync.RWMutex{} + func TableNamePatternRegexp(indexPattern string) *regexp.Regexp { + + patternCacheLock.RLock() + + pattern, ok := patternCache[indexPattern] + if ok { + patternCacheLock.RUnlock() + return pattern + } + + patternCacheLock.RUnlock() + patternCacheLock.Lock() + defer patternCacheLock.Unlock() + + // Clear cache if it's too big + const maxPatternCacheSize = 1000 + if len(patternCache) > maxPatternCacheSize { + patternCache = make(map[string]*regexp.Regexp) + } + var builder strings.Builder for _, char := range indexPattern { @@ -929,5 +951,7 @@ func TableNamePatternRegexp(indexPattern string) *regexp.Regexp { } } - return regexp.MustCompile(fmt.Sprintf("^%s$", builder.String())) + result := regexp.MustCompile(fmt.Sprintf("^%s$", builder.String())) + patternCache[indexPattern] = result + return result } diff --git a/quesma/util/utils_test.go b/quesma/util/utils_test.go index 914b889c2..9c0744306 100644 --- a/quesma/util/utils_test.go +++ b/quesma/util/utils_test.go @@ -3,8 +3,8 @@ package util import ( - "encoding/json" "fmt" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "net/http" "reflect"