Merge branch 'main' into issue-1038

NOTICE.MD

@@ -287,6 +287,45 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+--------------------------------------------------------------------------------
+#### Module  : github.com/H0llyW00dzZ/cidr
+Version : v1.2.1
+Time    : 2024-03-27T02:04:51Z
+Licence : BSD-3-Clause
+
+Contents of probable licence file $GOMODCACHE/github.com/!h0lly!w00dz!z/[email protected]/LICENSE:
+
+BSD 3-Clause License
+
+Copyright (c) 2024, H0llyW00dzZ
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 #### Module  : github.com/antlr4-go/antlr/v4
 Version : v4.13.1
@@ -1564,11 +1603,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 #### Module  : github.com/jackc/pgx/v5
-Version : v5.7.1
-Time    : 2024-09-10T12:25:07Z
+Version : v5.7.2
+Time    : 2024-12-21T15:25:36Z
 Licence : MIT
 
-Contents of probable licence file $GOMODCACHE/github.com/jackc/pgx/[email protected].1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/jackc/pgx/[email protected].2/LICENSE:
 
 Copyright (c) 2013-2021 Jack Christensen
 

quesma/quesma/elastic_http_frontend_connector.go → quesma/frontend_connectors/elastic_http_frontend_connector.go

@@ -1,25 +1,19 @@
 // Copyright Quesma, licensed under the Elastic License 2.0.
 // SPDX-License-Identifier: Elastic-2.0
 
-package quesma
+package frontend_connectors
 
 import (
 	"context"
 	"net/http"
 	"quesma/clickhouse"
-	"quesma/frontend_connectors"
 	"quesma/quesma/config"
 	"quesma/schema"
 	quesma_api "quesma_v2/core"
-	"quesma_v2/core/diag"
 )
 
 type ElasticHttpIngestFrontendConnector struct {
-	*frontend_connectors.BasicHTTPFrontendConnector
-
-	Config *config.QuesmaConfiguration
-
-	phoneHomeClient diag.PhoneHomeClient
+	*BasicHTTPFrontendConnector
 }
 
 func NewElasticHttpIngestFrontendConnector(endpoint string,
@@ -28,7 +22,7 @@ func NewElasticHttpIngestFrontendConnector(endpoint string,
 	config *config.QuesmaConfiguration, router quesma_api.Router) *ElasticHttpIngestFrontendConnector {
 
 	fc := &ElasticHttpIngestFrontendConnector{
-		BasicHTTPFrontendConnector: frontend_connectors.NewBasicHTTPFrontendConnector(endpoint, config),
+		BasicHTTPFrontendConnector: NewBasicHTTPFrontendConnector(endpoint, config),
 	}
 	fallback := func(ctx context.Context, req *quesma_api.Request, writer http.ResponseWriter) (*quesma_api.Result, error) {
 		fc.BasicHTTPFrontendConnector.GetRouterInstance().ElasticFallback(req.Decision, ctx, writer, req.OriginalRequest, []byte(req.Body), logManager, registry)
@@ -41,23 +35,8 @@ func NewElasticHttpIngestFrontendConnector(endpoint string,
 	return fc
 }
 
-func (h *ElasticHttpIngestFrontendConnector) GetChildComponents() []interface{} {
-	components := make([]interface{}, 0)
-	if h.BasicHTTPFrontendConnector != nil {
-		components = append(components, h.BasicHTTPFrontendConnector)
-	}
-
-	return components
-}
-
-func (h *ElasticHttpIngestFrontendConnector) SetDependencies(deps quesma_api.Dependencies) {
-	h.phoneHomeClient = deps.PhoneHomeAgent()
-}
-
 type ElasticHttpQueryFrontendConnector struct {
-	*frontend_connectors.BasicHTTPFrontendConnector
-
-	phoneHomeClient diag.PhoneHomeClient
+	*BasicHTTPFrontendConnector
 }
 
 func NewElasticHttpQueryFrontendConnector(endpoint string,
@@ -66,7 +45,7 @@ func NewElasticHttpQueryFrontendConnector(endpoint string,
 	config *config.QuesmaConfiguration, router quesma_api.Router) *ElasticHttpIngestFrontendConnector {
 
 	fc := &ElasticHttpIngestFrontendConnector{
-		BasicHTTPFrontendConnector: frontend_connectors.NewBasicHTTPFrontendConnector(endpoint, config),
+		BasicHTTPFrontendConnector: NewBasicHTTPFrontendConnector(endpoint, config),
 	}
 	fallback := func(ctx context.Context, req *quesma_api.Request, writer http.ResponseWriter) (*quesma_api.Result, error) {
 		fc.BasicHTTPFrontendConnector.GetRouterInstance().ElasticFallback(req.Decision, ctx, writer, req.OriginalRequest, []byte(req.Body), logManager, registry)
@@ -76,15 +55,3 @@ func NewElasticHttpQueryFrontendConnector(endpoint string,
 	fc.AddRouter(router)
 	return fc
 }
-
-func (h *ElasticHttpQueryFrontendConnector) GetChildComponents() []interface{} {
-	components := make([]interface{}, 0)
-	if h.BasicHTTPFrontendConnector != nil {
-		components = append(components, h.BasicHTTPFrontendConnector)
-	}
-	return components
-}
-
-func (h *ElasticHttpQueryFrontendConnector) SetDependencies(deps quesma_api.Dependencies) {
-	h.phoneHomeClient = deps.PhoneHomeAgent()
-}

quesma/go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/gorilla/sessions v1.4.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/jackc/pgx/v4 v4.18.3
-	github.com/jackc/pgx/v5 v5.7.1
+	github.com/jackc/pgx/v5 v5.7.2
 	github.com/k0kubun/pp v3.0.1+incompatible
 	github.com/knadh/koanf/parsers/json v0.1.0
 	github.com/knadh/koanf/parsers/yaml v0.1.0
@@ -40,6 +40,7 @@ require (
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/H0llyW00dzZ/cidr v1.2.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect

quesma/go.sum

@@ -7,6 +7,8 @@ github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h
 github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo=
 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
 github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
+github.com/H0llyW00dzZ/cidr v1.2.1 h1:DfRHX+RqVVKZijQGO1aJSaWvN9Saan8sycK/4wrfY5g=
+github.com/H0llyW00dzZ/cidr v1.2.1/go.mod h1:S+EgYkMandSAN27mGNG/CB3jeoXDAyalsvvVFpWdnXc=
 github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U=
 github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
@@ -116,8 +118,8 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ
 github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
 github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA=
 github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw=
-github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
-github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
+github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI=
+github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=

quesma/logger/log_with_throttling.go

@@ -10,7 +10,7 @@ import (
 
 // throttleMap: (reason name -> last logged time)
 // We log only once per throttleDuration for each reason name, so that we don't spam the logs.
-var throttleMap = util.SyncMap[string, time.Time]{}
+var throttleMap = util.NewSyncMap[string, time.Time]()
 
 const throttleDuration = 30 * time.Minute
 

quesma/main_test.go

@@ -25,14 +25,20 @@ func Test_Main(m *testing.T) {
 	_ = buildIngestOnlyQuesma()
 }
 
-func emitRequests(stop chan os.Signal) {
+func emitRequests(stop chan os.Signal, t *testing.T, testData []struct {
+	url              string
+	expectedResponse string
+}) {
 	go func() {
 		time.Sleep(1 * time.Second)
 		requestBody := []byte(`{"query": {"match_all": {}}}`)
-		sendRequest("http://localhost:8888/_bulk", requestBody)
-		sendRequest("http://localhost:8888/_doc", requestBody)
-		sendRequest("http://localhost:8888/_search", requestBody)
-		sendRequest("http://localhost:8888/_search", requestBody)
+		var resp string
+		var err error
+		for _, test := range testData {
+			resp, err = sendRequest(test.url, requestBody)
+			assert.NoError(t, err)
+			assert.Contains(t, test.expectedResponse, resp)
+		}
 		signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
 		close(stop)
 	}()
@@ -144,7 +150,16 @@ func Test_fallbackScenario(t *testing.T) {
 	q1, _ := qBuilder.Build()
 	q1.Start()
 	stop := make(chan os.Signal, 1)
-	emitRequests(stop)
+	testData := []struct {
+		url              string
+		expectedResponse string
+	}{
+		{"http://localhost:8888/_bulk", "unknown\n"},
+		{"http://localhost:8888/_doc", "unknown\n"},
+		{"http://localhost:8888/_search", "unknown\n"},
+		{"http://localhost:8888/_search", "unknown\n"},
+	}
+	emitRequests(stop, t, testData)
 	<-stop
 	q1.Stop(context.Background())
 	atomic.LoadInt32(&fallbackCalled)
@@ -155,7 +170,20 @@ func Test_scenario1(t *testing.T) {
 	q1 := ab_testing_scenario()
 	q1.Start()
 	stop := make(chan os.Signal, 1)
-	emitRequests(stop)
+	testData := []struct {
+		url              string
+		expectedResponse string
+	}{
+		{"http://localhost:8888/_bulk", `bulk->IngestProcessor->InnerIngestProcessor1->0ABIngestTestProcessor
+bulk->IngestProcessor->InnerIngestProcessor2->0ABIngestTestProcessor
+`},
+		{"http://localhost:8888/_doc", `doc->IngestProcessor->InnerIngestProcessor1->0ABIngestTestProcessor
+doc->IngestProcessor->InnerIngestProcessor2->0ABIngestTestProcessor
+`},
+		{"http://localhost:8888/_search", "ABTestProcessor processor: Responses are equal\n"},
+		{"http://localhost:8888/_search", "ABTestProcessor processor: Responses are not equal\n"},
+	}
+	emitRequests(stop, t, testData)
 	<-stop
 	q1.Stop(context.Background())
 }
@@ -215,7 +243,17 @@ func Test_middleware(t *testing.T) {
 		quesmaBuilder.Build()
 		quesmaBuilder.Start()
 		stop := make(chan os.Signal, 1)
-		emitRequests(stop)
+		testData := []struct {
+			url              string
+			expectedResponse string
+		}{
+			{"http://localhost:8888/_bulk", "middleware\n"},
+			{"http://localhost:8888/_doc", "middleware\n"},
+			{"http://localhost:8888/_search", "middleware\n"},
+			{"http://localhost:8888/_search", "middleware\n"},
+		}
+		emitRequests(stop, t, testData)
+
 		<-stop
 		quesmaBuilder.Stop(context.Background())
 		atomic.LoadInt32(&middlewareCallCount)
@@ -227,7 +265,16 @@ func Test_middleware(t *testing.T) {
 		quesmaBuilder.Build()
 		quesmaBuilder.Start()
 		stop := make(chan os.Signal, 1)
-		emitRequests(stop)
+		testData := []struct {
+			url              string
+			expectedResponse string
+		}{
+			{"http://localhost:8888/_bulk", "middleware\n"},
+			{"http://localhost:8888/_doc", "middleware\n"},
+			{"http://localhost:8888/_search", "middleware\n"},
+			{"http://localhost:8888/_search", "middleware\n"},
+		}
+		emitRequests(stop, t, testData)
 		<-stop
 		quesmaBuilder.Stop(context.Background())
 		atomic.LoadInt32(&middlewareCallCount)