Merge remote-tracking branch 'origin/e2e-test' into e2e-test

NOTICE.MD

@@ -1594,11 +1594,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 #### Module  : github.com/knadh/koanf/providers/env
-Version : v0.1.0
-Time    : 2023-02-17T12:50:22Z
+Version : v1.0.0
+Time    : 2024-09-26T07:31:31Z
 Licence : MIT
 
-Contents of probable licence file $GOMODCACHE/github.com/knadh/koanf/providers/env@v0.1.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/knadh/koanf/providers/env@v1.0.0/LICENSE:
 
 The MIT License
 
@@ -1625,11 +1625,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 #### Module  : github.com/knadh/koanf/providers/file
-Version : v1.1.0
-Time    : 2024-07-29T18:42:38Z
+Version : v1.1.2
+Time    : 2024-10-02T14:59:55Z
 Licence : MIT
 
-Contents of probable licence file $GOMODCACHE/github.com/knadh/koanf/providers/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/knadh/koanf/providers/[email protected].2/LICENSE:
 
 The MIT License
 
@@ -3303,39 +3303,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
---------------------------------------------------------------------------------
-#### Module  : github.com/creack/pty
-Version : v1.1.9
-Time    : 2019-09-25T15:36:33Z
-Licence : MIT
-
-Contents of probable licence file $GOMODCACHE/github.com/creack/[email protected]/LICENSE:
-
-Copyright (c) 2011 Keith Rarick
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated
-documentation files (the "Software"), to deal in the
-Software without restriction, including without limitation
-the rights to use, copy, modify, merge, publish, distribute,
-sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall
-be included in all copies or substantial portions of the
-Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
-PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
-OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
 --------------------------------------------------------------------------------
 #### Module  : github.com/davecgh/go-spew
 Version : v1.1.2-0.20180830191138-d8f796af33cc

bin/build-image.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Rebuilds the Quesma Docker image and tags it as quesma:latest
+# Rebuilds the Quesma Docker image and tags it as quesma:nightly
 set -e
 cd "$(dirname "$0/")/.."
-docker build -f quesma/Dockerfile -t quesma/quesma:latest quesma
+docker build -f quesma/Dockerfile -t quesma/quesma:nightly quesma

quesma/clickhouse/clickhouse.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"math"
 	"quesma/concurrent"
 	"quesma/end_user_errors"
 	"quesma/index"
@@ -248,35 +247,22 @@ func (lm *LogManager) isConnectedToPaidService(service PaidServiceName) (bool, e
 	return false, nil
 }
 
-// CheckIfConnectedPaidService executes simple query with exponential backoff
 func (lm *LogManager) CheckIfConnectedPaidService(service PaidServiceName) (returnedErr error) {
 	if _, ok := paidServiceChecks[service]; !ok {
 		return fmt.Errorf("service %s is not supported", service)
 	}
-	totalCheckTime := time.Minute
-	startTimeInSeconds := 2.0
-	start := time.Now()
-	attempt := 0
 	for {
 		isConnectedToPaidService, err := lm.isConnectedToPaidService(service)
 		if err != nil {
-			returnedErr = fmt.Errorf("error checking connection to database, attempt #%d, err=%v", attempt+1, err)
+			logger.Error().Msgf("Licensing checker failed to connect with the database")
 		}
 		if isConnectedToPaidService {
 			return fmt.Errorf("detected %s-specific table engine, which is not allowed", service)
 		} else if err == nil { // no paid service detected, no conn errors
 			returnedErr = nil
 			break
 		}
-		if time.Since(start) > totalCheckTime {
-			break
-		}
-		attempt++
-		sleepDuration := time.Duration(math.Pow(startTimeInSeconds, float64(attempt))) * time.Second
-		if remaining := time.Until(start.Add(totalCheckTime)); remaining < sleepDuration {
-			sleepDuration = remaining
-		}
-		time.Sleep(sleepDuration)
+		time.Sleep(3 * time.Second)
 	}
 	return returnedErr
 }

quesma/elasticsearch/client.go

@@ -57,9 +57,7 @@ func (es *SimpleClient) doRequest(ctx context.Context, method, endpoint string,
 	if err != nil {
 		return nil, err
 	}
-	if es.config.User != "" && es.config.Password != "" {
-		req.SetBasicAuth(es.config.User, es.config.Password)
-	}
+	req = AddBasicAuthIfNeeded(req, es.config.User, es.config.Password)
 	req.Header.Set("Content-Type", "application/json")
 	for key, values := range headers {
 		for _, value := range values {

quesma/elasticsearch/index_resolver.go

@@ -52,7 +52,10 @@ func NormalizePattern(p string) string {
 }
 
 func (im *indexResolver) Resolve(indexPattern string) (Sources, bool, error) {
-	req, _ := http.NewRequest("GET", im.Url+"/_resolve/index/"+indexPattern+"?expand_wildcards=open", bytes.NewBuffer([]byte{}))
+	req, err := http.NewRequest("GET", im.Url+"/_resolve/index/"+indexPattern+"?expand_wildcards=open", bytes.NewBuffer([]byte{}))
+	if err != nil {
+		return Sources{}, false, err
+	}
 	response, err := im.httpClient.Do(req)
 	if err != nil {
 		return Sources{}, false, err

quesma/elasticsearch/requests.go

@@ -23,3 +23,10 @@ func IsWriteRequest(req *http.Request) bool {
 	}
 	return false
 }
+
+func AddBasicAuthIfNeeded(req *http.Request, user, password string) *http.Request {
+	if user != "" && password != "" {
+		req.SetBasicAuth(user, password)
+	}
+	return req
+}

quesma/go.mod

@@ -13,8 +13,8 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/k0kubun/pp v3.0.1+incompatible
 	github.com/knadh/koanf/parsers/yaml v0.1.0
-	github.com/knadh/koanf/providers/env v0.1.0
-	github.com/knadh/koanf/providers/file v1.1.0
+	github.com/knadh/koanf/providers/env v1.0.0
+	github.com/knadh/koanf/providers/file v1.1.2
 	github.com/knadh/koanf/v2 v2.1.1
 	github.com/relvacode/iso8601 v1.4.0
 	github.com/rs/zerolog v1.33.0
@@ -30,7 +30,6 @@ require (
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect

quesma/go.sum

@@ -15,7 +15,6 @@ github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:h
 github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
 github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -63,10 +62,10 @@ github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NI
 github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
 github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w=
 github.com/knadh/koanf/parsers/yaml v0.1.0/go.mod h1:cvbUDC7AL23pImuQP0oRw/hPuccrNBS2bps8asS0CwY=
-github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg=
-github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ=
-github.com/knadh/koanf/providers/file v1.1.0 h1:MTjA+gRrVl1zqgetEAIaXHqYje0XSosxSiMD4/7kz0o=
-github.com/knadh/koanf/providers/file v1.1.0/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI=
+github.com/knadh/koanf/providers/env v1.0.0 h1:ufePaI9BnWH+ajuxGGiJ8pdTG0uLEUWC7/HDDPGLah0=
+github.com/knadh/koanf/providers/env v1.0.0/go.mod h1:mzFyRZueYhb37oPmC1HAv/oGEEuyvJDA98r3XAa8Gak=
+github.com/knadh/koanf/providers/file v1.1.2 h1:aCC36YGOgV5lTtAFz2qkgtWdeQsgfxUkxDOe+2nQY3w=
+github.com/knadh/koanf/providers/file v1.1.2/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI=
 github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM=
 github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=

quesma/health/elastic.go

@@ -7,24 +7,31 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"quesma/elasticsearch"
 	"quesma/logger"
 	"quesma/quesma/config"
 	"strconv"
 )
 
 type ElasticHealthChecker struct {
-	cfg *config.QuesmaConfiguration
+	cfg        *config.QuesmaConfiguration
+	httpClient *http.Client
 }
 
 func NewElasticHealthChecker(cfg *config.QuesmaConfiguration) Checker {
-	return &ElasticHealthChecker{cfg: cfg}
+	return &ElasticHealthChecker{cfg: cfg, httpClient: &http.Client{}}
 }
 
 func (c *ElasticHealthChecker) checkIfElasticsearchDiskIsFull() (isFull bool, reason string) {
 	const catAllocationPath = "/_cat/allocation?format=json"
 	const maxDiskPercent = 90
 
-	resp, err := http.Get(c.cfg.Elasticsearch.Url.String() + catAllocationPath)
+	req, err := http.NewRequest(http.MethodGet, c.cfg.Elasticsearch.Url.String()+catAllocationPath, nil)
+	if err != nil {
+		logger.Error().Err(err).Msgf("Can't create '%s' request", catAllocationPath)
+	}
+	req = elasticsearch.AddBasicAuthIfNeeded(req, c.cfg.Elasticsearch.User, c.cfg.Elasticsearch.Password)
+	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		return
 	}
@@ -62,7 +69,12 @@ func (c *ElasticHealthChecker) checkIfElasticsearchDiskIsFull() (isFull bool, re
 func (c *ElasticHealthChecker) CheckHealth() Status {
 	const elasticsearchHealthPath = "/_cluster/health/*"
 
-	resp, err := http.Get(c.cfg.Elasticsearch.Url.String() + elasticsearchHealthPath)
+	req, err := http.NewRequest(http.MethodGet, c.cfg.Elasticsearch.Url.String()+elasticsearchHealthPath, nil)
+	if err != nil {
+		return NewStatus("red", fmt.Sprintf("Can't create '%s' request", elasticsearchHealthPath), err.Error())
+	}
+	req = elasticsearch.AddBasicAuthIfNeeded(req, c.cfg.Elasticsearch.User, c.cfg.Elasticsearch.Password)
+	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		return NewStatus("red", "Ping failed", err.Error())
 	}

quesma/kibana/intervals.go

@@ -11,6 +11,21 @@ import (
 func ParseInterval(fixedInterval string) (time.Duration, error) {
 	var unit time.Duration
 
+	switch fixedInterval {
+	case "minute":
+		return time.Minute, nil
+	case "hour":
+		return time.Hour, nil
+	case "day":
+		return time.Hour * 24, nil
+	case "week":
+		return time.Hour * 24 * 7, nil
+	case "month":
+		return time.Hour * 24 * 30, nil
+	case "year":
+		return time.Hour * 24 * 365, nil
+	}
+
 	switch {
 	case strings.HasSuffix(fixedInterval, "d"):
 		unit = 24 * time.Hour

quesma/licensing/license_manager.go

@@ -51,6 +51,13 @@ func (l *LicenseModule) obtainLicenseKey() (err error) {
 	return nil
 }
 
+func FormatLicenseKey(licenseKey []byte) string {
+	if len(licenseKey) < 8 { // too short to be obfuscated, most probably it's invalid anyway
+		return "....."
+	}
+	return fmt.Sprintf("%s.....%s", string(licenseKey[:8]), string(licenseKey[len(licenseKey)-8:]))
+}
+
 // processLicense presents the license to the license server and receives an AllowList in return
 func (l *LicenseModule) processLicense() error {
 	if fetchedLicense, err := l.fetchLicense(); err != nil {

quesma/licensing/runner.go

@@ -33,8 +33,9 @@ func Init(config *config.QuesmaConfiguration) *LicenseModule {
 
 func (l *LicenseModule) Run() {
 	if len(l.LicenseKey) > 0 {
-		l.logInfo("License key [%s] already present, skipping license key obtainment.", l.LicenseKey)
+		l.logInfo("License key [%s] loaded from the configuration", FormatLicenseKey(l.LicenseKey))
 	} else {
+		l.logInfo("License key not supplied in the configuration, will attempt to obtain temporary license with limited functionalities")
 		l.setInstallationID()
 		if err := l.obtainLicenseKey(); err != nil {
 			PanicWithLicenseViolation(fmt.Errorf("failed to obtain license key: %v", err))
@@ -52,7 +53,7 @@ func (l *LicenseModule) validateConfig() error {
 	// Check if connectors are allowed
 	for _, conn := range l.Config.Connectors {
 		if !slices.Contains(l.License.Connectors, conn.ConnectorType) {
-			return fmt.Errorf("connector [%s] is not allowed within the current license", conn.ConnectorType)
+			return fmt.Errorf("connector of type [%s] is not allowed within the current license", conn.ConnectorType)
 		}
 	}
 	return nil