Merge branch 'main' into pr-ab-e2e-ignore-fields

QuesmaOrg · Jan 16, 2025 · f1ae5af · f1ae5af
2 parents d3f3fb7 + e44f0af
commit f1ae5af
Show file tree

Hide file tree

Showing 96 changed files with 709 additions and 335 deletions.
diff --git a/NOTICE.MD b/NOTICE.MD
diff --git a/quesma/Dockerfile b/quesma/Dockerfile
@@ -2,9 +2,6 @@ FROM golang:alpine AS builder
 
 
 WORKDIR /quesma
-## v2 is a seprate module, not package
-COPY v2/go.mod v2/go.mod
-COPY v2/go.sum v2/go.sum
 
 COPY go.mod go.sum ./
 

diff --git a/quesma/backend_connectors/clickhouse_backend_connector.go b/quesma/backend_connectors/clickhouse_backend_connector.go
@@ -8,7 +8,7 @@ import (
 	"database/sql"
 	"github.com/ClickHouse/clickhouse-go/v2"
 
-	quesma_api "quesma_v2/core"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 )
 
 type ClickHouseBackendConnector struct {

diff --git a/quesma/backend_connectors/elasticsearch_backend_connector.go b/quesma/backend_connectors/elasticsearch_backend_connector.go
@@ -10,8 +10,8 @@ import (
 	"fmt"
 	"github.com/QuesmaOrg/quesma/quesma/elasticsearch"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"net/http"
-	quesma_api "quesma_v2/core"
 	"time"
 )
 

diff --git a/quesma/backend_connectors/mysql_backend_connector.go b/quesma/backend_connectors/mysql_backend_connector.go
@@ -6,8 +6,8 @@ package backend_connectors
 import (
 	"context"
 	"database/sql"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	_ "github.com/go-sql-driver/mysql"
-	quesma_api "quesma_v2/core"
 )
 
 type MySqlRows struct {

diff --git a/quesma/backend_connectors/postgres_backend_connector.go b/quesma/backend_connectors/postgres_backend_connector.go
@@ -5,8 +5,8 @@ package backend_connectors
 
 import (
 	"context"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"github.com/jackc/pgx/v4"
-	quesma_api "quesma_v2/core"
 )
 
 type PostgresBackendConnector struct {

diff --git a/quesma/clickhouse/clickhouse.go b/quesma/clickhouse/clickhouse.go
@@ -13,8 +13,8 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/quesma/recovery"
 	"github.com/QuesmaOrg/quesma/quesma/schema"
 	"github.com/QuesmaOrg/quesma/quesma/util"
-	quesma_api "quesma_v2/core"
-	"quesma_v2/core/diag"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
+	"github.com/QuesmaOrg/quesma/quesma/v2/core/diag"
 	"slices"
 	"strings"
 	"sync/atomic"

diff --git a/quesma/clickhouse/connection.go b/quesma/clickhouse/connection.go
@@ -11,8 +11,8 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/buildinfo"
 	"github.com/QuesmaOrg/quesma/quesma/logger"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"net"
-	quesma_api "quesma_v2/core"
 	"strings"
 	"time"
 )

diff --git a/quesma/clickhouse/quesma_communicator.go b/quesma/clickhouse/quesma_communicator.go
@@ -10,9 +10,9 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/logger"
 	"github.com/QuesmaOrg/quesma/quesma/model"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/recovery"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
+	"github.com/QuesmaOrg/quesma/quesma/v2/core/tracing"
 	"math/rand"
-	quesma_api "quesma_v2/core"
-	tracing "quesma_v2/core/tracing"
 	"strconv"
 	"strings"
 	"sync/atomic"

diff --git a/quesma/clickhouse/table_discovery.go b/quesma/clickhouse/table_discovery.go
@@ -11,11 +11,13 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/logger"
 	"github.com/QuesmaOrg/quesma/quesma/persistence"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
+	"github.com/QuesmaOrg/quesma/quesma/quesma/types"
 	"github.com/QuesmaOrg/quesma/quesma/schema"
 	"github.com/QuesmaOrg/quesma/quesma/util"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"github.com/goccy/go-json"
-	quesma_api "quesma_v2/core"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 )
@@ -34,12 +36,15 @@ func (d DbKind) String() string {
 type TableDiscovery interface {
 	ReloadTableDefinitions()
 	TableDefinitions() *TableMap
+	AddTable(tableName string, table *Table)
 	TableDefinitionsFetchError() error
 
 	LastAccessTime() time.Time
 	LastReloadTime() time.Time
 	ForceReloadCh() <-chan chan<- struct{}
 	AutodiscoveryEnabled() bool
+
+	RegisterTablesReloadListener(ch chan<- types.ReloadMessage)
 }
 
 type tableDiscovery struct {
@@ -51,6 +56,9 @@ type tableDiscovery struct {
 	forceReloadCh                     chan chan<- struct{}
 	ReloadTablesError                 error
 	virtualTableStorage               persistence.JSONDatabase
+
+	reloadObserversMutex sync.Mutex
+	reloadObservers      []chan<- types.ReloadMessage
 }
 
 type columnMetadata struct {
@@ -79,6 +87,10 @@ type TableDiscoveryTableProviderAdapter struct {
 	TableDiscovery
 }
 
+func (t TableDiscoveryTableProviderAdapter) RegisterTablesReloadListener(ch chan<- types.ReloadMessage) {
+	t.TableDiscovery.RegisterTablesReloadListener(ch)
+}
+
 func (t TableDiscoveryTableProviderAdapter) TableDefinitions() map[string]schema.Table {
 
 	// here we filter out our internal columns
@@ -125,6 +137,31 @@ func NewTableDiscoveryWith(cfg *config.QuesmaConfiguration, dbConnPool quesma_ap
 	return result
 }
 
+func (td *tableDiscovery) AddTable(tableName string, table *Table) {
+	td.tableDefinitions.Load().Store(tableName, table)
+	td.notifyObservers()
+}
+
+func (td *tableDiscovery) RegisterTablesReloadListener(ch chan<- types.ReloadMessage) {
+	td.reloadObserversMutex.Lock()
+	defer td.reloadObserversMutex.Unlock()
+	td.reloadObservers = append(td.reloadObservers, ch)
+}
+
+func (td *tableDiscovery) notifyObservers() {
+
+	td.reloadObserversMutex.Lock()
+	defer td.reloadObserversMutex.Unlock()
+
+	msg := types.ReloadMessage{Timestamp: time.Now()}
+	for _, observer := range td.reloadObservers {
+		fmt.Println("Sending message to observer", observer)
+		go func() {
+			observer <- msg
+		}()
+	}
+}
+
 func (td *tableDiscovery) TableDefinitionsFetchError() error {
 	return td.ReloadTablesError
 }
@@ -178,6 +215,8 @@ func (td *tableDiscovery) ReloadTableDefinitions() {
 
 	td.ReloadTablesError = nil
 	td.populateTableDefinitions(configuredTables, databaseName, td.cfg)
+
+	td.notifyObservers()
 }
 
 func (td *tableDiscovery) readVirtualTables(configuredTables map[string]discoveredTable) map[string]discoveredTable {
@@ -634,6 +673,9 @@ func NewEmptyTableDiscovery() *EmptyTableDiscovery {
 	}
 }
 
+func (td *EmptyTableDiscovery) RegisterTablesReloadListener(ch chan<- types.ReloadMessage) {
+}
+
 func (td *EmptyTableDiscovery) ReloadTableDefinitions() {
 }
 
@@ -660,3 +702,7 @@ func (td *EmptyTableDiscovery) ForceReloadCh() <-chan chan<- struct{} {
 func (td *EmptyTableDiscovery) AutodiscoveryEnabled() bool {
 	return td.Autodiscovery
 }
+
+func (td *EmptyTableDiscovery) AddTable(tableName string, table *Table) {
+	td.TableMap.Store(tableName, table)
+}
diff --git a/quesma/common_table/const.go b/quesma/common_table/const.go
@@ -5,7 +5,7 @@ package common_table
 import (
 	"context"
 	"github.com/QuesmaOrg/quesma/quesma/logger"
-	quesma_api "quesma_v2/core"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 )
 
 const TableName = "quesma_common_table"

diff --git a/quesma/connectors/connector.go b/quesma/connectors/connector.go
@@ -9,7 +9,7 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/logger"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
 	"github.com/QuesmaOrg/quesma/quesma/telemetry"
-	quesma_api "quesma_v2/core"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 )
 
 type Connector interface {

diff --git a/quesma/eql/e2e/query_integration_test.go b/quesma/eql/e2e/query_integration_test.go
@@ -7,10 +7,10 @@ package e2e
 import (
 	"bytes"
 	"fmt"
+	"github.com/QuesmaOrg/quesma/quesma/jsonprocessor"
 	"github.com/goccy/go-json"
 	"io"
 	"net/http"
-	"quesma/jsonprocessor"
 	"sort"
 	"strings"
 )

diff --git a/quesma/frontend_connectors/basic_http_frontend_connector.go b/quesma/frontend_connectors/basic_http_frontend_connector.go
@@ -9,10 +9,10 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/clickhouse"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
 	"github.com/QuesmaOrg/quesma/quesma/schema"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
+	"github.com/QuesmaOrg/quesma/quesma/v2/core/diag"
 	"io"
 	"net/http"
-	quesma_api "quesma_v2/core"
-	"quesma_v2/core/diag"
 	"sync"
 )
 

diff --git a/quesma/frontend_connectors/basic_tcp_connection_handler.go b/quesma/frontend_connectors/basic_tcp_connection_handler.go
@@ -5,9 +5,9 @@ package frontend_connectors
 
 import (
 	"fmt"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"io"
 	"net"
-	quesma_api "quesma_v2/core"
 )
 
 type BasicTcpConnectionHandler struct {

diff --git a/quesma/frontend_connectors/basic_tcp_connector.go b/quesma/frontend_connectors/basic_tcp_connector.go
@@ -6,8 +6,8 @@ package frontend_connectors
 import (
 	"context"
 	"fmt"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"net"
-	quesma_api "quesma_v2/core"
 	"sync/atomic"
 )
 

diff --git a/quesma/frontend_connectors/elastic_http_frontend_connector.go b/quesma/frontend_connectors/elastic_http_frontend_connector.go
@@ -8,8 +8,8 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/clickhouse"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
 	"github.com/QuesmaOrg/quesma/quesma/schema"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"net/http"
-	quesma_api "quesma_v2/core"
 )
 
 type ElasticHttpIngestFrontendConnector struct {

diff --git a/quesma/frontend_connectors/elasticsearch_ingest.go b/quesma/frontend_connectors/elasticsearch_ingest.go
@@ -6,8 +6,8 @@ package frontend_connectors
 import (
 	"context"
 	"github.com/QuesmaOrg/quesma/quesma/processors/es_to_ch_common"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"net/http"
-	quesma_api "quesma_v2/core"
 )
 
 type ElasticsearchIngestFrontendConnector struct {

diff --git a/quesma/frontend_connectors/elasticsearch_query.go b/quesma/frontend_connectors/elasticsearch_query.go
@@ -8,8 +8,8 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/elasticsearch"
 	"github.com/QuesmaOrg/quesma/quesma/processors/es_to_ch_common"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"net/http"
-	quesma_api "quesma_v2/core"
 )
 
 type ElasticsearchQueryFrontendConnector struct {

diff --git a/quesma/frontend_connectors/passthrough_tcp_connection_handler.go b/quesma/frontend_connectors/passthrough_tcp_connection_handler.go
@@ -5,9 +5,9 @@ package frontend_connectors
 
 import (
 	"fmt"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"io"
 	"net"
-	quesma_api "quesma_v2/core"
 )
 
 func (p *PassThroughConnectionHandler) copyData(src io.Reader, dest io.Writer) {

diff --git a/quesma/frontend_connectors/router_v2.go b/quesma/frontend_connectors/router_v2.go
@@ -22,12 +22,12 @@ import (
 	"github.com/QuesmaOrg/quesma/quesma/quesma/ui"
 	"github.com/QuesmaOrg/quesma/quesma/schema"
 	"github.com/QuesmaOrg/quesma/quesma/util"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
+	"github.com/QuesmaOrg/quesma/quesma/v2/core/diag"
+	"github.com/QuesmaOrg/quesma/quesma/v2/core/routes"
+	"github.com/QuesmaOrg/quesma/quesma/v2/core/tracing"
 	"io"
 	"net/http"
-	quesma_api "quesma_v2/core"
-	"quesma_v2/core/diag"
-	"quesma_v2/core/routes"
-	"quesma_v2/core/tracing"
 	"strings"
 	"sync/atomic"
 	"time"

diff --git a/quesma/frontend_connectors/tcp_postgres_connection_handler.go b/quesma/frontend_connectors/tcp_postgres_connection_handler.go
@@ -5,9 +5,9 @@ package frontend_connectors
 
 import (
 	"fmt"
+	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
 	"github.com/jackc/pgx/v5/pgproto3"
 	"net"
-	quesma_api "quesma_v2/core"
 )
 
 type TcpPostgresConnectionHandler struct {