Revert * functionality (#817)

Revert the `*` functionality in index configuration. For the time being
(for the next release), users of Quesma will have to explicitly mention
all enabled indexes and autodiscovery is not supported.

Please note that as of this commit, if some index X is not configured,
Quesma will route queries/ingest to index X into ElasticSearch.

ci/quesma/config/ci-config.yaml

@@ -37,8 +37,6 @@ processors:
                 type: text
         windows_logs:
           target: [ my-clickhouse-data-source ]
-        "*":
-          target: [ my-minimal-elasticsearch ]
   - name: p2
     type: quesma-v1-processor-ingest
     config:
@@ -57,8 +55,6 @@ processors:
                 type: text
         windows_logs:   # Used for EQL e2e tests
           target: [ my-clickhouse-data-source ]
-        "*":
-          target: [ my-minimal-elasticsearch ]
 
 pipelines:
   - name: p-query

examples/kibana-sample-data/quesma/config/local-dev.yaml

@@ -71,8 +71,6 @@ processors:
                 type: "keyword"
               severity:
                 type: "keyword"
-        "*":
-          target: [ my-minimal-elasticsearch ]
   - name: my-ingest-processor
     type: quesma-v1-processor-ingest
     config:
@@ -116,8 +114,6 @@ processors:
                 type: "keyword"
               severity:
                 type: "keyword"
-        "*":
-          target: [ my-minimal-elasticsearch ]
 pipelines:
   - name: my-pipeline-elasticsearch-query-clickhouse
     frontendConnectors: [ elastic-query ]

quesma/quesma/config/config.go

@@ -181,9 +181,6 @@ func (c *QuesmaConfiguration) validateDeprecated(indexName IndexConfiguration, r
 }
 
 func (c *QuesmaConfiguration) validateIndexName(indexName string, result error) error {
-	if indexName == DefaultWildcardIndexName {
-		return result
-	}
 	if strings.Contains(indexName, "*") || indexName == "_all" {
 		result = multierror.Append(result, fmt.Errorf("wildcard patterns are not allowed in index configuration: %s", indexName))
 	}

quesma/quesma/config/config_v2.go

@@ -75,10 +75,6 @@ type Processor struct {
 	Config QuesmaProcessorConfig `koanf:"config"`
 }
 
-// An index configuration under this name in IndexConfig
-// specifies the default configuration for all (non-configured) indexes
-const DefaultWildcardIndexName = "*"
-
 // Configuration of QuesmaV1ProcessorQuery and QuesmaV1ProcessorIngest
 type QuesmaProcessorConfig struct {
 	IndexConfig map[string]IndexConfiguration `koanf:"indexes"`
@@ -279,9 +275,6 @@ func (c *QuesmaNewConfiguration) validatePipelines() error {
 					return fmt.Errorf("index '%s' is defined in query processor, but not in ingest processor", indexName)
 				}
 			}
-			if _, found := queryProcessor.Config.IndexConfig[DefaultWildcardIndexName]; !found {
-				return fmt.Errorf("the default index configuration (under the name '%s') must be defined in the query processor", DefaultWildcardIndexName)
-			}
 			for indexName, queryIndexConf := range queryProcessor.Config.IndexConfig {
 				ingestIndexConf := ingestProcessor.Config.IndexConfig[indexName]
 				if queryIndexConf.Override != ingestIndexConf.Override {
@@ -340,7 +333,7 @@ func (c *QuesmaNewConfiguration) validateProcessor(p Processor) error {
 	}
 	if p.Type == QuesmaV1ProcessorQuery || p.Type == QuesmaV1ProcessorIngest {
 		for indexName, indexConfig := range p.Config.IndexConfig {
-			if indexName != DefaultWildcardIndexName && strings.ContainsAny(indexName, "*,") {
+			if strings.ContainsAny(indexName, "*,") {
 				return fmt.Errorf("index name '%s' in processor configuration is an index pattern, not allowed", indexName)
 			}
 			if p.Type == QuesmaV1ProcessorQuery {
@@ -545,19 +538,6 @@ func (c *QuesmaNewConfiguration) TranslateToLegacyConfig() QuesmaConfiguration {
 
 			conf.IndexConfig[indexName] = processedConfig
 		}
-
-		// Handle default index configuration
-		defaultConfig := conf.IndexConfig[DefaultWildcardIndexName]
-		if len(defaultConfig.QueryTarget) != 1 {
-			errAcc = multierror.Append(errAcc, fmt.Errorf("the target configuration of default index ('%s') of query processor is not currently supported", DefaultWildcardIndexName))
-		}
-		if defaultConfig.QueryTarget[0] == ClickhouseTarget {
-			conf.AutodiscoveryEnabled = true
-		}
-		if !reflect.DeepEqual(defaultConfig.IngestTarget, []string{ElasticsearchTarget}) {
-			errAcc = multierror.Append(errAcc, fmt.Errorf("the target configuration of default index ('%s') of ingest processor is not currently supported", DefaultWildcardIndexName))
-		}
-		delete(conf.IndexConfig, DefaultWildcardIndexName)
 	}
 
 END:

quesma/quesma/config/test_configs/quesma_adding_two_hydrolix_tables.yaml

@@ -42,8 +42,6 @@ processors:
           target: [my-hydrolix-instance]
         logs:
           target: [my-hydrolix-instance]
-        "*":
-          target: [ my-minimal-elasticsearch ]
   - name: my-ingest-processor
     type: quesma-v1-processor-ingest
     config:
@@ -52,8 +50,6 @@ processors:
           target: [ my-minimal-elasticsearch ]
         logs:
           target: [ my-minimal-elasticsearch ]
-        "*":
-          target: [ my-minimal-elasticsearch ]
 pipelines:
   - name: my-elasticsearch-proxy-read
     frontendConnectors: [ elastic-query ]

quesma/quesma/config/test_configs/test_config_v2.yaml

@@ -74,8 +74,6 @@ processors:
             fields:
               message:
                 type: text
-        "*":
-          target: [ my-minimal-elasticsearch ]
   - name: my-ingest-processor
     type: quesma-v1-processor-ingest
     config:
@@ -125,8 +123,6 @@ processors:
             fields:
               message:
                 type: text
-        "*":
-          target: [ my-minimal-elasticsearch ]
 pipelines:
   - name: my-pipeline-elasticsearch-query-clickhouse
     frontendConnectors: [ elastic-query ]