Add few real-life configuration files to test config parsing (#804)

FYSA: @trzysiek

quesma/quesma/config/config_v2_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestQuesmaConfigurationLoading(t *testing.T) {
 
-	os.Setenv(configFileLocationEnvVar, "./test_config_v2.yaml")
+	os.Setenv(configFileLocationEnvVar, "./test_configs/test_config_v2.yaml")
 
 	logLevelPassedAsEnvVar := "debug"
 	licenseKeyPassedAsEnvVar := "arbitraty-license-key"
@@ -62,6 +62,35 @@ func TestQuesmaConfigurationLoading(t *testing.T) {
 	}
 }
 
+func TestQuesmaTransparentProxyConfiguration(t *testing.T) {
+	os.Setenv(configFileLocationEnvVar, "./test_configs/quesma_as_transparent_proxy.yml")
+	cfg := LoadV2Config()
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("error validating config: %v", err)
+	}
+	legacyConf := cfg.TranslateToLegacyConfig()
+	assert.True(t, legacyConf.TransparentProxy)
+}
+
+func TestQuesmaAddingHydrolixTablesToExistingElasticsearch(t *testing.T) {
+	os.Setenv(configFileLocationEnvVar, "./test_configs/quesma_adding_two_hydrolix_tables.yaml")
+	cfg := LoadV2Config()
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("error validating config: %v", err)
+	}
+	legacyConf := cfg.TranslateToLegacyConfig()
+	assert.False(t, legacyConf.TransparentProxy)
+	assert.Equal(t, 2, len(legacyConf.IndexConfig))
+	siemIndexConf := legacyConf.IndexConfig["siem"]
+	logsIndexConf := legacyConf.IndexConfig["logs"]
+
+	assert.Equal(t, []string{"clickhouse"}, siemIndexConf.QueryTarget)
+	assert.Equal(t, []string{"elasticsearch"}, siemIndexConf.IngestTarget)
+
+	assert.Equal(t, []string{"clickhouse"}, logsIndexConf.QueryTarget)
+	assert.Equal(t, []string{"elasticsearch"}, logsIndexConf.IngestTarget)
+}
+
 func TestMatchName(t *testing.T) {
 	type args struct {
 		indexName        string

quesma/quesma/config/test_configs/quesma_adding_two_hydrolix_tables.yaml

@@ -0,0 +1,65 @@
+# Use case:
+# * user has some indices in Elasticsearch
+# * user has two tables in Hydrolix named `siem` and `logs`
+#
+#  User wants to see those two Hydrolix tables as Elasticsearch indices in Kibana
+#  User wants to see all their Elasticsearch indices in Kibana as they were before
+#
+#  ( ingest is not the case here, but the config has to be present due to impl. constraints )
+
+logging:
+  level: info
+frontendConnectors:
+  - name: elastic-ingest
+    type: elasticsearch-fe-ingest
+    config:
+      listenPort: 8080
+  - name: elastic-query
+    type: elasticsearch-fe-query
+    config:
+      listenPort: 8080
+backendConnectors:
+  - name: my-minimal-elasticsearch
+    type: elasticsearch
+    config:
+      url: "http://elasticsearch:9200"
+      user: elastic
+      password: quesmaquesma
+  - name: my-hydrolix-instance
+    type: hydrolix
+    config:
+      url: "clickhouse://localhost:9000"
+      user: "u"
+      password: "p"
+      database: "d"
+ingestStatistics: true
+processors:
+  - name: my-query-processor
+    type: quesma-v1-processor-query
+    config:
+      indexes:
+        siem:
+          target: [my-hydrolix-instance]
+        logs:
+          target: [my-hydrolix-instance]
+        "*":
+          target: [ my-minimal-elasticsearch ]
+  - name: my-ingest-processor
+    type: quesma-v1-processor-ingest
+    config:
+      indexes:  # This is also virtual requirement, but it is what it is for now
+        siem:
+          target: [ my-minimal-elasticsearch ]
+        logs:
+          target: [ my-minimal-elasticsearch ]
+        "*":
+          target: [ my-minimal-elasticsearch ]
+pipelines:
+  - name: my-elasticsearch-proxy-read
+    frontendConnectors: [ elastic-query ]
+    processors: [ my-query-processor ]
+    backendConnectors: [ my-minimal-elasticsearch, my-hydrolix-instance ]
+  - name: my-elasticsearch-proxy-write
+    frontendConnectors: [ elastic-ingest ]
+    processors: [ my-ingest-processor ]
+    backendConnectors: [ my-minimal-elasticsearch, my-hydrolix-instance ]

quesma/quesma/config/test_configs/quesma_as_transparent_proxy.yml

@@ -0,0 +1,35 @@
+frontendConnectors:
+  - name: elastic-ingest
+    type: elasticsearch-fe-ingest
+    config:
+      listenPort: 8080
+  - name: elastic-query
+    type: elasticsearch-fe-query
+    config:
+      listenPort: 8080
+backendConnectors:
+  - name: my-minimal-elasticsearch
+    type: elasticsearch
+    config:
+      url: "http://elasticsearch:9200"
+      user: elastic
+      password: quesmaquesma
+  - name: my-hydrolix-instance    # Okay maybe this is weird  but we require this ATM
+    type: hydrolix
+    config:
+      url: "clickhouse://localhost:9440"
+      user: "u"
+      password: "p"
+      database: "d"
+processors:
+  - name: noop-query-processor
+    type: quesma-v1-processor-noop
+pipelines:
+  - name: my-elasticsearch-proxy-read
+    frontendConnectors: [ elastic-query ]
+    processors: [ noop-query-processor ]
+    backendConnectors: [ my-minimal-elasticsearch ]
+  - name: my-elasticsearch-proxy-write
+    frontendConnectors: [ elastic-ingest ]
+    processors: [ noop-query-processor ]
+    backendConnectors: [ my-minimal-elasticsearch ]