fix: file backend is not a list anymore

api/logs/logs.go

@@ -26,7 +26,7 @@ type SearchBackendConfig struct {
 	ElasticSearch *ElasticSearchBackendConfig    `json:"elasticsearch,omitempty"`
 	OpenSearch    *OpenSearchBackendConfig       `json:"opensearch,omitempty"`
 	Kubernetes    *KubernetesSearchBackendConfig `json:"kubernetes,omitempty"`
-	Files         []FileSearchBackendConfig      `json:"file,omitempty" yaml:"file,omitempty"`
+	Files         *FileSearchBackendConfig       `json:"file,omitempty" yaml:"file,omitempty"`
 }
 
 type SearchBackend struct {
@@ -35,7 +35,7 @@ type SearchBackend struct {
 	ElasticSearch *ElasticSearchBackendConfig    `json:"elasticsearch,omitempty"`
 	OpenSearch    *OpenSearchBackendConfig       `json:"opensearch,omitempty"`
 	Kubernetes    *KubernetesSearchBackendConfig `json:"kubernetes,omitempty"`
-	Files         []FileSearchBackendConfig      `json:"file,omitempty" yaml:"file,omitempty"`
+	File          *FileSearchBackendConfig       `json:"file,omitempty" yaml:"file,omitempty"`
 }
 
 type Routes []SearchRoute
@@ -62,7 +62,7 @@ type CommonBackend struct {
 func (b SearchBackendConfig) ToSearchBackend() SearchBackend {
 	return SearchBackend{
 		Kubernetes:    b.Kubernetes,
-		Files:         b.Files,
+		File:          b.Files,
 		ElasticSearch: b.ElasticSearch,
 		OpenSearch:    b.OpenSearch,
 	}

chart/crds/apm-hub.flanksource.com_loggingbackends.yaml

@@ -214,36 +214,34 @@ spec:
                           type: object
                       type: object
                     file:
-                      items:
-                        properties:
-                          labels:
-                            additionalProperties:
-                              type: string
-                            description: Labels are custom labels specified in the
-                              configuration file for a backend that will be attached
-                              to each log line returned by that backend.
-                            type: object
-                          path:
-                            items:
-                              type: string
-                            type: array
-                          routes:
-                            items:
-                              properties:
-                                id_prefix:
-                                  type: string
-                                is_additive:
-                                  type: boolean
-                                labels:
-                                  additionalProperties:
-                                    type: string
-                                  type: object
-                                type:
+                      properties:
+                        labels:
+                          additionalProperties:
+                            type: string
+                          description: Labels are custom labels specified in the configuration
+                            file for a backend that will be attached to each log line
+                            returned by that backend.
+                          type: object
+                        path:
+                          items:
+                            type: string
+                          type: array
+                        routes:
+                          items:
+                            properties:
+                              id_prefix:
+                                type: string
+                              is_additive:
+                                type: boolean
+                              labels:
+                                additionalProperties:
                                   type: string
-                              type: object
-                            type: array
-                        type: object
-                      type: array
+                                type: object
+                              type:
+                                type: string
+                            type: object
+                          type: array
+                      type: object
                     kubernetes:
                       properties:
                         kubeconfig:

pkg/config.go

@@ -76,21 +76,17 @@ func AttachSearchAPIToBackend(kommonsClient *kommons.Client, backend *logs.Searc
 		backend.API = k8s.NewKubernetesSearchBackend(k8sclient, backend.Kubernetes)
 	}
 
-	if len(backend.Files) > 0 {
+	if backend.File != nil {
 		// If the paths are not absolute,
 		// They should be parsed with respect to the current path
-		for i, f := range backend.Files {
-			for j, p := range f.Paths {
-				if !filepath.IsAbs(p) {
-					currentPath, _ := os.Getwd()
-					backend.Files[i].Paths[j] = filepath.Join(currentPath, p)
-				}
+		for j, p := range backend.File.Paths {
+			if !filepath.IsAbs(p) {
+				currentPath, _ := os.Getwd()
+				backend.File.Paths[j] = filepath.Join(currentPath, p)
 			}
 		}
 
-		backend.API = &files.FileSearch{
-			FilesBackendConfig: backend.Files,
-		}
+		backend.API = files.NewFileSearchBackend(backend.File)
 	}
 
 	if backend.ElasticSearch != nil {

pkg/files/search.go

@@ -12,32 +12,28 @@ import (
 	"github.com/flanksource/commons/logger"
 )
 
+func NewFileSearchBackend(config *logs.FileSearchBackendConfig) *FileSearch {
+	return &FileSearch{
+		config: config,
+	}
+}
+
 type FileSearch struct {
-	FilesBackendConfig []logs.FileSearchBackendConfig
+	config *logs.FileSearchBackendConfig
 }
 
 func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err error) {
 	var res logs.SearchResults
-
-	for _, b := range t.FilesBackendConfig {
-		files := readFilesLines(b.Paths, collections.MergeMap(b.Labels, q.Labels))
-		for _, content := range files {
-			res.Results = append(res.Results, content...)
-		}
+	lines := readFilesLines(t.config.Paths, collections.MergeMap(t.config.Labels, q.Labels))
+	for _, content := range lines {
+		res.Results = append(res.Results, content...)
 	}
 
 	return res, nil
 }
 
 func (t *FileSearch) MatchRoute(q *logs.SearchParams) (match bool, isAdditive bool) {
-	for _, k := range t.FilesBackendConfig {
-		match, isAdditive := k.Routes.MatchRoute(q)
-		if match {
-			return match, isAdditive
-		}
-	}
-
-	return false, false
+	return t.config.CommonBackend.Routes.MatchRoute(q)
 }
 
 type logsPerFile map[string][]logs.Result

samples/config-file.yaml

@@ -1,12 +1,21 @@
 backends:
   - file:
-      - labels:
-          name: acmehost
-          type: Nginx
-        path:
-          - nginx-access.log
-      - labels:
-          name: all
-          type: Nginx
-        path:
-          - "*.log"
+      routes:
+        - idPrefix: "nginx-"
+          labels:
+            type: "access"
+      labels:
+        name: acmehost
+        type: Nginx
+      path:
+        - samples/data/nginx-access.log
+  - file:
+      routes:
+        - idPrefix: "nginx-"
+          labels:
+            type: "error"
+      labels:
+        name: acmehost
+        type: Nginx
+      path:
+        - samples/data/nginx-error.log

samples/config-kubernetes.yaml

@@ -0,0 +1,5 @@
+backends:
+  - kubernetes:
+      routes:
+        - idPrefix: "cluster-main"
+      kubeconfig: