Merge pull request #46 from flanksource/file-config

fix: file config shouldn't be a list. Added readme

Readme.md

@@ -0,0 +1,46 @@
+# APM HUB
+
+APM Hub is a powerful tool designed to aggregate logs from various sources and provide a centralized location for querying them.
+
+It is able to integrate with the following sources:
+
+- Elastic search
+- Files
+- Kubernetes
+
+## Documentation
+
+Read the documentation at [https://docs.flanksource.com/apm-hub/overview/](https://docs.flanksource.com/apm-hub/overview/)
+
+## Samples
+
+Check out the samples directory for example configurations.
+
+## CLI
+
+**Usage**
+
+```bash
+apm-hub [command]
+```
+
+**Available Commands:**
+
+```
+  completion  Generate the autocompletion script for the specified shell
+  help        Help about any command
+  operator    Start the kubernetes operator
+  serve       Start the for querying the logs
+  version     Print the version of apm-hub
+
+Flags:
+      --db string             Connection string for the postgres database (default "DB_URL")
+      --db-log-level string    (default "warn")
+      --db-migrations         Run database migrations
+      --db-schema string       (default "public")
+  -h, --help                  help for apm-hub
+      --json-logs             Print logs in json format to stderr
+  -v, --loglevel count        Increase logging level
+
+Use "apm-hub [command] --help" for more information about a command.
+```

api/logs/logs.go

@@ -17,25 +17,25 @@ var GlobalBackends []SearchBackend
 // that consists of configuration for a list of backends.
 type SearchConfig struct {
 	// Path is the path of this config file
-	Path     string               `yaml:"-"`
-	Backends SearchBackendConfigs `yaml:"backends,omitempty"`
+	Path     string               `yaml:"-" json:"-"`
+	Backends SearchBackendConfigs `yaml:"backends,omitempty" json:"backends,omitempty"`
 }
 
 // +kubebuilder:object:generate=true
 type SearchBackendConfig struct {
-	ElasticSearch *ElasticSearchBackendConfig    `json:"elasticsearch,omitempty"`
-	OpenSearch    *OpenSearchBackendConfig       `json:"opensearch,omitempty"`
-	Kubernetes    *KubernetesSearchBackendConfig `json:"kubernetes,omitempty"`
-	Files         []FileSearchBackendConfig      `json:"file,omitempty" yaml:"file,omitempty"`
+	ElasticSearch *ElasticSearchBackendConfig    `json:"elasticsearch,omitempty" yaml:"elasticsearch,omitempty"`
+	OpenSearch    *OpenSearchBackendConfig       `json:"opensearch,omitempty" yaml:"opensearch,omitempty"`
+	Kubernetes    *KubernetesSearchBackendConfig `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty"`
+	File          *FileSearchBackendConfig       `json:"file,omitempty" yaml:"file,omitempty"`
 }
 
 type SearchBackend struct {
-	Name          string                         `json:"name"`
-	API           SearchAPI                      `json:"-"`
-	ElasticSearch *ElasticSearchBackendConfig    `json:"elasticsearch,omitempty"`
-	OpenSearch    *OpenSearchBackendConfig       `json:"opensearch,omitempty"`
-	Kubernetes    *KubernetesSearchBackendConfig `json:"kubernetes,omitempty"`
-	Files         []FileSearchBackendConfig      `json:"file,omitempty" yaml:"file,omitempty"`
+	Name          string
+	API           SearchAPI
+	ElasticSearch *ElasticSearchBackendConfig
+	OpenSearch    *OpenSearchBackendConfig
+	Kubernetes    *KubernetesSearchBackendConfig
+	File          *FileSearchBackendConfig
 }
 
 type Routes []SearchRoute
@@ -62,7 +62,7 @@ type CommonBackend struct {
 func (b SearchBackendConfig) ToSearchBackend() SearchBackend {
 	return SearchBackend{
 		Kubernetes:    b.Kubernetes,
-		Files:         b.Files,
+		File:          b.File,
 		ElasticSearch: b.ElasticSearch,
 		OpenSearch:    b.OpenSearch,
 	}

chart/crds/apm-hub.flanksource.com_loggingbackends.yaml

@@ -214,36 +214,34 @@ spec:
                           type: object
                       type: object
                     file:
-                      items:
-                        properties:
-                          labels:
-                            additionalProperties:
-                              type: string
-                            description: Labels are custom labels specified in the
-                              configuration file for a backend that will be attached
-                              to each log line returned by that backend.
-                            type: object
-                          path:
-                            items:
-                              type: string
-                            type: array
-                          routes:
-                            items:
-                              properties:
-                                id_prefix:
-                                  type: string
-                                is_additive:
-                                  type: boolean
-                                labels:
-                                  additionalProperties:
-                                    type: string
-                                  type: object
-                                type:
+                      properties:
+                        labels:
+                          additionalProperties:
+                            type: string
+                          description: Labels are custom labels specified in the configuration
+                            file for a backend that will be attached to each log line
+                            returned by that backend.
+                          type: object
+                        path:
+                          items:
+                            type: string
+                          type: array
+                        routes:
+                          items:
+                            properties:
+                              id_prefix:
+                                type: string
+                              is_additive:
+                                type: boolean
+                              labels:
+                                additionalProperties:
                                   type: string
-                              type: object
-                            type: array
-                        type: object
-                      type: array
+                                type: object
+                              type:
+                                type: string
+                            type: object
+                          type: array
+                      type: object
                     kubernetes:
                       properties:
                         kubeconfig:

pkg/config.go

@@ -76,21 +76,17 @@ func AttachSearchAPIToBackend(kommonsClient *kommons.Client, backend *logs.Searc
 		backend.API = k8s.NewKubernetesSearchBackend(k8sclient, backend.Kubernetes)
 	}
 
-	if len(backend.Files) > 0 {
+	if backend.File != nil {
 		// If the paths are not absolute,
 		// They should be parsed with respect to the current path
-		for i, f := range backend.Files {
-			for j, p := range f.Paths {
-				if !filepath.IsAbs(p) {
-					currentPath, _ := os.Getwd()
-					backend.Files[i].Paths[j] = filepath.Join(currentPath, p)
-				}
+		for j, p := range backend.File.Paths {
+			if !filepath.IsAbs(p) {
+				currentPath, _ := os.Getwd()
+				backend.File.Paths[j] = filepath.Join(currentPath, p)
 			}
 		}
 
-		backend.API = &files.FileSearch{
-			FilesBackendConfig: backend.Files,
-		}
+		backend.API = files.NewFileSearchBackend(backend.File)
 	}
 
 	if backend.ElasticSearch != nil {

pkg/files/search.go

@@ -12,32 +12,28 @@ import (
 	"github.com/flanksource/commons/logger"
 )
 
+func NewFileSearchBackend(config *logs.FileSearchBackendConfig) *FileSearch {
+	return &FileSearch{
+		config: config,
+	}
+}
+
 type FileSearch struct {
-	FilesBackendConfig []logs.FileSearchBackendConfig
+	config *logs.FileSearchBackendConfig
 }
 
 func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err error) {
 	var res logs.SearchResults
-
-	for _, b := range t.FilesBackendConfig {
-		files := readFilesLines(b.Paths, collections.MergeMap(b.Labels, q.Labels))
-		for _, content := range files {
-			res.Results = append(res.Results, content...)
-		}
+	lines := readFilesLines(t.config.Paths, collections.MergeMap(t.config.Labels, q.Labels))
+	for _, content := range lines {
+		res.Results = append(res.Results, content...)
 	}
 
 	return res, nil
 }
 
 func (t *FileSearch) MatchRoute(q *logs.SearchParams) (match bool, isAdditive bool) {
-	for _, k := range t.FilesBackendConfig {
-		match, isAdditive := k.Routes.MatchRoute(q)
-		if match {
-			return match, isAdditive
-		}
-	}
-
-	return false, false
+	return t.config.CommonBackend.Routes.MatchRoute(q)
 }
 
 type logsPerFile map[string][]logs.Result

samples/config-file.yaml

@@ -1,12 +1,21 @@
 backends:
   - file:
-      - labels:
-          name: acmehost
-          type: Nginx
-        path:
-          - nginx-access.log
-      - labels:
-          name: all
-          type: Nginx
-        path:
-          - "*.log"
+      routes:
+        - idPrefix: "nginx-"
+          labels:
+            type: "access"
+      labels:
+        name: acmehost
+        type: Nginx
+      path:
+        - samples/data/nginx-access.log
+  - file:
+      routes:
+        - idPrefix: "nginx-"
+          labels:
+            type: "error"
+      labels:
+        name: acmehost
+        type: Nginx
+      path:
+        - samples/data/nginx-error.log

samples/config-kubernetes.yaml

@@ -0,0 +1,5 @@
+backends:
+  - kubernetes:
+      routes:
+        - idPrefix: "cluster-main"
+      kubeconfig: