From f7af6b82fe31abf3b5aae7f01390b0e0e423153b Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 24 Mar 2023 15:06:21 +0545 Subject: [PATCH 1/3] fix: file backend is not a list anymore --- api/logs/logs.go | 6 +- api/logs/zz_generated.deepcopy.go | 6 +- ...m-hub.flanksource.com_loggingbackends.yaml | 56 +++++++++---------- pkg/config.go | 16 ++---- pkg/files/search.go | 26 ++++----- samples/config-file.yaml | 29 ++++++---- samples/config-kubernetes.yaml | 5 ++ samples/config.yaml | 3 - samples/{ => data}/nginx-access.log | 0 samples/{ => data}/nginx-error.log | 0 10 files changed, 73 insertions(+), 74 deletions(-) create mode 100644 samples/config-kubernetes.yaml delete mode 100644 samples/config.yaml rename samples/{ => data}/nginx-access.log (100%) rename samples/{ => data}/nginx-error.log (100%) diff --git a/api/logs/logs.go b/api/logs/logs.go index 89b80bc..359fe82 100644 --- a/api/logs/logs.go +++ b/api/logs/logs.go @@ -26,7 +26,7 @@ type SearchBackendConfig struct { ElasticSearch *ElasticSearchBackendConfig `json:"elasticsearch,omitempty"` OpenSearch *OpenSearchBackendConfig `json:"opensearch,omitempty"` Kubernetes *KubernetesSearchBackendConfig `json:"kubernetes,omitempty"` - Files []FileSearchBackendConfig `json:"file,omitempty" yaml:"file,omitempty"` + Files *FileSearchBackendConfig `json:"file,omitempty" yaml:"file,omitempty"` } type SearchBackend struct { @@ -35,7 +35,7 @@ type SearchBackend struct { ElasticSearch *ElasticSearchBackendConfig `json:"elasticsearch,omitempty"` OpenSearch *OpenSearchBackendConfig `json:"opensearch,omitempty"` Kubernetes *KubernetesSearchBackendConfig `json:"kubernetes,omitempty"` - Files []FileSearchBackendConfig `json:"file,omitempty" yaml:"file,omitempty"` + File *FileSearchBackendConfig `json:"file,omitempty" yaml:"file,omitempty"` } type Routes []SearchRoute @@ -62,7 +62,7 @@ type CommonBackend struct { func (b SearchBackendConfig) ToSearchBackend() SearchBackend { return SearchBackend{ Kubernetes: b.Kubernetes, - Files: b.Files, + File: b.Files, ElasticSearch: b.ElasticSearch, OpenSearch: b.OpenSearch, } diff --git a/api/logs/zz_generated.deepcopy.go b/api/logs/zz_generated.deepcopy.go index 2c099d6..d2f2e3f 100644 --- a/api/logs/zz_generated.deepcopy.go +++ b/api/logs/zz_generated.deepcopy.go @@ -200,10 +200,8 @@ func (in *SearchBackendConfig) DeepCopyInto(out *SearchBackendConfig) { } if in.Files != nil { in, out := &in.Files, &out.Files - *out = make([]FileSearchBackendConfig, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + *out = new(FileSearchBackendConfig) + (*in).DeepCopyInto(*out) } } diff --git a/chart/crds/apm-hub.flanksource.com_loggingbackends.yaml b/chart/crds/apm-hub.flanksource.com_loggingbackends.yaml index fc05eae..98c8132 100644 --- a/chart/crds/apm-hub.flanksource.com_loggingbackends.yaml +++ b/chart/crds/apm-hub.flanksource.com_loggingbackends.yaml @@ -214,36 +214,34 @@ spec: type: object type: object file: - items: - properties: - labels: - additionalProperties: - type: string - description: Labels are custom labels specified in the - configuration file for a backend that will be attached - to each log line returned by that backend. - type: object - path: - items: - type: string - type: array - routes: - items: - properties: - id_prefix: - type: string - is_additive: - type: boolean - labels: - additionalProperties: - type: string - type: object - type: + properties: + labels: + additionalProperties: + type: string + description: Labels are custom labels specified in the configuration + file for a backend that will be attached to each log line + returned by that backend. + type: object + path: + items: + type: string + type: array + routes: + items: + properties: + id_prefix: + type: string + is_additive: + type: boolean + labels: + additionalProperties: type: string - type: object - type: array - type: object - type: array + type: object + type: + type: string + type: object + type: array + type: object kubernetes: properties: kubeconfig: diff --git a/pkg/config.go b/pkg/config.go index 90817f5..5098956 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -76,21 +76,17 @@ func AttachSearchAPIToBackend(kommonsClient *kommons.Client, backend *logs.Searc backend.API = k8s.NewKubernetesSearchBackend(k8sclient, backend.Kubernetes) } - if len(backend.Files) > 0 { + if backend.File != nil { // If the paths are not absolute, // They should be parsed with respect to the current path - for i, f := range backend.Files { - for j, p := range f.Paths { - if !filepath.IsAbs(p) { - currentPath, _ := os.Getwd() - backend.Files[i].Paths[j] = filepath.Join(currentPath, p) - } + for j, p := range backend.File.Paths { + if !filepath.IsAbs(p) { + currentPath, _ := os.Getwd() + backend.File.Paths[j] = filepath.Join(currentPath, p) } } - backend.API = &files.FileSearch{ - FilesBackendConfig: backend.Files, - } + backend.API = files.NewFileSearchBackend(backend.File) } if backend.ElasticSearch != nil { diff --git a/pkg/files/search.go b/pkg/files/search.go index 2694d46..70874bb 100644 --- a/pkg/files/search.go +++ b/pkg/files/search.go @@ -12,32 +12,28 @@ import ( "github.com/flanksource/commons/logger" ) +func NewFileSearchBackend(config *logs.FileSearchBackendConfig) *FileSearch { + return &FileSearch{ + config: config, + } +} + type FileSearch struct { - FilesBackendConfig []logs.FileSearchBackendConfig + config *logs.FileSearchBackendConfig } func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err error) { var res logs.SearchResults - - for _, b := range t.FilesBackendConfig { - files := readFilesLines(b.Paths, collections.MergeMap(b.Labels, q.Labels)) - for _, content := range files { - res.Results = append(res.Results, content...) - } + lines := readFilesLines(t.config.Paths, collections.MergeMap(t.config.Labels, q.Labels)) + for _, content := range lines { + res.Results = append(res.Results, content...) } return res, nil } func (t *FileSearch) MatchRoute(q *logs.SearchParams) (match bool, isAdditive bool) { - for _, k := range t.FilesBackendConfig { - match, isAdditive := k.Routes.MatchRoute(q) - if match { - return match, isAdditive - } - } - - return false, false + return t.config.CommonBackend.Routes.MatchRoute(q) } type logsPerFile map[string][]logs.Result diff --git a/samples/config-file.yaml b/samples/config-file.yaml index 319aa9d..4d0df43 100644 --- a/samples/config-file.yaml +++ b/samples/config-file.yaml @@ -1,12 +1,21 @@ backends: - file: - - labels: - name: acmehost - type: Nginx - path: - - nginx-access.log - - labels: - name: all - type: Nginx - path: - - "*.log" + routes: + - idPrefix: "nginx-" + labels: + type: "access" + labels: + name: acmehost + type: Nginx + path: + - samples/data/nginx-access.log + - file: + routes: + - idPrefix: "nginx-" + labels: + type: "error" + labels: + name: acmehost + type: Nginx + path: + - samples/data/nginx-error.log diff --git a/samples/config-kubernetes.yaml b/samples/config-kubernetes.yaml new file mode 100644 index 0000000..7f82bf3 --- /dev/null +++ b/samples/config-kubernetes.yaml @@ -0,0 +1,5 @@ +backends: + - kubernetes: + routes: + - idPrefix: "cluster-main" + kubeconfig: diff --git a/samples/config.yaml b/samples/config.yaml deleted file mode 100644 index a3d34d3..0000000 --- a/samples/config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -backends: - - kubernetes: - kubeconfig: diff --git a/samples/nginx-access.log b/samples/data/nginx-access.log similarity index 100% rename from samples/nginx-access.log rename to samples/data/nginx-access.log diff --git a/samples/nginx-error.log b/samples/data/nginx-error.log similarity index 100% rename from samples/nginx-error.log rename to samples/data/nginx-error.log From 4bc730d7c9a8f550097dbb4d11d93ec14017fdd1 Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 24 Mar 2023 15:12:16 +0545 Subject: [PATCH 2/3] feat: added readme --- Readme.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 Readme.md diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..7133636 --- /dev/null +++ b/Readme.md @@ -0,0 +1,46 @@ +# APM HUB + +APM Hub is a powerful tool designed to aggregate logs from various sources and provide a centralized location for querying them. + +It is able to integrate with the following sources: + +- Elastic search +- Files +- Kubernetes + +## Documentation + +Read the documentation at [https://docs.flanksource.com/apm-hub/overview/](https://docs.flanksource.com/apm-hub/overview/) + +## Samples + +Check out the samples directory for example configurations. + +## CLI + +**Usage** + +```bash +apm-hub [command] +``` + +**Available Commands:** + +``` + completion Generate the autocompletion script for the specified shell + help Help about any command + operator Start the kubernetes operator + serve Start the for querying the logs + version Print the version of apm-hub + +Flags: + --db string Connection string for the postgres database (default "DB_URL") + --db-log-level string (default "warn") + --db-migrations Run database migrations + --db-schema string (default "public") + -h, --help help for apm-hub + --json-logs Print logs in json format to stderr + -v, --loglevel count Increase logging level + +Use "apm-hub [command] --help" for more information about a command. +``` From 9a1e6ece9dff2c7ba22aad626dfe163534a01af1 Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 24 Mar 2023 15:21:04 +0545 Subject: [PATCH 3/3] chore: rename Files to File and some struct tags cleanup --- api/logs/logs.go | 26 +++++++++++++------------- api/logs/zz_generated.deepcopy.go | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/api/logs/logs.go b/api/logs/logs.go index 359fe82..a2e2c40 100644 --- a/api/logs/logs.go +++ b/api/logs/logs.go @@ -17,25 +17,25 @@ var GlobalBackends []SearchBackend // that consists of configuration for a list of backends. type SearchConfig struct { // Path is the path of this config file - Path string `yaml:"-"` - Backends SearchBackendConfigs `yaml:"backends,omitempty"` + Path string `yaml:"-" json:"-"` + Backends SearchBackendConfigs `yaml:"backends,omitempty" json:"backends,omitempty"` } // +kubebuilder:object:generate=true type SearchBackendConfig struct { - ElasticSearch *ElasticSearchBackendConfig `json:"elasticsearch,omitempty"` - OpenSearch *OpenSearchBackendConfig `json:"opensearch,omitempty"` - Kubernetes *KubernetesSearchBackendConfig `json:"kubernetes,omitempty"` - Files *FileSearchBackendConfig `json:"file,omitempty" yaml:"file,omitempty"` + ElasticSearch *ElasticSearchBackendConfig `json:"elasticsearch,omitempty" yaml:"elasticsearch,omitempty"` + OpenSearch *OpenSearchBackendConfig `json:"opensearch,omitempty" yaml:"opensearch,omitempty"` + Kubernetes *KubernetesSearchBackendConfig `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty"` + File *FileSearchBackendConfig `json:"file,omitempty" yaml:"file,omitempty"` } type SearchBackend struct { - Name string `json:"name"` - API SearchAPI `json:"-"` - ElasticSearch *ElasticSearchBackendConfig `json:"elasticsearch,omitempty"` - OpenSearch *OpenSearchBackendConfig `json:"opensearch,omitempty"` - Kubernetes *KubernetesSearchBackendConfig `json:"kubernetes,omitempty"` - File *FileSearchBackendConfig `json:"file,omitempty" yaml:"file,omitempty"` + Name string + API SearchAPI + ElasticSearch *ElasticSearchBackendConfig + OpenSearch *OpenSearchBackendConfig + Kubernetes *KubernetesSearchBackendConfig + File *FileSearchBackendConfig } type Routes []SearchRoute @@ -62,7 +62,7 @@ type CommonBackend struct { func (b SearchBackendConfig) ToSearchBackend() SearchBackend { return SearchBackend{ Kubernetes: b.Kubernetes, - File: b.Files, + File: b.File, ElasticSearch: b.ElasticSearch, OpenSearch: b.OpenSearch, } diff --git a/api/logs/zz_generated.deepcopy.go b/api/logs/zz_generated.deepcopy.go index d2f2e3f..29984db 100644 --- a/api/logs/zz_generated.deepcopy.go +++ b/api/logs/zz_generated.deepcopy.go @@ -198,8 +198,8 @@ func (in *SearchBackendConfig) DeepCopyInto(out *SearchBackendConfig) { *out = new(KubernetesSearchBackendConfig) (*in).DeepCopyInto(*out) } - if in.Files != nil { - in, out := &in.Files, &out.Files + if in.File != nil { + in, out := &in.File, &out.File *out = new(FileSearchBackendConfig) (*in).DeepCopyInto(*out) }