Merge pull request #1 from flanksource/kubernetes-search-api

feat: add kubernetes search api

.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: Create Release
+on:
+  push:
+    branches:
+      - main
+jobs:
+  semantic-release:
+    runs-on: ubuntu-latest
+    outputs:
+      release-version: ${{ steps.semantic.outputs.release-version }}
+      new-release-published: ${{ steps.semantic.outputs.new-release-published }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: codfish/semantic-release-action@v1
+        id: semantic
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  binary:
+    runs-on: ubuntu-latest
+    needs: semantic-release
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: v1.17.x
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            .bin
+          key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
+          restore-keys: |
+            cache-
+      - run: make release
+        env:
+          VERSION: v${{ needs.semantic-release.outputs.release-version }}
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: ./.release/*
+          tag: v${{ needs.semantic-release.outputs.release-version }}
+          overwrite: true
+          file_glob: true
+  docker:
+    needs: semantic-release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Publish to Registry
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: flanksource/apm-hub
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          snapshot: true
+          tags: "latest,v${{ needs.semantic-release.outputs.release-version }}"

.gitignore

@@ -0,0 +1,7 @@
+##IDEs
+.idea
+.vscode
+
+#Build
+.bin/
+.release

.releaserc

@@ -0,0 +1,28 @@
+branches:
+  - name: main
+plugins:
+  - - "@semantic-release/commit-analyzer"
+    - releaseRules:
+        - { type: doc, scope: README, release: patch }
+        - { type: fix, release: patch }
+        - { type: chore, release: patch }
+        - { type: refactor, release: patch }
+        - { type: feat, release: patch }
+        - { type: ci, release: false }
+        - { type: style, release: false }
+      parserOpts:
+        noteKeywords:
+          - MAJOR RELEASE
+  - "@semantic-release/release-notes-generator"
+  - - "@semantic-release/github"
+    - assets:
+        - path: ./.bin/apm-hub_linux_amd64
+          name: apm-hub-amd64
+        - path: ./.bin/apm-hub_linux_arm64
+          name: apm-hub-amd64
+        - path: ./.bin/apm-hub.exe
+          name: apm-hub.exe
+        - path: ./.bin/apm-hub_darwin_amd64
+          name: apm-hub_osx-amd64
+        - path: ./.bin/apm-hub_darwin_arm64
+          name: apm-hub_osx-arm64

Dockerfile

@@ -0,0 +1,24 @@
+FROM golang:1.17 as builder
+WORKDIR /app
+ARG VERSION
+COPY go.mod /app/go.mod
+COPY go.sum /app/go.sum
+RUN go mod download
+COPY ./ ./
+WORKDIR /app
+RUN go version
+RUN make build
+
+FROM ubuntu:bionic
+WORKDIR /app
+
+# install CA certificates
+RUN apt-get update && \
+  apt-get install -y ca-certificates && \
+  rm -Rf /var/lib/apt/lists/*  && \
+  rm -Rf /usr/share/doc && rm -Rf /usr/share/man  && \
+  apt-get clean
+
+COPY --from=builder /app/.bin/apm-hub /app
+
+ENTRYPOINT ["/app/apm-hub"]

Makefile

@@ -0,0 +1,44 @@
+NAME=apm-hub
+OS   = $(shell uname -s | tr '[:upper:]' '[:lower:]')
+ARCH = $(shell uname -m | sed 's/x86_64/amd64/')
+
+ifeq ($(VERSION),)
+  VERSION_TAG=$(shell git describe --abbrev=0 --tags --exact-match 2>/dev/null || echo latest)
+else
+  VERSION_TAG=$(VERSION)
+endif
+
+# Image URL to use all building/pushing image targets
+IMG ?= docker.io/flanksource/apm-hub:${VERSION_TAG}
+
+.PHONY: build
+build:
+	go build -o ./.bin/$(NAME) -ldflags "-X \"main.version=$(VERSION_TAG)\""  main.go
+
+.PHONY: linux
+linux:
+	GOOS=linux GOARCH=amd64 go build -o ./.bin/$(NAME)_linux_amd64 -ldflags "-X \"main.version=$(VERSION_TAG)\""  main.go
+	GOOS=linux GOARCH=arm64 go build -o ./.bin/$(NAME)_linux_arm64 -ldflags "-X \"main.version=$(VERSION_TAG)\""  main.go
+
+.PHONY: darwin
+darwin:
+	GOOS=darwin GOARCH=amd64 go build -o ./.bin/$(NAME)_darwin_amd64 -ldflags "-X \"main.version=$(VERSION_TAG)\""  main.go
+	GOOS=darwin GOARCH=arm64 go build -o ./.bin/$(NAME)_darwin_arm64 -ldflags "-X \"main.version=$(VERSION_TAG)\""  main.go
+
+.PHONY: windows
+windows:
+	GOOS=windows GOARCH=amd64 go build -o ./.bin/$(NAME).exe -ldflags "-X \"main.version=$(VERSION_TAG)\""  main.go
+
+.PHONY: compress
+compress:
+	# upx 3.95 has issues compressing darwin binaries - https://github.com/upx/upx/issues/301
+	which upx 2>&1 >  /dev/null  || (sudo apt-get update && sudo apt-get install -y xz-utils && wget -nv -O upx.tar.xz https://github.com/upx/upx/releases/download/v3.96/upx-3.96-amd64_linux.tar.xz; tar xf upx.tar.xz; mv upx-3.96-amd64_linux/upx /usr/bin )
+	upx -5 ./.bin/$(NAME)_linux_amd64 ./.bin/$(NAME)_linux_arm64 ./.bin/$(NAME)_darwin_amd64 ./.bin/$(NAME)_darwin_arm64 ./.bin/$(NAME).exe
+
+.PHONY: binaries
+binaries: linux darwin windows compress
+
+.PHONY: release
+release: binaries
+	mkdir -p .release
+	cp .bin/$(NAME)* .release/

api/context.go

@@ -0,0 +1,11 @@
+package api
+
+import (
+	"github.com/flanksource/kommons"
+	"github.com/labstack/echo/v4"
+)
+
+type Context struct {
+	echo.Context
+	Kommons *kommons.Client
+}

api/logs/logs.go

@@ -1,8 +1,22 @@
 package logs
 
+import (
+	"fmt"
+	"time"
+
+	"github.com/flanksource/kommons"
+)
+
+var GlobalBackends []SearchBackend
+
+type SearchConfig struct {
+	Backends []SearchBackend `yaml:"backends,omitempty"`
+}
+
 type SearchBackend struct {
-	Routes     []SearchRoute           `json:"routes,omitempty"`
-	Kubernetes KubernetesSearchBackend `json:"kubernetes,omitempty"`
+	Routes     []SearchRoute            `json:"routes,omitempty"`
+	Backend    SearchAPI                `json:"-"`
+	Kubernetes *KubernetesSearchBackend `json:"kubernetes,omitempty"`
 }
 
 type SearchRoute struct {
@@ -12,19 +26,24 @@ type SearchRoute struct {
 }
 
 type KubernetesSearchBackend struct {
-	Kubeconfig string `json:"kubeconfig,omitempty"`
+	// empty kubeconfig indicates to use the current kubeconfig for connection
+	Kubeconfig *kommons.EnvVar `json:"kubeconfig,omitempty"`
+	//namespace to search the kommons.EnvVar in
+	Namespace string `json:"namespace,omitempty"`
 }
 
 type SearchParams struct {
 	// Limit is the maximum number of results to return.
-	Limit int `json:"limit,omitempty"`
+	Limit      int64 `json:"limit,omitempty"`
+	LimitBytes int64 `json:"limitBytes,omitempty"`
 	// The page token, returned by a previous call, to request the next page of results.
-	Page   string            `json:"page,omitempty"`
+	Page string `json:"page,omitempty"`
+	// comma separated list of labels to filter the results. key1=value1,key2=value2
 	Labels map[string]string `json:"labels,omitempty"`
 	// A generic query string, that is rewritten to the underlying system,
 	// If the underlying system does not support queries, than this query is applied on the returned results
 	Query string `json:"query,omitempty"`
-	// A RFC3339 timestamp or an age string (e.g. "1h", "2d", "1w")
+	// A RFC3339 timestamp or an age string (e.g. "1h", "2d", "1w"), default to 1h
 	Start string `json:"start,omitempty"`
 	// A RFC3339 timestamp or an age string (e.g. "1h", "2d", "1w")
 	End string `json:"end,omitempty"`
@@ -33,6 +52,69 @@ type SearchParams struct {
 	// The identifier of the type of logs to find, e.g. k8s-node-1, k8s-service-1, k8s-pod-1, vm-1, etc.
 	// The ID should include include any cluster/namespace/account information required for routing
 	Id string `json:"id,omitempty"`
+
+	// Limits the number of log messages return per item, e.g. pod
+	LimitPerItem int64 `json:"limitPerItem,omitempty"`
+	// Limits the number of bytes returned per item, e.g. pod
+	LimitBytesPerItem int64 `json:"limitBytesPerItem,omitempty"`
+
+	start *time.Time `json:"-"`
+	end   *time.Time `json:"-"`
+}
+
+func (p SearchParams) GetStart() *time.Time {
+	if p.start != nil {
+		return p.start
+	}
+	if duration, err := time.ParseDuration(p.Start); err == nil {
+		t := time.Now().Add(-duration)
+		p.start = &t
+	} else if t, err := time.Parse(time.RFC3339, p.Start); err == nil {
+		p.start = &t
+	}
+	return p.start
+}
+
+func (p SearchParams) GetEnd() *time.Time {
+	if p.end != nil {
+		return p.end
+	}
+	if duration, err := time.ParseDuration(p.End); err == nil {
+		t := time.Now().Add(-duration)
+		p.end = &t
+	} else if t, err := time.Parse(time.RFC3339, p.End); err == nil {
+		p.end = &t
+	}
+	return p.start
+}
+
+func (q SearchParams) String() string {
+	s := ""
+	if q.Type != "" {
+		s += fmt.Sprintf("type=%s ", q.Type)
+	}
+	if q.Id != "" {
+		s += fmt.Sprintf("id=%s ", q.Id)
+	}
+	if q.Start != "" {
+		s += fmt.Sprintf("start=%s ", q.Start)
+	}
+	if q.Query != "" {
+		s += fmt.Sprintf("query=%s ", q.Query)
+	}
+	if q.Labels != nil && len(q.Labels) > 0 {
+		s += fmt.Sprintf("labels=%v ", q.Labels)
+	}
+	if q.End != "" {
+		s += fmt.Sprintf("end=%s ", q.End)
+	}
+	if q.Limit > 0 {
+		s += fmt.Sprintf("limit=%d ", q.Limit)
+	}
+	if q.Page != "" {
+		s += fmt.Sprintf("page=%s ", q.Page)
+	}
+	return s
 }
 
 type SearchResults struct {
@@ -51,9 +133,9 @@ type Result struct {
 }
 
 type SearchAPI interface {
-	Search(q SearchParams) (r SearchResults, err error)
+	Search(q *SearchParams) (r SearchResults, err error)
 }
 
 type SearchMapper interface {
-	MapSearchParams(p SearchParams) ([]SearchParams, error)
+	MapSearchParams(p *SearchParams) ([]SearchParams, error)
 }

cmd/root.go

@@ -0,0 +1,25 @@
+package cmd
+
+import (
+	"github.com/flanksource/commons/logger"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+var Root = &cobra.Command{
+	Use: "apm-hub",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		logger.UseZap(cmd.Flags())
+	},
+}
+
+var httpPort int
+
+func ServerFlags(flags *pflag.FlagSet) {
+	flags.IntVar(&httpPort, "httpPort", 8080, "Port to expose a health dashboard ")
+}
+
+func init() {
+	logger.BindFlags(Root.PersistentFlags())
+	Root.AddCommand(Serve)
+}