feat:add vlogs server.

.github/workflows/services.yml

@@ -87,7 +87,7 @@ jobs:
     strategy:
       matrix:
         ## TODO: add more modules
-        module: [ database, pay, account, minio, launchpad, exceptionmonitor, aiproxy, devbox ]
+        module: [ database, pay, account, minio, launchpad, exceptionmonitor, aiproxy, devbox, vlogs ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -184,7 +184,7 @@ jobs:
     strategy:
       matrix:
         ## TODO: add more modules
-        module: [ database, pay, account, minio, launchpad, exceptionmonitor, aiproxy, devbox ]
+        module: [ database, pay, account, minio, launchpad, exceptionmonitor, aiproxy, devbox, vlogs ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4

service/go.work

@@ -9,6 +9,7 @@ use (
 	./launchpad
 	./pay
 	./devbox
+	./vlogs
 )
 
 replace (

service/go.work.sum

@@ -746,17 +746,24 @@ github.com/astaxie/beego v1.12.3 h1:SAQkdD2ePye+v8Gn1r4X6IKZM1wd28EyUOVQ3PDSOOQ=
 github.com/astaxie/beego v1.12.3/go.mod h1:p3qIm0Ryx7zeBHLljmd7omloyca1s4yu1a8kM1FkpIA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 h1:woXadbf0c7enQ2UGCi8gW/WuKmE0xIzxBF/eD94jMKQ=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19/go.mod h1:zminj5ucw7w0r65bP6nhyOd3xL6veAUMc3ElGMoLVb4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4/go.mod h1:4GQbF1vJzG60poZqWatZlhP31y8PGCCVTvIGPdaaYJ0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug=
 github.com/aws/aws-sdk-go-v2/service/sso v1.24.5/go.mod h1:wrMCEwjFPms+V86TCQQeOxQF/If4vT44FGIOFiMC2ck=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4/go.mod h1:Tp/ly1cTjRLGBBmNccFumbZ8oqpZlpdhFf80SrRh4is=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY=
 github.com/aws/aws-sdk-go-v2/service/sts v1.32.4/go.mod h1:9XEUty5v5UAsMiFOBJrNibZgwCeOma73jgGwwhgffa8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8=
+github.com/bazelbuild/rules_go v0.49.0 h1:5vCbuvy8Q11g41lseGJDc5vxhDjJtfxr6nM/IC4VmqM=
 github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -1870,6 +1877,7 @@ google.golang.org/genproto/googleapis/bytestream v0.0.0-20231030173426-d783a09b4
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231030173426-d783a09b4405/go.mod h1:GRUCuLdzVqZte8+Dl/D4N25yLzcGqqWaYkeVOwulFqw=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20241021214115-324edc3d5d38 h1:42FIsHvG4GOaVHLDMcy/YMqC4clCbgAPojbcA2hXp5w=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20241021214115-324edc3d5d38/go.mod h1:T8O3fECQbif8cez15vxAcjbwXxvL2xbnvbQ7ZfiMAMs=
+google.golang.org/genproto/googleapis/bytestream v0.0.0-20241118233622-e639e219e697 h1:rY93Be8/KL+EtFM4im9lxMzjGn796GnwVUd75cyFCJg=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20241118233622-e639e219e697/go.mod h1:qUsLYwbwz5ostUWtuFuXPlHmSJodC5NI/88ZlHj4M1o=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=

service/pkg/api/req.go

@@ -70,6 +70,28 @@ type Stats struct {
 	ExecutionTimeMsec int    `json:"executionTimeMsec"`
 }
 
+type JSONQuery struct {
+	Key string
+	// There are a total of four modes,"=" equal,"!" Not equal,"~" including,"!~" Not included.
+	Mode  string
+	Value string
+}
+
+type VlogsRequest struct {
+	Time        string      `json:"time"`
+	Namespace   string      `json:"namespace"`
+	App         string      `json:"app"`
+	Limit       string      `json:"limit"`
+	JSONMode    string      `json:"jsonMode"`
+	StderrMode  string      `json:"stderrMode"`
+	NumberMode  string      `json:"numberMode"`
+	NumberLevel string      `json:"numberLevel"`
+	Pod         []string    `json:"pod"`
+	Container   []string    `json:"container"`
+	Keyword     string      `json:"keyword"`
+	JSONQuery   []JSONQuery `json:"jsonQuery"`
+}
+
 var (
 	Mysql = map[string]string{
 		"cpu":           "round(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=~\"#\",pod=~\"@-mysql-\\\\d\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=~\"#\",pod=~\"@-mysql-\\\\d\"}) by (pod)*100,0.01)",

service/vlogs/Dockerfile

@@ -0,0 +1,7 @@
+FROM gcr.io/distroless/static:nonroot
+ARG TARGETARCH
+COPY bin/service-vlogs-$TARGETARCH /manager
+EXPOSE 8428
+USER 65532:65532
+
+ENTRYPOINT [ "/manager", "config/config.yml" ]

service/vlogs/Makefile

@@ -0,0 +1,55 @@
+IMG ?= ghcr.io/labring/sealos-vlogs-service:latest
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+# only support linux, non cgo
+PLATFORMS ?= linux_arm64 linux_amd64
+GOOS=linux
+CGO_ENABLED=0
+GOARCH=$(shell go env GOARCH)
+
+GO_BUILD_FLAGS=-trimpath -ldflags "-s -w"
+
+.PHONY: all
+all: build
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Build
+
+.PHONY: clean
+clean:
+	rm -f $(SERVICE_NAME)
+
+.PHONY: build
+build: clean ## Build service-hub binary.
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) go build $(GO_BUILD_FLAGS) -o bin/manager main.go
+
+.PHONY: docker-build
+docker-build: build
+	mv bin/manager bin/service-vlogs-${TARGETARCH}
+	docker build -t $(IMG) .
+
+.PHONY: docker-push
+docker-push:
+	docker push $(IMG)

service/vlogs/README.md

@@ -0,0 +1,54 @@
+# RESTServer
+
+## Description
+
+restserver for victoria logs
+
+## Getting Started
+
+### Running on the cluster
+
+1. Build and push your image to the location specified by `IMG`:
+
+```sh
+make docker-build docker-push IMG=<some-registry>/sealos-vlogs-service:latest
+```
+
+2. Deploy the restserver:
+
+```sh
+kubectl apply -f deploy/manifests/
+```
+
+### How it works
+
+To enable the database frontend application to retrieve monitoring data, you need to modify the environment variable `MONITOR_URL` of the frontend deployment to the corresponding address of the restserver.
+
+Additionally, to configure the data source, you need to set the environment variable `VM_SERVICE_HOST` of the restserver deployment to the correct address.
+
+```
+e.g.
+http://prometheus.sealos.svc.cluster.local
+```
+
+## License
+
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+
+distributed under the License is distributed on an "AS IS" BASIS,
+
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+See the License for the specific language governing permissions and
+
+limitations under the License.

service/vlogs/config/config.yml

@@ -0,0 +1,2 @@
+server:
+  addr: ":8428"

service/vlogs/deploy/Kubefile

@@ -0,0 +1,5 @@
+FROM scratch
+COPY registry registry
+COPY manifests manifests
+
+CMD ["kubectl apply -f manifests/deploy.yaml"]

service/vlogs/deploy/README.md

@@ -0,0 +1,16 @@
+### docker image build and deploy
+```bash
+make docker-build IMG=$(YourImageName)
+# edit deploy/manifests/depoly.yaml and deploy
+kubectl apply -f deploy/manifests/depoly.yaml
+```
+
+### cluster image build and deploy
+```bash
+```
+
+### Victoria Metrics
+
+In order to prevent performance degradation or abnormal behavior caused by excessive data size in Prometheus, VictoriaMetrics is utilized for data collection.
+
+> By default, we use kb-prometheus-server for the data collection service.

service/vlogs/deploy/manifests/deploy.yaml

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: service-vlogs
+  name: service-vlogs-config
+  namespace: sealos
+data:
+  config.yml: |
+    server:
+      addr: ":8428"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: service-vlogs
+  name: service-vlogs-deployment
+  namespace: sealos
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: service-vlogs
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: service-vlogs
+    spec:
+      containers:
+        - args:
+            - /config/config.yml
+          command:
+            - /manager
+          image: ghcr.io/labring/sealos-vlogs-service:latest
+          imagePullPolicy: Always
+          name: service-vlogs
+          ports:
+            - containerPort: 8428
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 500m
+              memory: 1Gi
+            requests:
+              cpu: 5m
+              memory: 64Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            runAsNonRoot: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          env:
+            - name: WHITELIST_KUBERNETES_HOSTS
+              value: "https://usw.sailos.io:6443"
+          volumeMounts:
+            - mountPath: /config
+              name: config-vol
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      volumes:
+        - configMap:
+            defaultMode: 420
+            name: service-vlogs-config
+          name: config-vol
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: service-vlogs
+  name: service-vlogs
+  namespace: sealos
+spec:
+  ports:
+    - name: http
+      port: 8428
+      protocol: TCP
+      targetPort: 8428
+  selector:
+    app: service-vlogs

service/vlogs/go.mod

@@ -0,0 +1,10 @@
+module github.com/labring/sealos/service/vlogs
+
+go 1.22.7
+
+require (
+	github.com/labring/sealos/service v0.0.0
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+replace github.com/labring/sealos/service => ../

service/vlogs/main.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"os"
+
+	vlogsServer "github.com/labring/sealos/service/vlogs/server"
+)
+
+type RestartableServer struct {
+	configFile string
+}
+
+func (rs *RestartableServer) Serve(c *vlogsServer.Config) {
+	var vs, err = vlogsServer.NewVLogsServer(c)
+	if err != nil {
+		fmt.Printf("Failed to create auth server: %s\n", err)
+		return
+	}
+
+	hs := &http.Server{
+		Addr:    c.Server.ListenAddress,
+		Handler: vs,
+	}
+
+	var listener net.Listener
+	listener, err = net.Listen("tcp", c.Server.ListenAddress)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Printf("Serve on %s\n", c.Server.ListenAddress)
+	if err := hs.Serve(listener); err != nil {
+		fmt.Println(err)
+		return
+	}
+}
+
+func main() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+	flag.Parse()
+
+	cf := flag.Arg(0)
+	if cf == "" {
+		fmt.Println("Config file not sepcified")
+		return
+	}
+
+	config, err := vlogsServer.InitConfig(cf)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	rs := RestartableServer{
+		configFile: cf,
+	}
+	rs.Serve(config)
+}