Read data and store data (#1)

* Read target data details
* Save meta data
* Dump all metrics for defined jobs
* Add gitgub automation
* Update read me

.github/workflows/go.yml

@@ -0,0 +1,40 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.14
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+              dep ensure
+          fi
+      - name: Build
+        run: go build -v -ldflags="-X 'main.Version=0.0.1' -X 'main.Branch=$(git rev-parse --short HEAD)' -X 'main.BuildDate=$(date -Is)' -X main.BuildUser='$(id -u -n)'" .
+
+      - name: Test
+        run: go test -v .
+
+      - name: Show Data
+        run: ./prometehus_export --version

.github/workflows/release.yml

@@ -0,0 +1,43 @@
+name: Release Go Binaries
+
+on:
+  release:
+    types: [created]
+
+#env:
+#  CMD_PATH: ./cmd/vt2geojson
+
+
+jobs:
+  releases-matrix:
+    name: Release Matrix
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, windows, darwin]
+        goarch: [amd64, arm64]
+        exclude:
+          # windows/arm64 and darwin/arm64 seems useless
+          - goarch: arm64
+            goos: darwin
+          - goarch: arm64
+            goos: windows
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set APP_VERSION env
+        run: echo ::set-env name=APP_VERSION::$(echo ${GITHUB_REF} | rev | cut -d'/' -f 1 | rev )
+      - name: Set BUILD_TIME env
+        run: echo ::set-env name=BUILD_TIME::$(date)
+      - name: Environment Printer
+        uses: managedkaos/[email protected]
+
+      - uses: wangyoucao577/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          pre_command: go get -v ./...
+          #          project_path: "${{ env.CMD_PATH }}"
+          build_flags: -v
+          ldflags: -X "main.Version=${{ env.APP_VERSION }}" -X "main.BuildDate=${{ env.BUILD_TIME }}" -X main.Branch=${{ github.sha }} -X main.Revision=${{ github.ref }}

.gitignore

@@ -91,4 +91,5 @@ vendor/
 .idea/misc.xml
 .idea/modules.xml
 .idea/vcs.xml
-/.idea/nut_exporter.iml
+/.idea/prometehus_export.iml
+dump/

README.md

@@ -1,3 +1,27 @@
 # Prometheus Export
-Dump data from prometheus to JSON file
+Dump data from prometheus to JSON files.
 
+Program dump all metrics for selected jobs into JSON files.
+
+## Configuration
+
+Program has config file.
+```yaml
+server: perftcl2.perflab.zoomint.com
+path: ./dump
+days: 2
+jobs:
+  - node_exporter_zqm
+  - cucm_monitor
+``` 
+
+- **server** - FQDN or IP address of prometheus server
+- **path** - Path for store export data
+- **days** - Number of day to exports (1-60)
+- **jobs** - limit data only for target jobs. If omitted or empty mean export all jobs
+
+## Line parameters
+- **--config.show** - show actual configuration and exit
+- **--config.file=cfg.yml** - define config file, defaul is cfg.yml
+- **--path=./dump** - overwrite path defined in  config file
+- **--server=IP** - FQDN or IP address of prometheus server

api-reader.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"github.com/go-kit/kit/log/level"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+const UriFormat = "http://%s:9090/api/v1/%s"
+
+func getFormUri(uri string) (data []byte, err error) {
+	uri = fmt.Sprintf(UriFormat, config.Server, uri)
+	_ = level.Info(logger).Log("msg", "read data from server ", "uri", uri)
+	req, err := http.NewRequest("GET", uri, nil)
+	if err != nil {
+		_ = level.Error(logger).Log("msg", "problem create request for uri "+uri, "error", err)
+		return nil, err
+	}
+	return finishApiRequest(req)
+}
+
+//func postFromUri(uri string, body []byte) (data []byte, err error) {
+//	uri = fmt.Sprintf(UriFormat, config.Server, uri)
+//	_ = level.Info(logger).Log("msg", "post and read data from server ", "uri", uri)
+//	req, err := http.NewRequest("POST", uri, bytes.NewBuffer(body))
+//	if err != nil {
+//		_ = level.Error(logger).Log("msg", "problem create request for uri "+uri, "error", err)
+//		return nil, err
+//	}
+//	return finishApiRequest(req)
+//}
+
+func finishApiRequest(req *http.Request) (data []byte, err error) {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Timeout: time.Duration(connectionTimeout) * time.Second, Transport: tr}
+	_ = level.Debug(logger).Log("msg", "try read data from uri")
+	resp, err := client.Do(req)
+	if err != nil {
+		_ = level.Error(logger).Log("msg", "problem get data from server", "error", err)
+		return nil, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	bodies, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		_ = level.Error(logger).Log("msg", "problem read data from response", "error", err)
+		return nil, err
+	}
+	_ = level.Debug(logger).Log("msg", "success read "+string(len(bodies))+" bytes from uri")
+	return bodies, nil
+}

api-response.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"encoding/json"
+	"github.com/go-kit/kit/log/level"
+)
+
+type errorType string
+
+type ApiResponse struct {
+	Status    string           `json:"status"`
+	Data      *json.RawMessage `json:"data,omitempty"`
+	ErrorType errorType        `json:"errorType,omitempty"`
+	Error     string           `json:"error,omitempty"`
+	Warnings  []string         `json:"warnings,omitempty"`
+	Response  []byte
+}
+
+func GetApiData(uri string) (*ApiResponse, error) {
+	data, err := getFormUri(uri)
+	if err != nil {
+		_ = level.Error(logger).Log("msg", "problem collect targets details")
+		return nil, err
+	}
+	var api ApiResponse
+	if err := json.Unmarshal(data, &api); err != nil {
+		return nil, err
+	}
+	api.Response = make([]byte, len(data))
+	copy(api.Response, data)
+	return &api, nil
+}
+
+func (a *ApiResponse) statusSuccess() bool {
+	return a.Status == "success"
+}

cfg.yml

@@ -0,0 +1,6 @@
+server: perftcl2.perflab.zoomint.com
+path: ./dump
+days: 2
+jobs:
+  - node_exporter_zqm
+  - cucm_monitor

config.go

@@ -3,28 +3,37 @@ package main
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"gopkg.in/alecthomas/kingpin.v2"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"regexp"
+	"strings"
 )
 
+const connectionTimeout = 10
+
 type Config struct {
-	Server string `yaml:"server" json:"server"`
-	Path   string `yaml:"path" json:"path"`
-	Days   int    `yaml:"days" json:"days"`
+	Server string   `yaml:"server" json:"server"` // FQDN or IP address of server
+	Path   string   `yaml:"path" json:"path"`     // path to store directory
+	Days   int      `yaml:"days" json:"days"`
+	Jobs   []string `yaml:"jobs" json:"jobs"`
+	Step   int      `yaml:"step" json:"step"`
 }
 
 var (
-	showConfig = kingpin.Flag("config.show", "Show actual configuration and ends").Default("false").Bool()
-	configFile = kingpin.Flag("config.file", "Configuration file default is \"cfg.yml\".").PlaceHolder("cfg.yml").Default("cfg.yml").String()
-	path       = kingpin.Flag("path", "Path where store export json data").PlaceHolder("path").Default("./dump").String()
-	server     = kingpin.Flag("server", "Prometheus server FQDN or IP address").PlaceHolder("server").Default("").String()
-	config     = &Config{
+	showConfig    = kingpin.Flag("config.show", "Show actual configuration and ends").Default("false").Bool()
+	configFile    = kingpin.Flag("config.file", "Configuration file default is \"cfg.yml\".").PlaceHolder("cfg.yml").Default("cfg.yml").String()
+	directoryData = kingpin.Flag("path", "Path where store export json data").PlaceHolder("path").Default("./dump").String()
+	server        = kingpin.Flag("server", "Prometheus server FQDN or IP address").PlaceHolder("server").Default("").String()
+	config        = &Config{
 		Server: "",
 		Path:   "./dump",
 		Days:   1,
+		Jobs:   []string{},
+		Step:   10,
 	}
 )
 
@@ -44,8 +53,25 @@ func dirExists(path string) bool {
 	return info.IsDir()
 }
 
-func dirAccessible(path string) bool {
-
+func dirAccessible(directory string) bool {
+	var file string
+	dir, err := filepath.Abs(directory)
+	if err != nil {
+		dir = directory
+	}
+	for ok := true; ok; ok = fileExists(file) {
+		file = filepath.Join(dir, RandomString()+".tmp")
+	}
+	f, err := os.Create(file)
+	if err == nil {
+		_ = f.Close()
+		e := os.Remove(file)
+		if e != nil {
+			fmt.Printf("Problem test access to store directory. " + e.Error())
+			os.Exit(1)
+		}
+	}
+	return err == nil
 }
 
 func (c *Config) LoadFile(filename string) error {
@@ -65,8 +91,8 @@ func (c *Config) LoadFile(filename string) error {
 	if len(*server) > 0 {
 		c.Server = *server
 	}
-	if len(*path) > 0 {
-		c.Path = *path
+	if len(*directoryData) > 0 {
+		c.Path = *directoryData
 	}
 
 	match, err := regexp.MatchString("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", c.Server)
@@ -79,9 +105,16 @@ func (c *Config) LoadFile(filename string) error {
 	if len(c.Path) < 1 {
 		c.Path = "./dump"
 	}
+	p, err := filepath.Abs(c.Path)
+	if err == nil {
+		c.Path = p
+	}
 	if c.Days < 1 || c.Days > 60 {
 		return errors.New("defined days back not valid (1 - 60)")
 	}
+	if c.Step < 1 || c.Step > 60*60 {
+		c.Step = 10
+	}
 	if !dirExists(c.Path) {
 		return errors.New("path not exists")
 	}
@@ -90,3 +123,20 @@ func (c *Config) LoadFile(filename string) error {
 	}
 	return nil
 }
+
+func (c *Config) print() string {
+	a := fmt.Sprintf("\r\n%s\r\nActual configuration:\r\n", applicationName)
+	a = fmt.Sprintf("%sServer:       [%s]\r\n", a, c.Server)
+	a = fmt.Sprintf("%sData path:    [%s]\r\n", a, c.Path)
+	a = fmt.Sprintf("%sDays back:    [%d]\r\n", a, c.Days)
+	if len(c.Jobs) == 0 {
+		a = fmt.Sprintf("%sTargets:      [--all--]\r\n", a)
+	} else {
+		a = fmt.Sprintf("%sJobs:         [%s]\r\n", a, strings.Join(c.Jobs, ", "))
+	}
+	return a
+}
+
+func (c *Config) filePath(fileName string) string {
+	return filepath.Join(c.Path, fileName)
+}

go.mod

@@ -4,7 +4,11 @@ go 1.14
 
 require (
 	github.com/go-kit/kit v0.9.0
+	github.com/opentracing/opentracing-go v1.1.0 // indirect
+	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/common v0.10.0
+	github.com/prometheus/prometheus v2.5.0+incompatible // indirect
+	github.com/prometheus/tsdb v0.10.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v2 v2.2.5
 )