Prepare for first version (#2)

* Fix after project rename
* Add unit tests
* More unit testing testing
* Prepare for public

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Release Go Binaries
+name: Release
 
 on:
   release:
@@ -14,12 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux, windows, darwin]
+        goos: [linux, windows]
         goarch: [amd64, arm64]
         exclude:
           # windows/arm64 and darwin/arm64 seems useless
-          - goarch: arm64
-            goos: darwin
           - goarch: arm64
             goos: windows
     steps:

.github/workflows/go.yml → .github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: Build
+name: Test
 
 on:
   push:
@@ -35,6 +35,3 @@ jobs:
 
       - name: Test
         run: go test -v .
-
-      - name: Show Data
-        run: ./prometehus_export --version

.gitignore

@@ -91,5 +91,5 @@ vendor/
 .idea/misc.xml
 .idea/modules.xml
 .idea/vcs.xml
-/.idea/prometehus_export.iml
 dump/
+/.idea/prometheus_data_dump.iml

README.md

@@ -1,15 +1,31 @@
-# Prometheus Export
-Dump data from prometheus to JSON files.
+![Build Status](https://github.com/pokornyIt/prometheus_data_dump/workflows/Build/badge.svg)
+[![License](https://img.shields.io/github/license/pokornyIt/prometheus_data_dump)](/LICENSE)
+[![Go Report Card](https://goreportcard.com/badge/github.com/pokornyIt/prometheus_data_dump)](https://goreportcard.com/report/github.com/pokornyIt/nut_exporter)
+![GitHub tag (latest SemVer)](https://img.shields.io/github/v/tag/pokornyit/prometheus_data_dump?label=latest)
 
-Program dump all metrics for selected jobs into JSON files.
+# Prometheus Data Dump
+Project designed to export data from the Prometheus database.
+Exports are intend for further processing in other systems that do not support 
+direct integration to the Prometheus system as a data source.
 
-## Configuration
+Each Prometheus metric is export to a separate file. 
+The data is export for a defined number of days back and can be limited to selected "jobs". 
+Existing exported data is overwrite by the new export. 
+A special file "metrics-meta.json" is exported, which contains a description of individual metrics. 
+
+# Program start
+
+The program requires the entry of selected configuration parameters for its start. 
+This is mainly the address of the Prometheus server from which the data will be exported.
+
+## Configuration file
 
 Program has config file.
 ```yaml
-server: perftcl2.perflab.zoomint.com
+server: prometehus.server
 path: ./dump
 days: 2
+step: 10
 jobs:
   - node_exporter_zqm
   - cucm_monitor
@@ -18,10 +34,16 @@ jobs:
 - **server** - FQDN or IP address of prometheus server
 - **path** - Path for store export data
 - **days** - Number of day to exports (1-60)
+- **step** - Step for time slice in seconds (5 - 3600), default 10
 - **jobs** - limit data only for target jobs. If omitted or empty mean export all jobs
 
-## Line parameters
+## Configuration line parameters
 - **--config.show** - show actual configuration and exit
-- **--config.file=cfg.yml** - define config file, defaul is cfg.yml
+- **--config.file=cfg.yml** - define config file, default is cfg.yml
 - **--path=./dump** - overwrite path defined in  config file
 - **--server=IP** - FQDN or IP address of prometheus server
+
+# Contribute
+We welcome any contributions. Please fork the project on GitHub and open Pull Requests for any proposed changes.
+
+Please note that we will not merge any changes that encourage insecure behaviour. If in doubt please open an Issue first to discuss your proposal. 

api-reader.go

@@ -13,7 +13,7 @@ const UriFormat = "http://%s:9090/api/v1/%s"
 
 func getFormUri(uri string) (data []byte, err error) {
 	uri = fmt.Sprintf(UriFormat, config.Server, uri)
-	_ = level.Info(logger).Log("msg", "read data from server ", "uri", uri)
+	_ = level.Debug(logger).Log("msg", "read data from server ", "uri", uri)
 	req, err := http.NewRequest("GET", uri, nil)
 	if err != nil {
 		_ = level.Error(logger).Log("msg", "problem create request for uri "+uri, "error", err)
@@ -50,6 +50,6 @@ func finishApiRequest(req *http.Request) (data []byte, err error) {
 		_ = level.Error(logger).Log("msg", "problem read data from response", "error", err)
 		return nil, err
 	}
-	_ = level.Debug(logger).Log("msg", "success read "+string(len(bodies))+" bytes from uri")
+	_ = level.Debug(logger).Log("msg", fmt.Sprintf("success read %d bytes from uri", len(bodies)))
 	return bodies, nil
 }

config.go

@@ -74,27 +74,16 @@ func dirAccessible(directory string) bool {
 	return err == nil
 }
 
-func (c *Config) LoadFile(filename string) error {
-	if fileExists(filename) {
-		content, err := ioutil.ReadFile(filename)
-		if err != nil {
-			return err
-		}
-		err = yaml.UnmarshalStrict(content, c)
-		if err != nil {
-			err = json.Unmarshal(content, c)
-			if err != nil {
-				return err
-			}
-		}
-	}
+func (c *Config) overWriteFromLine() {
 	if len(*server) > 0 {
 		c.Server = *server
 	}
 	if len(*directoryData) > 0 {
 		c.Path = *directoryData
 	}
+}
 
+func (c *Config) validate() error {
 	match, err := regexp.MatchString("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", c.Server)
 	if !match || err != nil {
 		match, err = regexp.MatchString("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$", c.Server)
@@ -112,7 +101,7 @@ func (c *Config) LoadFile(filename string) error {
 	if c.Days < 1 || c.Days > 60 {
 		return errors.New("defined days back not valid (1 - 60)")
 	}
-	if c.Step < 1 || c.Step > 60*60 {
+	if c.Step < 5 || c.Step > 3600 {
 		c.Step = 10
 	}
 	if !dirExists(c.Path) {
@@ -124,6 +113,24 @@ func (c *Config) LoadFile(filename string) error {
 	return nil
 }
 
+func (c *Config) LoadFile(filename string) error {
+	if fileExists(filename) {
+		content, err := ioutil.ReadFile(filename)
+		if err != nil {
+			return err
+		}
+		err = yaml.UnmarshalStrict(content, c)
+		if err != nil {
+			err = json.Unmarshal(content, c)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	c.overWriteFromLine()
+	return c.validate()
+}
+
 func (c *Config) print() string {
 	a := fmt.Sprintf("\r\n%s\r\nActual configuration:\r\n", applicationName)
 	a = fmt.Sprintf("%sServer:       [%s]\r\n", a, c.Server)

config_test.go

@@ -0,0 +1,82 @@
+package main
+
+import "testing"
+
+func TestConfig_overWriteFromLine(t *testing.T) {
+	type args struct {
+		server string
+		path   string
+	}
+	type fields struct {
+		Server string
+		Path   string
+		Days   int
+		Jobs   []string
+		Step   int
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		expect args
+	}{
+		{"none", fields{"server.local", "/my", 1, nil, 10}, args{}, args{"server.local", "/my"}},
+		{"server", fields{"server.local", "/my", 1, nil, 10}, args{"server.new", ""}, args{"server.new", "/my"}},
+		{"path", fields{"server.local", "/my", 1, nil, 10}, args{"", "/newPath"}, args{"server.local", "/newPath"}},
+		{"both", fields{"server.local", "/my", 1, nil, 10}, args{"server.new", "/newPath"}, args{"server.new", "/newPath"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &Config{
+				Server: tt.fields.Server,
+				Path:   tt.fields.Path,
+				Days:   tt.fields.Days,
+				Jobs:   tt.fields.Jobs,
+				Step:   tt.fields.Step,
+			}
+			server = &tt.args.server
+			directoryData = &tt.args.path
+			c.overWriteFromLine()
+			if c.Server != tt.expect.server {
+				t.Errorf("overWriteFromLine() = server %s, want %s", c.Server, tt.expect.server)
+			}
+			if c.Path != tt.expect.path {
+				t.Errorf("overWriteFromLine() = path %s, want %s", c.Path, tt.expect.path)
+			}
+		})
+	}
+}
+
+func TestConfig_validate(t *testing.T) {
+	type fields struct {
+		Server string
+		Path   string
+		Days   int
+		Jobs   []string
+		Step   int
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantErr bool
+	}{
+		{"all valid", fields{"server.local", "./", 2, nil, 10}, false},
+		{"no server", fields{"", "./", 2, nil, 10}, true},
+		{"no path", fields{"", "", 2, nil, 10}, true},
+		{"wrong day", fields{"", "./", -2, nil, 10}, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &Config{
+				Server: tt.fields.Server,
+				Path:   tt.fields.Path,
+				Days:   tt.fields.Days,
+				Jobs:   tt.fields.Jobs,
+				Step:   tt.fields.Step,
+			}
+			if err := c.validate(); (err != nil) != tt.wantErr {
+				t.Errorf("validate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

go.mod

@@ -1,14 +1,11 @@
-module github.com/pokornyIt/prometehus_export
+module github.com/pokornyIt/prometehus_data_dump
 
 go 1.14
 
 require (
 	github.com/go-kit/kit v0.9.0
-	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/common v0.10.0
-	github.com/prometheus/prometheus v2.5.0+incompatible // indirect
-	github.com/prometheus/tsdb v0.10.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v2 v2.2.5
 )