diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..18f3a9d --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,40 @@ +name: Build + +on: + push: + branches: + - '**' + pull_request: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.14 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + go get -v -t -d ./... + if [ -f Gopkg.toml ]; then + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + dep ensure + fi + - name: Build + run: go build -v -ldflags="-X 'main.Version=0.0.1' -X 'main.Branch=$(git rev-parse --short HEAD)' -X 'main.BuildDate=$(date -Is)' -X main.BuildUser='$(id -u -n)'" . + + - name: Test + run: go test -v . + + - name: Show Data + run: ./prometehus_export --version \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..778cde0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Release Go Binaries + +on: + release: + types: [created] + +#env: +# CMD_PATH: ./cmd/vt2geojson + + +jobs: + releases-matrix: + name: Release Matrix + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux, windows, darwin] + goarch: [amd64, arm64] + exclude: + # windows/arm64 and darwin/arm64 seems useless + - goarch: arm64 + goos: darwin + - goarch: arm64 + goos: windows + steps: + - uses: actions/checkout@v2 + + - name: Set APP_VERSION env + run: echo ::set-env name=APP_VERSION::$(echo ${GITHUB_REF} | rev | cut -d'/' -f 1 | rev ) + - name: Set BUILD_TIME env + run: echo ::set-env name=BUILD_TIME::$(date) + - name: Environment Printer + uses: managedkaos/print-env@v1.0 + + - uses: wangyoucao577/go-release-action@v1.5 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + pre_command: go get -v ./... + # project_path: "${{ env.CMD_PATH }}" + build_flags: -v + ldflags: -X "main.Version=${{ env.APP_VERSION }}" -X "main.BuildDate=${{ env.BUILD_TIME }}" -X main.Branch=${{ github.sha }} -X main.Revision=${{ github.ref }} diff --git a/.gitignore b/.gitignore index 2a190f8..42d4bbd 100644 --- a/.gitignore +++ b/.gitignore @@ -91,4 +91,5 @@ vendor/ .idea/misc.xml .idea/modules.xml .idea/vcs.xml -/.idea/nut_exporter.iml +/.idea/prometehus_export.iml +dump/ diff --git a/.idea/prometehus_export.iml b/.idea/prometehus_export.iml deleted file mode 100644 index 5e764c4..0000000 --- a/.idea/prometehus_export.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/README.md b/README.md index e986150..a1f3f4b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,27 @@ # Prometheus Export -Dump data from prometheus to JSON file +Dump data from prometheus to JSON files. +Program dump all metrics for selected jobs into JSON files. + +## Configuration + +Program has config file. +```yaml +server: perftcl2.perflab.zoomint.com +path: ./dump +days: 2 +jobs: + - node_exporter_zqm + - cucm_monitor +``` + +- **server** - FQDN or IP address of prometheus server +- **path** - Path for store export data +- **days** - Number of day to exports (1-60) +- **jobs** - limit data only for target jobs. If omitted or empty mean export all jobs + +## Line parameters +- **--config.show** - show actual configuration and exit +- **--config.file=cfg.yml** - define config file, defaul is cfg.yml +- **--path=./dump** - overwrite path defined in config file +- **--server=IP** - FQDN or IP address of prometheus server diff --git a/api-reader.go b/api-reader.go new file mode 100644 index 0000000..8725a74 --- /dev/null +++ b/api-reader.go @@ -0,0 +1,55 @@ +package main + +import ( + "crypto/tls" + "fmt" + "github.com/go-kit/kit/log/level" + "io/ioutil" + "net/http" + "time" +) + +const UriFormat = "http://%s:9090/api/v1/%s" + +func getFormUri(uri string) (data []byte, err error) { + uri = fmt.Sprintf(UriFormat, config.Server, uri) + _ = level.Info(logger).Log("msg", "read data from server ", "uri", uri) + req, err := http.NewRequest("GET", uri, nil) + if err != nil { + _ = level.Error(logger).Log("msg", "problem create request for uri "+uri, "error", err) + return nil, err + } + return finishApiRequest(req) +} + +//func postFromUri(uri string, body []byte) (data []byte, err error) { +// uri = fmt.Sprintf(UriFormat, config.Server, uri) +// _ = level.Info(logger).Log("msg", "post and read data from server ", "uri", uri) +// req, err := http.NewRequest("POST", uri, bytes.NewBuffer(body)) +// if err != nil { +// _ = level.Error(logger).Log("msg", "problem create request for uri "+uri, "error", err) +// return nil, err +// } +// return finishApiRequest(req) +//} + +func finishApiRequest(req *http.Request) (data []byte, err error) { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Timeout: time.Duration(connectionTimeout) * time.Second, Transport: tr} + _ = level.Debug(logger).Log("msg", "try read data from uri") + resp, err := client.Do(req) + if err != nil { + _ = level.Error(logger).Log("msg", "problem get data from server", "error", err) + return nil, err + } + defer func() { _ = resp.Body.Close() }() + bodies, err := ioutil.ReadAll(resp.Body) + if err != nil { + _ = level.Error(logger).Log("msg", "problem read data from response", "error", err) + return nil, err + } + _ = level.Debug(logger).Log("msg", "success read "+string(len(bodies))+" bytes from uri") + return bodies, nil +} diff --git a/api-response.go b/api-response.go new file mode 100644 index 0000000..22a9ae1 --- /dev/null +++ b/api-response.go @@ -0,0 +1,36 @@ +package main + +import ( + "encoding/json" + "github.com/go-kit/kit/log/level" +) + +type errorType string + +type ApiResponse struct { + Status string `json:"status"` + Data *json.RawMessage `json:"data,omitempty"` + ErrorType errorType `json:"errorType,omitempty"` + Error string `json:"error,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Response []byte +} + +func GetApiData(uri string) (*ApiResponse, error) { + data, err := getFormUri(uri) + if err != nil { + _ = level.Error(logger).Log("msg", "problem collect targets details") + return nil, err + } + var api ApiResponse + if err := json.Unmarshal(data, &api); err != nil { + return nil, err + } + api.Response = make([]byte, len(data)) + copy(api.Response, data) + return &api, nil +} + +func (a *ApiResponse) statusSuccess() bool { + return a.Status == "success" +} diff --git a/cfg.yml b/cfg.yml new file mode 100644 index 0000000..1ae28db --- /dev/null +++ b/cfg.yml @@ -0,0 +1,6 @@ +server: perftcl2.perflab.zoomint.com +path: ./dump +days: 2 +jobs: + - node_exporter_zqm + - cucm_monitor diff --git a/config.go b/config.go index 2c6d624..2ca5aab 100644 --- a/config.go +++ b/config.go @@ -3,28 +3,37 @@ package main import ( "encoding/json" "errors" + "fmt" "gopkg.in/alecthomas/kingpin.v2" "gopkg.in/yaml.v2" "io/ioutil" "os" + "path/filepath" "regexp" + "strings" ) +const connectionTimeout = 10 + type Config struct { - Server string `yaml:"server" json:"server"` - Path string `yaml:"path" json:"path"` - Days int `yaml:"days" json:"days"` + Server string `yaml:"server" json:"server"` // FQDN or IP address of server + Path string `yaml:"path" json:"path"` // path to store directory + Days int `yaml:"days" json:"days"` + Jobs []string `yaml:"jobs" json:"jobs"` + Step int `yaml:"step" json:"step"` } var ( - showConfig = kingpin.Flag("config.show", "Show actual configuration and ends").Default("false").Bool() - configFile = kingpin.Flag("config.file", "Configuration file default is \"cfg.yml\".").PlaceHolder("cfg.yml").Default("cfg.yml").String() - path = kingpin.Flag("path", "Path where store export json data").PlaceHolder("path").Default("./dump").String() - server = kingpin.Flag("server", "Prometheus server FQDN or IP address").PlaceHolder("server").Default("").String() - config = &Config{ + showConfig = kingpin.Flag("config.show", "Show actual configuration and ends").Default("false").Bool() + configFile = kingpin.Flag("config.file", "Configuration file default is \"cfg.yml\".").PlaceHolder("cfg.yml").Default("cfg.yml").String() + directoryData = kingpin.Flag("path", "Path where store export json data").PlaceHolder("path").Default("./dump").String() + server = kingpin.Flag("server", "Prometheus server FQDN or IP address").PlaceHolder("server").Default("").String() + config = &Config{ Server: "", Path: "./dump", Days: 1, + Jobs: []string{}, + Step: 10, } ) @@ -44,8 +53,25 @@ func dirExists(path string) bool { return info.IsDir() } -func dirAccessible(path string) bool { - +func dirAccessible(directory string) bool { + var file string + dir, err := filepath.Abs(directory) + if err != nil { + dir = directory + } + for ok := true; ok; ok = fileExists(file) { + file = filepath.Join(dir, RandomString()+".tmp") + } + f, err := os.Create(file) + if err == nil { + _ = f.Close() + e := os.Remove(file) + if e != nil { + fmt.Printf("Problem test access to store directory. " + e.Error()) + os.Exit(1) + } + } + return err == nil } func (c *Config) LoadFile(filename string) error { @@ -65,8 +91,8 @@ func (c *Config) LoadFile(filename string) error { if len(*server) > 0 { c.Server = *server } - if len(*path) > 0 { - c.Path = *path + if len(*directoryData) > 0 { + c.Path = *directoryData } match, err := regexp.MatchString("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", c.Server) @@ -79,9 +105,16 @@ func (c *Config) LoadFile(filename string) error { if len(c.Path) < 1 { c.Path = "./dump" } + p, err := filepath.Abs(c.Path) + if err == nil { + c.Path = p + } if c.Days < 1 || c.Days > 60 { return errors.New("defined days back not valid (1 - 60)") } + if c.Step < 1 || c.Step > 60*60 { + c.Step = 10 + } if !dirExists(c.Path) { return errors.New("path not exists") } @@ -90,3 +123,20 @@ func (c *Config) LoadFile(filename string) error { } return nil } + +func (c *Config) print() string { + a := fmt.Sprintf("\r\n%s\r\nActual configuration:\r\n", applicationName) + a = fmt.Sprintf("%sServer: [%s]\r\n", a, c.Server) + a = fmt.Sprintf("%sData path: [%s]\r\n", a, c.Path) + a = fmt.Sprintf("%sDays back: [%d]\r\n", a, c.Days) + if len(c.Jobs) == 0 { + a = fmt.Sprintf("%sTargets: [--all--]\r\n", a) + } else { + a = fmt.Sprintf("%sJobs: [%s]\r\n", a, strings.Join(c.Jobs, ", ")) + } + return a +} + +func (c *Config) filePath(fileName string) string { + return filepath.Join(c.Path, fileName) +} diff --git a/go.mod b/go.mod index 93145de..dfbf95d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,11 @@ go 1.14 require ( github.com/go-kit/kit v0.9.0 + github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/prometheus/client_golang v1.0.0 github.com/prometheus/common v0.10.0 + github.com/prometheus/prometheus v2.5.0+incompatible // indirect + github.com/prometheus/tsdb v0.10.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.2.5 ) diff --git a/go.sum b/go.sum index 9754cbf..1712a84 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -5,13 +6,17 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -22,6 +27,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -30,6 +36,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -39,6 +47,7 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -50,11 +59,16 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.0 h1:wCi7urQOGBsYcQROHqpUUX4ct84xp40t9R9JX0FuA/U= github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -66,12 +80,19 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/prometheus v1.8.2 h1:PAL466mnJw1VolZPm1OarpdUpqukUy/eX4tagia17DM= +github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= +github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= +github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= +github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -82,11 +103,13 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= diff --git a/main.go b/main.go index ca4014d..0b04bba 100644 --- a/main.go +++ b/main.go @@ -1,29 +1,109 @@ package main import ( + "fmt" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/common/promlog" "github.com/prometheus/common/promlog/flag" "github.com/prometheus/common/version" "gopkg.in/alecthomas/kingpin.v2" + "math/rand" + "os" + "strings" + "sync" + "time" ) const ( applicationName = "prometheus_export" + letterBytes = "abcdefghijklmnopqrstuvwxyz" // map for random string + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + sb.WriteByte(letterBytes[idx]) + i-- + } + cache >>= letterIdxBits + remain-- + } + return sb.String() +} -func main() { +func containsString(slice []string, item string) bool { + set := make(map[string]struct{}, len(slice)) + for _, s := range slice { + set[s] = struct{}{} + } + + _, ok := set[item] + return ok +} + +func saveMetaData() { + l, _ := readTargetsList() + metricsMeta = NewMetricsMetaList(*l) + metricsMeta.onlyForJobs(config.Jobs) + metricsMeta.saveList() + + _ = level.Info(logger).Log("msg", fmt.Sprintf("metrics in meta data %d", len(*metricsMeta))) +} + +func saveData() { + var wg sync.WaitGroup + _ = level.Debug(logger).Log("msg", fmt.Sprintf("start %d goroutines for collect metrics for %d days ", len(*metricsMeta), config.Days)) + for _, metrics := range *metricsMeta { + wg.Add(1) + go collectOneMetrics(&wg, metrics.Metric) + } + _ = level.Debug(logger).Log("msg", "all goroutines started") + wg.Wait() + _ = level.Debug(logger).Log("msg", "data store finish") +} + +func collectOneMetrics(wg *sync.WaitGroup, metricName string) { + defer wg.Done() + m := Matrix{} + for i := 0; i < config.Days; i++ { + c, err := getRangeDay(metricName+"{}", 1) + if err != nil { + continue + } + for _, series := range *c { + m = append(m, series) + } + } + if len(m) == 0 { + _ = level.Error(logger).Log("msg", "for metrics "+metricName+" not any data") + } else { + m.save(metricName) + } +} + +func main() { promlogConfig := &promlog.Config{} flag.AddFlags(kingpin.CommandLine, promlogConfig) version.Branch = Branch @@ -35,8 +115,20 @@ func main() { kingpin.HelpFlag.Short('h') kingpin.Parse() logger = promlog.New(promlogConfig) - _ = level.Info(logger).Log("msg", "Starting NUT exporter on ups "+config.UpsName, "version", version.Info()) + _ = level.Info(logger).Log("msg", "Starting prometheus data export ", "version", version.Info()) err := config.LoadFile(*configFile) + if *showConfig { + _ = level.Info(logger).Log("msg", "show only configuration ane exit") + fmt.Print(config.print()) + os.Exit(0) + } + if err != nil { + _ = level.Error(logger).Log("msg", "problem with configuration", "error", err) + fmt.Printf("Program did not start due to configuration error! \r\n\tError: %s", err) + os.Exit(1) + } + saveMetaData() + saveData() -} \ No newline at end of file +} diff --git a/metrics-meta.go b/metrics-meta.go new file mode 100644 index 0000000..293a8ca --- /dev/null +++ b/metrics-meta.go @@ -0,0 +1,88 @@ +package main + +import ( + "encoding/json" + "github.com/go-kit/kit/log/level" + "os" +) + +const MetricsMetaFileName = "metrics-meta.json" + +type MetricMeta struct { + Job string `json:"job,omitempty"` + Metric string `json:"metric,omitempty"` + Type string `json:"type,omitempty"` + Help string `json:"help,omitempty"` + Unit string `json:"unit,omitempty"` +} + +type MetricsMetaList []MetricMeta + +func NewMetrics(target TargetData) *MetricMeta { + return &MetricMeta{ + Job: target.Target["job"], + Metric: target.Metric, + Type: target.Type, + Help: target.Help, + Unit: target.Unit, + } +} + +func (m *MetricMeta) isSame(meta MetricMeta) bool { + return m.Job == meta.Job && m.Metric == meta.Metric +} + +func (m *MetricMeta) isSameTarget(target TargetData) bool { + return m.Job == target.Target["job"] && m.Metric == target.Metric +} + +func NewMetricsMetaList(t TargetList) *MetricsMetaList { + m := MetricsMetaList{} + for _, targetData := range t { + if !m.exitInList(targetData) { + m = append(m, *NewMetrics(targetData)) + } + } + return &m +} + +func (m *MetricsMetaList) exitInList(data TargetData) bool { + for i := 0; i < len(*m); i++ { + if (*m)[i].isSameTarget(data) { + return true + } + } + return false +} + +func (m *MetricsMetaList) onlyForJobs(jobNames []string) { + if jobNames == nil || len(jobNames) == 0 { + return + } + for i := len(*m) - 1; i >= 0; i-- { + if !containsString(jobNames, (*m)[i].Job) { + *m = append((*m)[:i], (*m)[i+1:]...) + } + } +} + +func (m *MetricsMetaList) saveList() { + name := config.filePath(MetricsMetaFileName) + f, err := os.Create(name) + if err != nil { + _ = level.Error(logger).Log("msg", "problem create meta data file ", "file", name, "error", err) + return + } + defer func() { _ = f.Close() }() + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + _ = level.Error(logger).Log("msg", "problem create prepare target meta data", "file", name, "error", err) + return + } + _, err = f.Write(data) + if err != nil { + _ = level.Error(logger).Log("msg", "problem write data to file ", "file", name, "error", err) + return + } + _ = level.Debug(logger).Log("msg", "target meta data success write to file ", "file", name) +} diff --git a/query_range.go b/query_range.go new file mode 100644 index 0000000..128fc88 --- /dev/null +++ b/query_range.go @@ -0,0 +1,141 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "os" + "regexp" + "strconv" + "strings" + "time" +) + +var pointRex = regexp.MustCompile(`\[(\d+\.?\d*)\,\"([^\"]+)\"\]`) + +// Point represents a single data point for a given timestamp. +type Point struct { + T model.Time + V float64 + //U string +} + +type Series struct { + Metric prometheus.Labels `json:"metric"` + Points []Point `json:"values"` +} + +type Matrix []Series + +func (p Point) String() string { + v := strconv.FormatFloat(p.V, 'f', -1, 64) + return fmt.Sprintf("%v @[%v]", v, p.T) +} + +type Result struct { + ResultType string `json:"resultType"` + Result Matrix `json:"result"` +} + +func (p *Point) UnmarshalJSON(data []byte) error { + s := string(data) + match := pointRex.FindStringSubmatch(s) + if len(match) < 3 { + return errors.New("problem parse point values") + } + err := json.Unmarshal([]byte(match[1]), &p.T) + if err != nil { + return err + } + //p.U = p.T.Time().Format(time.RFC3339) + p.V, err = strconv.ParseFloat(match[2], 64) + if err != nil { + return err + } + return nil +} + +func (s Series) String() string { + vals := make([]string, len(s.Points)) + for i, v := range s.Points { + vals[i] = v.String() + } + return fmt.Sprintf("%s =>\n%s", s.Metric, strings.Join(vals, "\n")) +} + +func (m Matrix) String() string { + // TODO(fabxc): sort, or can we rely on order from the querier? + strs := make([]string, len(m)) + + for i, ss := range m { + strs[i] = ss.String() + } + + return strings.Join(strs, "\n") +} + +func (m *Matrix) UnmarshalJSON(data []byte) error { + var s []Series + s = []Series{} + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + *m = s + return nil +} + +func UnmarshalResult(data []byte) (*Result, error) { + var r Result + err := json.Unmarshal(data, &r) + if err != nil { + return nil, err + } + return &r, nil +} + +func getRangeDay(job string, dayBack int) (*Matrix, error) { + start := time.Now().UTC().Add(time.Duration(-24*dayBack) * time.Hour).Truncate(time.Hour * 24) + end := start.Add(24 * time.Hour) + uri := fmt.Sprintf("query_range?query=%s&start=%s&end=%s&step=%d", job, start.Format(time.RFC3339), end.Format(time.RFC3339), config.Step) + + read, err := GetApiData(uri) + if err != nil { + _ = level.Error(logger).Log("msg", "problem collect data for job") + return nil, err + } + if !read.statusSuccess() { + _ = level.Error(logger).Log("msg", "target meta data read return error", "error", read.Error, "errorType", read.ErrorType) + return nil, errors.New(read.Error) + } + s := string(*read.Data) + t, err := UnmarshalResult([]byte(s)) + if err != nil { + return nil, err + } + return &t.Result, nil +} + +func (m *Matrix) save(metricsName string) { + name := config.filePath(metricsName + ".json") + f, err := os.Create(name) + if err != nil { + _ = level.Error(logger).Log("msg", "problem create meta data file ", "file", name, "error", err) + return + } + defer func() { _ = f.Close() }() + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + _ = level.Error(logger).Log("msg", "problem create prepare target meta data", "file", name, "error", err) + return + } + _, err = f.Write(data) + if err != nil { + _ = level.Error(logger).Log("msg", "problem write data to file ", "file", name, "error", err) + return + } + _ = level.Debug(logger).Log("msg", "target meta data success write to file ", "file", name) +} diff --git a/targets.go b/targets.go new file mode 100644 index 0000000..6ae295b --- /dev/null +++ b/targets.go @@ -0,0 +1,56 @@ +package main + +import ( + "encoding/json" + "errors" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" + //"github.com/prometheus/common" +) + +type TargetData struct { + Target prometheus.Labels `json:"target,omitempty"` + Metric string `json:"metric,omitempty"` + Type string `json:"type,omitempty"` + Help string `json:"help,omitempty"` + Unit string `json:"unit,omitempty"` +} + +type TargetList []TargetData + +func UnmarshalTargets(data []byte) (*TargetList, error) { + var t TargetList + if err := json.Unmarshal(data, &t); err != nil { + return nil, err + } + return &t, nil +} + +func readTargetsList() (*TargetList, error) { + read, err := GetApiData("targets/metadata") + if err != nil { + _ = level.Error(logger).Log("msg", "problem collect targets details") + return nil, err + } + if !read.statusSuccess() { + _ = level.Error(logger).Log("msg", "target meta data read return error", "error", read.Error, "errorType", read.ErrorType) + return nil, errors.New(read.Error) + } + s := string(*read.Data) + t, err := UnmarshalTargets([]byte(s)) + return t, err +} + +func (t *TargetList) cleanAndFilterJobs(jobNames []string) *MetricsMetaList { + m := MetricsMetaList{} + if jobNames == nil || len(jobNames) == 0 { + return &m + } + for i := len(*t) - 1; i >= 0; i-- { + if !containsString(jobNames, (*t)[i].Target["job"]) { + *t = append((*t)[:i], (*t)[i+1:]...) + } + } + + return &m +}