diff --git a/.travis.yml b/.travis.yml index 88ae5e1eb..dafb99096 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,13 @@ before_install: - sudo add-apt-repository ppa:masterminds/glide -y - sudo apt-get update -q - sudo apt-get install glide -y + # Build go tools in separate $GOPATH. + # That allows to keep main $GOPATH clean and be sure that only glide.lock deps used. + - mkdir $HOME/tools + - GOPATH=$HOME/tools make tools + - export PATH=$HOME/tools/bin:$PATH install: - - make tools - - go get -t `glide novendor` + - glide install script: make travis diff --git a/cli/cli.go b/cli/cli.go index a74786165..dd3bcabf3 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - "log" "net/http" "os" "os/signal" @@ -154,7 +153,7 @@ func startMonitoring(conf monitoringConfig) (stop func()) { if conf.CPUProfile != "" { f, err := os.Create(conf.MemProfile) if err != nil { - log.Fatal(err) + zap.L().Fatal("Memory profile file create fail", zap.Error(err)) } pprof.StartCPUProfile(f) stops = append(stops, func() { @@ -163,9 +162,9 @@ func startMonitoring(conf monitoringConfig) (stop func()) { }) } if conf.MemProfile != "" { - f, err := os.Create(conf.MemProfile) + f, err := os.Create(conf.CPUProfile) if err != nil { - log.Fatal(err) + zap.L().Fatal("CPU profile file create fail", zap.Error(err)) } stops = append(stops, func() { pprof.WriteHeapProfile(f) diff --git a/components/phttp/ammo/simple/ammo.go b/components/phttp/ammo/simple/ammo.go index fee62d4de..5f2132d29 100644 --- a/components/phttp/ammo/simple/ammo.go +++ b/components/phttp/ammo/simple/ammo.go @@ -17,19 +17,25 @@ type Ammo struct { // Need to research is it possible. http.Transport can hold reference to http.Request. req *http.Request tag string -} - -func NewAmmo(req *http.Request, tag string) *Ammo { - return &Ammo{req, tag} + id int } func (a *Ammo) Request() (*http.Request, *netsample.Sample) { sample := netsample.Acquire(a.tag) + sample.SetId(a.id) return a.req, sample } func (a *Ammo) Reset(req *http.Request, tag string) { - *a = Ammo{req, tag} + *a = Ammo{req, tag, -1} +} + +func (a *Ammo) SetId(id int) { + a.id = id +} + +func (a *Ammo) Id() int { + return a.id } var _ phttp.Ammo = (*Ammo)(nil) diff --git a/components/phttp/ammo/simple/jsonline/provider.go b/components/phttp/ammo/simple/jsonline/provider.go index 506657980..045948f5f 100644 --- a/components/phttp/ammo/simple/jsonline/provider.go +++ b/components/phttp/ammo/simple/jsonline/provider.go @@ -10,7 +10,7 @@ import ( "context" "net/http" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" "github.com/spf13/afero" "go.uber.org/zap" @@ -49,7 +49,7 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { data := scanner.Bytes() a, err := decodeAmmo(data, p.Pool.Get().(*simple.Ammo)) if err != nil { - return stackerr.Newf("failed to decode ammo at line: %v; data: %q; error: %s", line, data, err) + return errors.Wrapf(err, "failed to decode ammo at line: %v; data: %q", line, data) } ammoNum++ select { @@ -71,7 +71,7 @@ func decodeAmmo(jsonDoc []byte, am *simple.Ammo) (*simple.Ammo, error) { var data data err := data.UnmarshalJSON(jsonDoc) if err != nil { - return nil, stackerr.Wrap(err) + return nil, errors.WithStack(err) } req, err := data.ToRequest() if err != nil { @@ -85,7 +85,7 @@ func (d *data) ToRequest() (req *http.Request, err error) { uri := "http://" + d.Host + d.Uri req, err = http.NewRequest(d.Method, uri, nil) if err != nil { - return nil, stackerr.Wrap(err) + return nil, errors.WithStack(err) } for k, v := range d.Headers { req.Header.Set(k, v) diff --git a/components/phttp/ammo/simple/provider.go b/components/phttp/ammo/simple/provider.go index 1619dc472..3fb31e30f 100644 --- a/components/phttp/ammo/simple/provider.go +++ b/components/phttp/ammo/simple/provider.go @@ -9,8 +9,10 @@ import ( "context" "sync" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" "github.com/spf13/afero" + "go.uber.org/atomic" + "github.com/yandex/pandora/core" ) @@ -25,16 +27,20 @@ func NewProvider(fs afero.Fs, fileName string, start func(ctx context.Context, f } type Provider struct { - fs afero.Fs - fileName string - start func(ctx context.Context, file afero.File) error - Sink chan *Ammo - Pool sync.Pool + fs afero.Fs + fileName string + start func(ctx context.Context, file afero.File) error + Sink chan *Ammo + Pool sync.Pool + idCounter atomic.Int64 } -func (p *Provider) Acquire() (ammo core.Ammo, ok bool) { - ammo, ok = <-p.Sink - return +func (p *Provider) Acquire() (core.Ammo, bool) { + ammo, ok := <-p.Sink + if ok { + ammo.SetId(int(p.idCounter.Inc() - 1)) + } + return ammo, ok } func (p *Provider) Release(a core.Ammo) { @@ -45,8 +51,7 @@ func (p *Provider) Run(ctx context.Context) error { defer close(p.Sink) file, err := p.fs.Open(p.fileName) if err != nil { - // TODO(skipor): instead of passing stacktrace log error here. - return stackerr.Newf("failed to open ammo Sink: %v", err) + return errors.Wrap(err, "failed to open ammo file") } defer file.Close() return p.start(ctx, file) diff --git a/components/phttp/ammo/simple/raw/decoder.go b/components/phttp/ammo/simple/raw/decoder.go index d59b0ed50..1e380b4c9 100644 --- a/components/phttp/ammo/simple/raw/decoder.go +++ b/components/phttp/ammo/simple/raw/decoder.go @@ -13,8 +13,6 @@ func decodeHeader(headerString []byte) (reqSize int, tag string, err error) { reqSize, err = strconv.Atoi(parts[0]) if len(parts) > 1 { tag = parts[1] - } else { - tag = "__EMPTY__" } return } diff --git a/components/phttp/ammo/simple/raw/decoder_test.go b/components/phttp/ammo/simple/raw/decoder_test.go index 95a5ce6d5..57b51b7fb 100644 --- a/components/phttp/ammo/simple/raw/decoder_test.go +++ b/components/phttp/ammo/simple/raw/decoder_test.go @@ -23,7 +23,7 @@ var _ = Describe("Decoder", func() { reqSize, tag, err := decodeHeader([]byte(raw)) Expect(err).To(BeNil()) Expect(reqSize).To(Equal(123)) - Expect(tag).To(Equal("__EMPTY__")) + Expect(tag).To(Equal("")) }) It("should parse GET request", func() { raw := "GET /some/path HTTP/1.0\r\n" + diff --git a/components/phttp/ammo/simple/raw/provider.go b/components/phttp/ammo/simple/raw/provider.go index 821a5673f..0f01a0797 100644 --- a/components/phttp/ammo/simple/raw/provider.go +++ b/components/phttp/ammo/simple/raw/provider.go @@ -4,13 +4,12 @@ import ( "bufio" "context" "io" - "log" - "os" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" "github.com/spf13/afero" "github.com/yandex/pandora/components/phttp/ammo/simple" + "go.uber.org/zap" ) /* @@ -44,7 +43,7 @@ User-Agent: xxx (shell 1) */ func filePosition(file afero.File) (position int64) { - position, _ = file.Seek(0, os.SEEK_CUR) + position, _ = file.Seek(0, io.SeekCurrent) return } @@ -80,13 +79,13 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { for p.Limit == 0 || ammoNum < p.Limit { data, isPrefix, err := reader.ReadLine() if isPrefix { - return stackerr.Newf("Too long header in ammo at position %v", filePosition(ammoFile)) + return errors.Errorf("too long header in ammo at position %v", filePosition(ammoFile)) } if err == io.EOF { break // start over from the beginning } if err != nil { - return stackerr.Newf("error reading ammo at position: %v; error: %s", filePosition(ammoFile), err) + return errors.Wrapf(err, "reading ammo failed at position: %v", filePosition(ammoFile)) } if len(data) == 0 { continue // skip empty lines @@ -97,11 +96,11 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { } buff := make([]byte, reqSize) if n, err := io.ReadFull(reader, buff); err != nil { - return stackerr.Newf("failed to read ammo at position: %v; tried to read: %v; have read: %v; error: %s", filePosition(ammoFile), reqSize, n, err) + return errors.Wrapf(err, "failed to read ammo at position: %v; tried to read: %v; have read: %v", filePosition(ammoFile), reqSize, n) } req, err := decodeRequest(buff) if err != nil { - return stackerr.Newf("failed to decode ammo at position: %v; data: %q; error: %s", filePosition(ammoFile), buff, err) + return errors.Wrapf(err, "failed to decode ammo at position: %v; data: %q", filePosition(ammoFile), buff) } sh := p.Pool.Get().(*simple.Ammo) sh.Reset(req, tag) @@ -114,13 +113,13 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { } } if ammoNum == 0 { - return stackerr.Newf("no ammo in file") + return errors.New("no ammo in file") } if p.Passes != 0 && passNum >= p.Passes { break } ammoFile.Seek(0, 0) } - log.Println("Ran out of ammo") + zap.L().Debug("Ran out of ammo") return nil } diff --git a/components/phttp/ammo/simple/raw/provider_test.go b/components/phttp/ammo/simple/raw/provider_test.go index 11e6ef1a1..30da4bfa3 100644 --- a/components/phttp/ammo/simple/raw/provider_test.go +++ b/components/phttp/ammo/simple/raw/provider_test.go @@ -21,8 +21,8 @@ const testFile = "./ammo.stpd" const testFileData = "../../../testdata/ammo.stpd" var testData = []ammoData{ - {"GET", "www.ya.ru", "/", http.Header{"Connection": []string{"close"}}, "__EMPTY__", ""}, - {"GET", "www.ya.ru", "/test", http.Header{"Connection": []string{"close"}}, "__EMPTY__", ""}, + {"GET", "www.ya.ru", "/", http.Header{"Connection": []string{"close"}}, "", ""}, + {"GET", "www.ya.ru", "/test", http.Header{"Connection": []string{"close"}}, "", ""}, {"GET", "www.ya.ru", "/test2", http.Header{"Connection": []string{"close"}}, "tag", ""}, {"POST", "www.ya.ru", "/test3", http.Header{"Connection": []string{"close"}, "Content-Length": []string{"5"}}, "tag", "hello"}, } diff --git a/components/phttp/ammo/simple/uri/decoder.go b/components/phttp/ammo/simple/uri/decoder.go index 23f7c949f..5588f77bb 100644 --- a/components/phttp/ammo/simple/uri/decoder.go +++ b/components/phttp/ammo/simple/uri/decoder.go @@ -12,7 +12,7 @@ import ( "strings" "sync" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" "github.com/yandex/pandora/components/phttp/ammo/simple" ) @@ -37,7 +37,7 @@ func newDecoder(ctx context.Context, sink chan<- *simple.Ammo, pool *sync.Pool) func (d *decoder) Decode(line []byte) error { if len(line) == 0 { - return stackerr.Newf("empty line") + return errors.New("empty line") } switch line[0] { case '/': @@ -45,7 +45,7 @@ func (d *decoder) Decode(line []byte) error { case '[': return d.decodeHeader(line) } - return stackerr.Newf("every line should begin with '[' or '/'") + return errors.New("every line should begin with '[' or '/'") } func (d *decoder) decodeURI(line []byte) error { @@ -55,12 +55,10 @@ func (d *decoder) decodeURI(line []byte) error { var tag string if len(parts) > 1 { tag = parts[1] - } else { - tag = "__EMPTY__" } req, err := http.NewRequest("GET", string(url), nil) if err != nil { - return stackerr.Newf("uri decode error: ", err) + return errors.Wrap(err, "uri decode") } for k, v := range d.header { // http.Request.Write sends Host header based on Host or URL.Host. @@ -84,17 +82,17 @@ func (d *decoder) decodeURI(line []byte) error { func (d *decoder) decodeHeader(line []byte) error { if len(line) < 3 || line[0] != '[' || line[len(line)-1] != ']' { - return stackerr.Newf("header line should be like '[key: value]") + return errors.New("header line should be like '[key: value]") } line = line[1 : len(line)-1] colonIdx := bytes.IndexByte(line, ':') if colonIdx < 0 { - return stackerr.Newf("missing colon") + return errors.New("missing colon") } key := string(bytes.TrimSpace(line[:colonIdx])) val := string(bytes.TrimSpace(line[colonIdx+1:])) if key == "" { - return stackerr.Newf("missing header key") + return errors.New("missing header key") } d.header.Set(key, val) return nil diff --git a/components/phttp/ammo/simple/uri/decoder_test.go b/components/phttp/ammo/simple/uri/decoder_test.go index 404edd219..6b82e5b81 100644 --- a/components/phttp/ammo/simple/uri/decoder_test.go +++ b/components/phttp/ammo/simple/uri/decoder_test.go @@ -79,7 +79,7 @@ var _ = Describe("Decoder", func() { header.Set("Host", host) Expect(decoder.header).To(Equal(header)) Expect(decoder.ammoNum).To(Equal(1)) - Expect(sample.Tags()).To(Equal("__EMPTY__")) + Expect(sample.Tags()).To(Equal("")) }) It("uri and tag", func() { header := http.Header{"a": []string{"b"}, "c": []string{"d"}} diff --git a/components/phttp/ammo/simple/uri/provider_test.go b/components/phttp/ammo/simple/uri/provider_test.go index cb39ed748..b5c29790b 100644 --- a/components/phttp/ammo/simple/uri/provider_test.go +++ b/components/phttp/ammo/simple/uri/provider_test.go @@ -30,16 +30,16 @@ const testFileData = `/0 ` var testData = []ammoData{ - {"", "/0", http.Header{}, "__EMPTY__"}, - {"", "/1", http.Header{"A": []string{"b"}}, "__EMPTY__"}, + {"", "/0", http.Header{}, ""}, + {"", "/1", http.Header{"A": []string{"b"}}, ""}, {"example.com", "/2", http.Header{ "A": []string{"b"}, "C": []string{"d"}, - }, "__EMPTY__"}, + }, ""}, {"other.net", "/3", http.Header{ "A": []string{""}, "C": []string{"d"}, - }, "__EMPTY__"}, + }, ""}, {"other.net", "/4", http.Header{ "A": []string{""}, "C": []string{"d"}, diff --git a/components/phttp/base.go b/components/phttp/base.go index 9f2a1950d..6fc926b74 100644 --- a/components/phttp/base.go +++ b/components/phttp/base.go @@ -10,78 +10,142 @@ import ( "io" "io/ioutil" "net/http" + "net/url" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" "go.uber.org/zap" "github.com/yandex/pandora/core/aggregate/netsample" ) -// TODO: inject logger -type Base struct { +const ( + EmptyTag = "__EMPTY__" +) + +type BaseGunConfig struct { + AutoTag AutoTagConfig `config:"auto-tag"` +} + +// AutoTagConfig configure automatic tags generation based on ammo URI. First AutoTag URI path elements becomes tag. +// Example: /my/very/deep/page?id=23¶m=33 -> /my/very when uri-elements: 2. +type AutoTagConfig struct { + Enabled bool `config:"enabled"` + URIElements int `config:"uri-elements" validate:"min=1"` // URI elements used to autotagging + NoTagOnly bool `config:"no-tag-only"` // When true, autotagged only ammo that has no tag before. +} + +func NewDefaultBaseGunConfig() BaseGunConfig { + return BaseGunConfig{ + AutoTagConfig{ + Enabled: false, + URIElements: 2, + NoTagOnly: true, + }} +} + +type BaseGun struct { + Log *zap.Logger // If nil, zap.L() will be used. + DebugLog bool // Automaticaly set in Bind if Log accepts debug messages. + Config BaseGunConfig Do func(r *http.Request) (*http.Response, error) // Required. Connect func(ctx context.Context) error // Optional hook. OnClose func() error // Optional. Called on Close(). Aggregator netsample.Aggregator // Lazy set via BindResultTo. } -var _ Gun = (*Base)(nil) -var _ io.Closer = (*Base)(nil) +var _ Gun = (*BaseGun)(nil) +var _ io.Closer = (*BaseGun)(nil) + +// TODO(skipor): pass logger here in https://github.com/yandex/pandora/issues/57 +func (b *BaseGun) Bind(aggregator netsample.Aggregator) { + if b.Log == nil { + b.Log = zap.L() + } + if ent := b.Log.Check(zap.DebugLevel, "Gun bind"); ent != nil { + // Enable debug level logging during shooting. Creating log entries isn't free. + b.DebugLog = true + } -func (b *Base) Bind(aggregator netsample.Aggregator) { if b.Aggregator != nil { - zap.L().Panic("already binded") + b.Log.Panic("already binded") } if aggregator == nil { - zap.L().Panic("nil aggregator") + b.Log.Panic("nil aggregator") } b.Aggregator = aggregator } // Shoot is thread safe iff Do and Connect hooks are thread safe. -func (b *Base) Shoot(ctx context.Context, ammo Ammo) { +func (b *BaseGun) Shoot(ctx context.Context, ammo Ammo) { if b.Aggregator == nil { zap.L().Panic("must bind before shoot") } if b.Connect != nil { err := b.Connect(ctx) if err != nil { - zap.L().Warn("Connect fail", zap.Error(err)) + b.Log.Warn("Connect fail", zap.Error(err)) return } } req, sample := ammo.Request() + if b.DebugLog { + b.Log.Debug("Shoot", zap.Stringer("url", req.URL)) + } + + if b.Config.AutoTag.Enabled && (!b.Config.AutoTag.NoTagOnly || sample.Tags() == "") { + sample.AddTag(autotag(b.Config.AutoTag.URIElements, req.URL)) + } + if sample.Tags() == "" { + sample.AddTag(EmptyTag) + } + var err error defer func() { if err != nil { sample.SetErr(err) } b.Aggregator.Report(sample) - err = stackerr.WrapSkip(err, 1) + err = errors.WithStack(err) }() + var res *http.Response res, err = b.Do(req) if err != nil { - zap.L().Warn("Request fail", zap.Error(err)) + b.Log.Warn("Request fail", zap.Error(err)) return } + if b.DebugLog { + b.Log.Debug("Got response", zap.Int("status", res.StatusCode)) + } sample.SetProtoCode(res.StatusCode) defer res.Body.Close() // TODO: measure body read time - // TODO: buffer copy buffers. - _, err = io.Copy(ioutil.Discard, res.Body) + _, err = io.Copy(ioutil.Discard, res.Body) // Buffers are pooled for ioutil.Discard if err != nil { - zap.L().Warn("Body read fail", zap.Error(err)) + b.Log.Warn("Body read fail", zap.Error(err)) return } // TODO: verbose logging - return } -func (b *Base) Close() error { +func (b *BaseGun) Close() error { if b.OnClose != nil { return b.OnClose() } return nil } + +func autotag(depth int, URL *url.URL) string { + path := URL.Path + var ind int + for ; ind < len(path); ind++ { + if path[ind] == '/' { + if depth == 0 { + break + } + depth-- + } + } + return path[:ind] +} diff --git a/components/phttp/base_test.go b/components/phttp/base_test.go index ccb05dc58..aed99c95f 100644 --- a/components/phttp/base_test.go +++ b/components/phttp/base_test.go @@ -11,23 +11,27 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "net/url" "strings" . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" "github.com/yandex/pandora/components/phttp/mocks" "github.com/yandex/pandora/core/aggregate/netsample" + "github.com/yandex/pandora/core/coretest" ) -var _ = Describe("Base", func() { +var _ = Describe("BaseGun", func() { + var ( - base Base + base BaseGun ammo *ammomock.Ammo ) BeforeEach(func() { - base = Base{} + base = BaseGun{Config: NewDefaultBaseGunConfig()} ammo = &ammomock.Ammo{} }) @@ -69,6 +73,7 @@ var _ = Describe("Base", func() { ctx context.Context am *ammomock.Ammo req *http.Request + tag string res *http.Response sample *netsample.Sample results *netsample.TestAggregator @@ -77,14 +82,15 @@ var _ = Describe("Base", func() { BeforeEach(func() { ctx = context.Background() am = &ammomock.Ammo{} - req = httptest.NewRequest("GET", "/", nil) - sample = netsample.Acquire("REQUEST") - am.On("Request").Return(req, sample) + req = httptest.NewRequest("GET", "/1/2/3/4", nil) + tag = "" results = &netsample.TestAggregator{} base.Bind(results) }) JustBeforeEach(func() { + sample = netsample.Acquire(tag) + am.On("Request").Return(req, sample) res = &http.Response{ StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(body), @@ -107,6 +113,7 @@ var _ = Describe("Base", func() { It("ammo sample sent to results", func() { Expect(results.Samples).To(HaveLen(1)) Expect(results.Samples[0]).To(Equal(sample)) + Expect(sample.Tags()).To(Equal("__EMPTY__")) Expect(sample.ProtoCode()).To(Equal(res.StatusCode)) }) It("body read well", func() { @@ -114,6 +121,29 @@ var _ = Describe("Base", func() { _, err := body.Read([]byte{0}) Expect(err).To(Equal(io.EOF), "body should be read fully") }) + + Context("autotag options is set", func() { + BeforeEach(func() { base.Config.AutoTag.Enabled = true }) + It("autotagged", func() { + Expect(sample.Tags()).To(Equal("/1/2")) + }) + + Context("tag is already set", func() { + const presetTag = "TAG" + BeforeEach(func() { tag = presetTag }) + It("no tag added", func() { + Expect(sample.Tags()).To(Equal(presetTag)) + }) + + Context("no-tag-only set to false", func() { + BeforeEach(func() { base.Config.AutoTag.NoTagOnly = false }) + It("autotag added", func() { + Expect(sample.Tags()).To(Equal(presetTag + "|/1/2")) + }) + }) + }) + }) + Context("Connect set", func() { var connectCalled, doCalled bool BeforeEach(func() { @@ -151,4 +181,26 @@ var _ = Describe("Base", func() { }) }) }) + + DescribeTable("autotag", + func(path string, depth int, tag string) { + URL := &url.URL{Path: path} + Expect(autotag(depth, URL)).To(Equal(tag)) + }, + Entry("empty", "", 2, ""), + Entry("root", "/", 2, "/"), + Entry("exact depth", "/1/2", 2, "/1/2"), + Entry("more depth", "/1/2", 3, "/1/2"), + Entry("less depth", "/1/2", 1, "/1"), + ) + + It("config decode", func() { + var conf BaseGunConfig + coretest.DecodeAndValidate(` +auto-tag: + enabled: true + uri-elements: 3 + no-tag-only: false +`, &conf) + }) }) diff --git a/components/phttp/client.go b/components/phttp/client.go index ebf0be490..79c8c86f1 100644 --- a/components/phttp/client.go +++ b/components/phttp/client.go @@ -27,10 +27,9 @@ type Client interface { } type ClientConfig struct { - // May not be squashed until fix https://github.com/mitchellh/mapstructure/issues/70 - Transport TransportConfig - Dialer DialerConfig `config:"dial"` - Redirect bool // When true, follow HTTP redirects. + Redirect bool // When true, follow HTTP redirects. + Dialer DialerConfig `config:"dial"` + Transport TransportConfig `config:",squash"` } func NewDefaultClientConfig() ClientConfig { diff --git a/components/phttp/connect.go b/components/phttp/connect.go index df318f88e..4ea48b289 100644 --- a/components/phttp/connect.go +++ b/components/phttp/connect.go @@ -14,14 +14,15 @@ import ( "net/http/httputil" "net/url" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" ) type ConnectGunConfig struct { - Target string `validate:"endpoint,required"` - ConnectSSL bool `config:"connect-ssl"` // Defines if tunnel encrypted. - SSL bool // As in HTTP gun, defines scheme for http requests. - Client ClientConfig `config:",squash"` + Target string `validate:"endpoint,required"` + ConnectSSL bool `config:"connect-ssl"` // Defines if tunnel encrypted. + SSL bool // As in HTTP gun, defines scheme for http requests. + Client ClientConfig `config:",squash"` + BaseGunConfig `config:",squash"` } func NewConnectGun(conf ConnectGunConfig) *ConnectGun { @@ -32,8 +33,9 @@ func NewConnectGun(conf ConnectGunConfig) *ConnectGun { client := newConnectClient(conf) var g ConnectGun g = ConnectGun{ - Base: Base{ - Do: g.Do, + BaseGun: BaseGun{ + Config: conf.BaseGunConfig, + Do: g.Do, OnClose: func() error { client.CloseIdleConnections() return nil @@ -46,7 +48,7 @@ func NewConnectGun(conf ConnectGunConfig) *ConnectGun { } type ConnectGun struct { - Base + BaseGun scheme string client Client } @@ -91,7 +93,7 @@ func newConnectDialFunc(target string, connectSSL bool, dialer Dialer) DialerFun }() conn, err = dialer.DialContext(ctx, "tcp", target) if err != nil { - err = stackerr.Wrap(err) + err = errors.WithStack(err) return } if connectSSL { @@ -110,7 +112,7 @@ func newConnectDialFunc(target string, connectSSL bool, dialer Dialer) DialerFun // NOTE(skipor): any logic for CONNECT request can be easily added via hooks. err = req.Write(conn) if err != nil { - err = stackerr.Wrap(err) + err = errors.WithStack(err) return } // NOTE(skipor): according to RFC 2817 we can send origin at that moment and not wait @@ -118,7 +120,7 @@ func newConnectDialFunc(target string, connectSSL bool, dialer Dialer) DialerFun r := bufio.NewReader(conn) res, err := http.ReadResponse(r, req) if err != nil { - err = stackerr.Wrap(err) + err = errors.WithStack(err) return } // RFC 7230 3.3.3.2: Any 2xx (Successful) response to a CONNECT request implies that @@ -128,7 +130,7 @@ func newConnectDialFunc(target string, connectSSL bool, dialer Dialer) DialerFun // such a message. if res.StatusCode != http.StatusOK { dump, dumpErr := httputil.DumpResponse(res, false) - err = stackerr.Newf("Unexpected status code. Dumped response:\n%s\n Dump error: %s", + err = errors.Errorf("Unexpected status code. Dumped response:\n%s\n Dump error: %s", dump, dumpErr) return } @@ -137,7 +139,7 @@ func newConnectDialFunc(target string, connectSSL bool, dialer Dialer) DialerFun // Already receive something non HTTP from proxy or dialed server. // Anyway it is incorrect situation. peek, _ := r.Peek(r.Buffered()) - err = stackerr.Newf("Unexpected extra data after connect: %q", peek) + err = errors.Errorf("Unexpected extra data after connect: %q", peek) return } return diff --git a/components/phttp/core.go b/components/phttp/core.go index b64b91007..ee9e07aff 100644 --- a/components/phttp/core.go +++ b/components/phttp/core.go @@ -22,6 +22,8 @@ import ( type Ammo interface { // TODO(skipor): instead of sample use it wrapper with httptrace and more usable interface. Request() (*http.Request, *netsample.Sample) + // Id unique ammo id. Usually equals to ammo num got from provider. + Id() int } type Gun interface { diff --git a/components/phttp/http.go b/components/phttp/http.go index c50408da1..1b33c313a 100644 --- a/components/phttp/http.go +++ b/components/phttp/http.go @@ -7,11 +7,14 @@ package phttp import ( "net/http" + + "github.com/pkg/errors" ) type ClientGunConfig struct { Target string `validate:"endpoint,required"` SSL bool + Base BaseGunConfig `config:",squash"` } type HTTPGunConfig struct { @@ -20,8 +23,8 @@ type HTTPGunConfig struct { } type HTTP2GunConfig struct { - Target string `validate:"endpoint,required"` - Client ClientConfig `config:",squash"` + Gun ClientGunConfig `config:",squash"` + Client ClientConfig `config:",squash"` } func NewHTTPGun(conf HTTPGunConfig) *HTTPGun { @@ -31,14 +34,16 @@ func NewHTTPGun(conf HTTPGunConfig) *HTTPGun { } // NewHTTP2Gun return simple HTTP/2 gun that can shoot sequentially through one connection. -func NewHTTP2Gun(conf HTTP2GunConfig) *HTTPGun { +func NewHTTP2Gun(conf HTTP2GunConfig) (*HTTPGun, error) { + if !conf.Gun.SSL { + // Open issue on github if you need this feature. + return nil, errors.New("HTTP/2.0 over TCP is not supported. Please leave SSL option true by default.") + } transport := NewHTTP2Transport(conf.Client.Transport, NewDialer(conf.Client.Dialer).DialContext) client := newClient(transport, conf.Client.Redirect) // Will panic and cancel shooting whet target doesn't support HTTP/2. client = &panicOnHTTP1Client{client} - // NOTE: HTTP/2 gun not support HTTP/2 over TCP for now. - // Open issue on github if you need this feature. - return NewClientGun(client, ClientGunConfig{Target: conf.Target, SSL: true}) + return NewClientGun(client, conf.Gun), nil } func NewClientGun(client Client, conf ClientGunConfig) *HTTPGun { @@ -48,8 +53,9 @@ func NewClientGun(client Client, conf ClientGunConfig) *HTTPGun { } var g HTTPGun g = HTTPGun{ - Base: Base{ - Do: g.Do, + BaseGun: BaseGun{ + Config: conf.Base, + Do: g.Do, OnClose: func() error { client.CloseIdleConnections() return nil @@ -63,7 +69,7 @@ func NewClientGun(client Client, conf ClientGunConfig) *HTTPGun { } type HTTPGun struct { - Base + BaseGun scheme string target string client Client @@ -86,13 +92,17 @@ func NewDefaultHTTPGunConfig() HTTPGunConfig { } func NewDefaultHTTP2GunConfig() HTTP2GunConfig { - return HTTP2GunConfig{ + conf := HTTP2GunConfig{ Client: NewDefaultClientConfig(), + Gun: NewDefaultClientGunConfig(), } + conf.Gun.SSL = true + return conf } func NewDefaultClientGunConfig() ClientGunConfig { return ClientGunConfig{ - SSL: false, + SSL: false, + Base: NewDefaultBaseGunConfig(), } } diff --git a/components/phttp/http_test.go b/components/phttp/http_test.go index 57ca4e06b..1ed23fc2d 100644 --- a/components/phttp/http_test.go +++ b/components/phttp/http_test.go @@ -24,7 +24,7 @@ import ( "github.com/yandex/pandora/core/config" ) -var _ = Describe("Base", func() { +var _ = Describe("BaseGun", func() { It("GunClientConfig decode", func() { conf := NewDefaultHTTPGunConfig() data := map[interface{}]interface{}{ @@ -182,8 +182,8 @@ var _ = Describe("HTTP/2", func() { })) defer server.Close() conf := NewDefaultHTTP2GunConfig() - conf.Target = server.Listener.Addr().String() - gun := NewHTTP2Gun(conf) + conf.Gun.Target = server.Listener.Addr().String() + gun, _ := NewHTTP2Gun(conf) var results netsample.TestAggregator gun.Bind(&results) gun.Shoot(context.Background(), newAmmoURL("/")) @@ -196,8 +196,8 @@ var _ = Describe("HTTP/2", func() { })) defer server.Close() conf := NewDefaultHTTP2GunConfig() - conf.Target = server.Listener.Addr().String() - gun := NewHTTP2Gun(conf) + conf.Gun.Target = server.Listener.Addr().String() + gun, _ := NewHTTP2Gun(conf) var results netsample.TestAggregator gun.Bind(&results) var r interface{} @@ -211,6 +211,18 @@ var _ = Describe("HTTP/2", func() { Expect(r).To(ContainSubstring(notHTTP2PanicMsg)) }) + It("no SSL construction fails", func() { + server := httptest.NewTLSServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + zap.S().Info("Served") + })) + defer server.Close() + conf := NewDefaultHTTP2GunConfig() + conf.Gun.Target = server.Listener.Addr().String() + conf.Gun.SSL = false + _, err := NewHTTP2Gun(conf) + Expect(err).To(HaveOccurred()) + }) + }) func isHTTP2Request(req *http.Request) bool { diff --git a/components/phttp/import/import.go b/components/phttp/import/import.go index 70efce4c8..5ea87f375 100644 --- a/components/phttp/import/import.go +++ b/components/phttp/import/import.go @@ -33,8 +33,9 @@ func Import(fs afero.Fs) { return WrapGun(NewHTTPGun(conf)) }, NewDefaultHTTPGunConfig) - register.Gun("http2", func(conf HTTP2GunConfig) core.Gun { - return WrapGun(NewHTTP2Gun(conf)) + register.Gun("http2", func(conf HTTP2GunConfig) (core.Gun, error) { + gun, err := NewHTTP2Gun(conf) + return WrapGun(gun), err }, NewDefaultHTTP2GunConfig) register.Gun("connect", func(conf ConnectGunConfig) core.Gun { diff --git a/components/phttp/mocks/ammo.go b/components/phttp/mocks/ammo.go index 0e00ca261..230c6f87c 100644 --- a/components/phttp/mocks/ammo.go +++ b/components/phttp/mocks/ammo.go @@ -1,18 +1,29 @@ +// Code generated by mockery v1.0.0 package ammomock -import ( - "net/http" - - "github.com/stretchr/testify/mock" - - "github.com/yandex/pandora/core/aggregate/netsample" -) +import "net/http" +import "github.com/stretchr/testify/mock" +import "github.com/yandex/pandora/core/aggregate/netsample" // Ammo is an autogenerated mock type for the Ammo type type Ammo struct { mock.Mock } +// Id provides a mock function with given fields: +func (_m *Ammo) Id() int { + ret := _m.Called() + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + // Request provides a mock function with given fields: func (_m *Ammo) Request() (*http.Request, *netsample.Sample) { ret := _m.Called() diff --git a/core/aggregate/netsample/netsample_suite_test.go b/core/aggregate/netsample/netsample_suite_test.go new file mode 100644 index 000000000..bf0ed0e4c --- /dev/null +++ b/core/aggregate/netsample/netsample_suite_test.go @@ -0,0 +1,16 @@ +package netsample + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/yandex/pandora/lib/testutil" +) + +func TestNetsample(t *testing.T) { + RegisterFailHandler(Fail) + testutil.ReplaceGlobalLogger() + RunSpecs(t, "Netsample Suite") +} diff --git a/core/aggregate/netsample/phout.go b/core/aggregate/netsample/phout.go index f5630418e..ca60762ab 100644 --- a/core/aggregate/netsample/phout.go +++ b/core/aggregate/netsample/phout.go @@ -6,29 +6,45 @@ import ( "io" "os" "strconv" - "sync" "time" + "github.com/pkg/errors" "github.com/spf13/afero" ) -func GetPhout(fs afero.Fs, conf PhoutConfig) (Aggregator, error) { - return defaultPhoutAggregator.get(fs, conf) +type PhoutConfig struct { + Destination string // Destination file name + Id bool // Print ammo ids if true. } -type PhoutConfig struct { - Destination string +func NewPhout(fs afero.Fs, conf PhoutConfig) (a Aggregator, err error) { + filename := conf.Destination + var file afero.File = os.Stdout + if filename != "" { + file, err = fs.Create(conf.Destination) + } + if err != nil { + err = errors.Wrap(err, "phout output file open failed") + return + } + a = &phoutAggregator{ + config: conf, + sink: make(chan *Sample, 32*1024), + writer: bufio.NewWriterSize(file, 512*1024), + buf: make([]byte, 0, 1024), + file: file, + } + return } type phoutAggregator struct { + config PhoutConfig sink chan *Sample writer *bufio.Writer buf []byte file io.Closer } -var _ Aggregator = (*phoutAggregator)(nil) - func (a *phoutAggregator) Report(s *Sample) { a.sink <- s } func (a *phoutAggregator) Run(ctx context.Context) error { @@ -70,7 +86,7 @@ loop: } func (a *phoutAggregator) handle(s *Sample) error { - a.buf = appendPhout(s, a.buf) + a.buf = appendPhout(s, a.buf, a.config.Id) a.buf = append(a.buf, '\n') _, err := a.writer.Write(a.buf) a.buf = a.buf[:0] @@ -78,57 +94,16 @@ func (a *phoutAggregator) handle(s *Sample) error { return err } -var defaultPhoutAggregator = newPhoutResultListeners() - -type phoutAggregators struct { - sync.Mutex - aggregators map[string]Aggregator -} - -func newPhout(fs afero.Fs, conf PhoutConfig) (a *phoutAggregator, err error) { - filename := conf.Destination - var file afero.File = os.Stdout - if filename != "" { - file, err = fs.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE|os.O_SYNC, 0666) - } - if err != nil { - return - } - a = &phoutAggregator{ - sink: make(chan *Sample, 32*1024), - writer: bufio.NewWriterSize(file, 512*1024), - buf: make([]byte, 0, 1024), - file: file, - } - return -} - -func newPhoutResultListeners() *phoutAggregators { - return &phoutAggregators{aggregators: make(map[string]Aggregator)} -} - -func (l *phoutAggregators) get(fs afero.Fs, conf PhoutConfig) (Aggregator, error) { - dest := conf.Destination - l.Lock() - defer l.Unlock() - rl, ok := l.aggregators[dest] - if !ok { - rl, err := newPhout(fs, conf) - if err != nil { - return nil, err - } - l.aggregators[dest] = rl - return rl, nil - } - return rl, nil -} - const phoutDelimiter = '\t' -func appendPhout(s *Sample, dst []byte) []byte { +func appendPhout(s *Sample, dst []byte, id bool) []byte { dst = appendTimestamp(s.timeStamp, dst) dst = append(dst, phoutDelimiter) dst = append(dst, s.tags...) + if id { + dst = append(dst, '#') + dst = strconv.AppendInt(dst, int64(s.Id()), 10) + } for _, v := range s.fields { dst = append(dst, phoutDelimiter) dst = strconv.AppendInt(dst, int64(v), 10) diff --git a/core/aggregate/netsample/phout_test.go b/core/aggregate/netsample/phout_test.go new file mode 100644 index 000000000..89fb6bbcf --- /dev/null +++ b/core/aggregate/netsample/phout_test.go @@ -0,0 +1,85 @@ +package netsample + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/spf13/afero" +) + +var _ = Describe("Phout", func() { + const fileName = "out.txt" + var ( + fs afero.Fs + conf PhoutConfig + testee Aggregator + ctx context.Context + cancel context.CancelFunc + runErr chan error + ) + getOutput := func() string { + data, err := afero.ReadFile(fs, fileName) + Expect(err).NotTo(HaveOccurred()) + return string(data) + } + + BeforeEach(func() { + fs = afero.NewMemMapFs() + conf = PhoutConfig{Destination: fileName} + ctx, cancel = context.WithCancel(context.Background()) + }) + JustBeforeEach(func() { + var err error + testee, err = NewPhout(fs, conf) + Expect(err).NotTo(HaveOccurred()) + runErr = make(chan error) + go func() { + runErr <- testee.Run(ctx) + }() + }) + It("no id by default", func() { + testee.Report(newTestSample()) + testee.Report(newTestSample()) + cancel() + Expect(<-runErr).NotTo(HaveOccurred()) + Expect(getOutput()).To(Equal(strings.Repeat(testSampleNoIdPhout+"\n", 2))) + }, 1) + Context("id option set", func() { + BeforeEach(func() { + conf.Id = true + }) + It("id printed", func() { + testee.Report(newTestSample()) + cancel() + Expect(<-runErr).NotTo(HaveOccurred()) + Expect(getOutput()).To(Equal(testSamplePhout + "\n")) + }, 1) + + }) + +}) + +const ( + testSamplePhout = "1484660999.002 tag1|tag2#42 333333 0 0 0 0 0 0 0 13 999" + testSampleNoIdPhout = "1484660999.002 tag1|tag2 333333 0 0 0 0 0 0 0 13 999" +) + +func TestOUt(t *testing.T) { + fmt.Println(newTestSample().String()) +} + +func newTestSample() *Sample { + s := &Sample{} + s.timeStamp = time.Unix(1484660999, 002*1000000) + s.SetId(42) + s.AddTag("tag1|tag2") + s.setDuration(keyRTTMicro, time.Second/3) + s.set(keyErrno, 13) + s.set(keyProtoCode, ProtoCodeError) + return s +} diff --git a/core/aggregate/netsample/sample.go b/core/aggregate/netsample/sample.go index c86e25a4b..e74ed75f6 100644 --- a/core/aggregate/netsample/sample.go +++ b/core/aggregate/netsample/sample.go @@ -11,6 +11,12 @@ import ( "sync" "syscall" "time" + + "github.com/pkg/errors" +) + +const ( + ProtoCodeError = 999 ) const ( @@ -43,6 +49,7 @@ var samplePool = &sync.Pool{New: func() interface{} { return &Sample{} }} type Sample struct { timeStamp time.Time tags string + id int fields [fieldsNum]int err error } @@ -56,6 +63,9 @@ func (s *Sample) AddTag(tag string) { s.tags += "|" + tag } +func (s *Sample) Id() int { return s.id } +func (s *Sample) SetId(id int) { s.id = id } + func (s *Sample) ProtoCode() int { return s.get(keyProtoCode) } func (s *Sample) SetProtoCode(code int) { s.set(keyProtoCode, code) @@ -79,7 +89,7 @@ func (s *Sample) setRTT() { } func (s *Sample) String() string { - return string(appendPhout(s, nil)) + return string(appendPhout(s, nil, true)) } func getErrno(err error) int { @@ -94,6 +104,7 @@ func getErrno(err error) int { } err = typed.Underlying() } + err = errors.Cause(err) for { switch typed := err.(type) { case *net.OpError: @@ -104,7 +115,7 @@ func getErrno(err error) int { return int(typed) default: // Legacy default. - return 999 + return ProtoCodeError } } } diff --git a/core/aggregate/netsample/sample_test.go b/core/aggregate/netsample/sample_test.go index 45a546d6f..64948fba4 100644 --- a/core/aggregate/netsample/sample_test.go +++ b/core/aggregate/netsample/sample_test.go @@ -17,14 +17,17 @@ import ( "time" "github.com/facebookgo/stackerr" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) func TestSampleBehaviour(t *testing.T) { const tag = "test" const tag2 = "test2" + const id = 42 sample := Acquire(tag) sample.AddTag(tag2) + sample.SetId(id) const sleep = time.Millisecond time.Sleep(sleep) sample.SetErr(syscall.EINVAL) @@ -39,9 +42,9 @@ func TestSampleBehaviour(t *testing.T) { // 1484660999. 2 -> 1484660999.002 expectedTimeStamp = strings.Replace(expectedTimeStamp, " ", "0", -1) - expected := fmt.Sprintf("%s\t%s|%s\t%v\t0\t0\t0\t0\t0\t0\t0\t%v\t%v", + expected := fmt.Sprintf("%s\t%s|%s#%v\t%v\t0\t0\t0\t0\t0\t0\t0\t%v\t%v", expectedTimeStamp, - tag, tag2, + tag, tag2, id, sample.get(keyRTTMicro), int(syscall.EINVAL), http.StatusBadRequest, ) @@ -52,6 +55,7 @@ func TestGetErrno(t *testing.T) { var err error = syscall.EINVAL err = &os.SyscallError{Err: err} err = &net.OpError{Err: err} + err = errors.WithStack(err) err = stackerr.Wrap(err) assert.NotZero(t, getErrno(err)) assert.Equal(t, int(syscall.EINVAL), getErrno(err)) diff --git a/core/config/config.go b/core/config/config.go index 35ec0dba0..2cd507f89 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -8,9 +8,9 @@ package config import ( "sync" - "github.com/facebookgo/stackerr" "github.com/fatih/structs" "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" ) const TagName = "config" @@ -19,9 +19,9 @@ const TagName = "config" func Decode(conf interface{}, result interface{}) error { decoder, err := mapstructure.NewDecoder(newDecoderConfig(result)) if err != nil { - return stackerr.Wrap(err) + return errors.WithStack(err) } - return stackerr.Wrap(decoder.Decode(conf)) + return errors.WithStack(decoder.Decode(conf)) } func DecodeAndValidate(conf interface{}, result interface{}) error { diff --git a/core/config/config_test.go b/core/config/config_test.go index f7692cd2f..3ec92160b 100644 --- a/core/config/config_test.go +++ b/core/config/config_test.go @@ -213,8 +213,6 @@ func TestDeltaUpdate(t *testing.T) { } func TestNextSquash(t *testing.T) { - // TODO(skipor): fix mapstructure #70 - t.Skip("Skipped until fix https://github.com/mitchellh/mapstructure/issues/70") data := &struct { Level1 struct { Level2 struct { diff --git a/core/config/hooks.go b/core/config/hooks.go index 1c3255786..c4414dea6 100644 --- a/core/config/hooks.go +++ b/core/config/hooks.go @@ -6,7 +6,7 @@ package config import ( - "errors" + stderrors "errors" "fmt" "net" "net/url" @@ -15,10 +15,10 @@ import ( "github.com/asaskevich/govalidator" "github.com/c2h5oh/datasize" "github.com/facebookgo/stack" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" + "go.uber.org/zap" "github.com/yandex/pandora/lib/tag" - "go.uber.org/zap" ) var InvalidURLError = errors.New("string is not valid URL") @@ -39,11 +39,11 @@ func StringToURLHook(f reflect.Type, t reflect.Type, data interface{}) (interfac str := data.(string) if !govalidator.IsURL(str) { // checks more than url.Parse - return nil, stackerr.Wrap(InvalidURLError) + return nil, errors.WithStack(InvalidURLError) } urlPtr, err := url.Parse(str) if err != nil { - return nil, stackerr.Wrap(err) + return nil, errors.WithStack(err) } if t == urlType { @@ -52,7 +52,7 @@ func StringToURLHook(f reflect.Type, t reflect.Type, data interface{}) (interfac return urlPtr, nil } -var InvalidIPError = errors.New("string is not valid IP") +var InvalidIPError = stderrors.New("string is not valid IP") // StringToIPHook converts string to net.IP func StringToIPHook(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) { @@ -65,7 +65,7 @@ func StringToIPHook(f reflect.Type, t reflect.Type, data interface{}) (interface str := data.(string) ip := net.ParseIP(str) if ip == nil { - return nil, stackerr.Wrap(InvalidIPError) + return nil, errors.WithStack(InvalidIPError) } return ip, nil } @@ -90,6 +90,7 @@ func DebugHook(f reflect.Type, t reflect.Type, data interface{}) (p interface{}, return } callers := stack.Callers(2) + var decodeCallers int for _, caller := range callers { if caller.Name == "(*Decoder).decode" { diff --git a/core/config/validator.go b/core/config/validator.go index 9d6d8bfe6..fe23a3b6e 100644 --- a/core/config/validator.go +++ b/core/config/validator.go @@ -8,7 +8,8 @@ package config import ( "reflect" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" + "gopkg.in/go-playground/validator.v8" ) @@ -37,7 +38,7 @@ type validate struct { } func (v *validate) Validate(value interface{}) error { - return stackerr.Wrap(v.V.Struct(value)) + return errors.WithStack(v.V.Struct(value)) } func newValidator() *validate { diff --git a/core/config/validator_test.go b/core/config/validator_test.go index 2687c602f..39b1e0612 100644 --- a/core/config/validator_test.go +++ b/core/config/validator_test.go @@ -8,7 +8,6 @@ package config import ( "testing" - "github.com/facebookgo/stackerr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -47,7 +46,6 @@ func TestValidateOK(t *testing.T) { func TestValidateError(t *testing.T) { err := Validate(&Multi{0, 2}) require.Error(t, err) - assert.IsType(t, &stackerr.Error{}, err) err = Validate(&Multi{0, 0}) require.Error(t, err) diff --git a/core/coretest/config.go b/core/coretest/config.go new file mode 100644 index 000000000..c17bd1eea --- /dev/null +++ b/core/coretest/config.go @@ -0,0 +1,24 @@ +// Copyright (c) 2017 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package coretest + +import ( + "github.com/onsi/gomega" + "github.com/yandex/pandora/core/config" + "github.com/yandex/pandora/lib/testutil" +) + +func Decode(data string, result interface{}) { + conf := testutil.ParseYAML(data) + err := config.Decode(conf, result) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func DecodeAndValidate(data string, result interface{}) { + Decode(data, result) + err := config.Validate(result) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} diff --git a/core/engine/instance.go b/core/engine/instance.go index f647e858b..55e2d30d8 100644 --- a/core/engine/instance.go +++ b/core/engine/instance.go @@ -10,7 +10,7 @@ import ( "fmt" "io" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" "go.uber.org/zap" "github.com/yandex/pandora/core" @@ -78,7 +78,7 @@ func (i *instance) shoot(ctx context.Context, gun core.Gun, next *coreutil.Waite defer func() { r := recover() if r != nil { - err = stackerr.Newf("shoot panic: %s", r) + err = errors.Errorf("shoot panic: %s", r) } }() for { diff --git a/core/import/import.go b/core/import/import.go index 3b6f5556a..ed04ff7f5 100644 --- a/core/import/import.go +++ b/core/import/import.go @@ -23,7 +23,7 @@ import ( func Import(fs afero.Fs) { register.Aggregator("phout", func(conf netsample.PhoutConfig) (core.Aggregator, error) { - a, err := netsample.GetPhout(fs, conf) + a, err := netsample.NewPhout(fs, conf) return netsample.WrapAggregator(a), err }) diff --git a/core/plugin/plugin.go b/core/plugin/plugin.go index 77e4fef9f..2882c7e0a 100644 --- a/core/plugin/plugin.go +++ b/core/plugin/plugin.go @@ -35,7 +35,7 @@ import ( "fmt" "reflect" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" ) // Register registers plugin factory and optional default config factory, @@ -302,12 +302,12 @@ func newNameRegistryEntry(pluginType reflect.Type, newPluginImpl interface{}, ne func (r typeRegistry) get(pluginType reflect.Type, name string) (factory nameRegistryEntry, err error) { pluginReg, ok := r[pluginType] if !ok { - err = stackerr.Newf("no plugins for type %s has been registered", pluginType) + err = errors.Errorf("no plugins for type %s has been registered", pluginType) return } factory, ok = pluginReg[name] if !ok { - err = stackerr.Newf("no plugins of type %s has been registered for name %s", pluginType, name) + err = errors.Errorf("no plugins of type %s has been registered for name %s", pluginType, name) } return } diff --git a/core/plugin/plugin_test.go b/core/plugin/plugin_test.go index 78e8faeb8..27228b81f 100644 --- a/core/plugin/plugin_test.go +++ b/core/plugin/plugin_test.go @@ -6,14 +6,14 @@ package plugin import ( - "errors" + stderrors "errors" "fmt" "io" "reflect" "testing" - "github.com/facebookgo/stackerr" "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -110,19 +110,18 @@ func TestNew(t *testing.T) { assert.Equal(t, testNewOk(), testInitValue) }}, {"non-nil error", func(t *testing.T) { - expectedErr := errors.New("fill conf err") + expectedErr := stderrors.New("fill conf err") r.testRegister(func() (TestPlugin, error) { return nil, expectedErr }) _, err := testNew(r) require.Error(t, err) - errs := stackerr.Underlying(err) - err = errs[len(errs)-1] + err = errors.Cause(err) assert.Equal(t, expectedErr, err) }}, {"no conf, fill conf error", func(t *testing.T) { r.testRegister(newPlugin) - expectedErr := errors.New("fill conf err") + expectedErr := stderrors.New("fill conf err") _, err := testNew(r, func(_ interface{}) error { return expectedErr }) assert.Equal(t, expectedErr, err) }}, diff --git a/core/plugin/pluginconfig/hooks.go b/core/plugin/pluginconfig/hooks.go index bdebc63e7..681374379 100644 --- a/core/plugin/pluginconfig/hooks.go +++ b/core/plugin/pluginconfig/hooks.go @@ -13,11 +13,12 @@ import ( "reflect" "strings" - "github.com/facebookgo/stackerr" + "github.com/pkg/errors" + "go.uber.org/zap" + "github.com/yandex/pandora/core/config" "github.com/yandex/pandora/core/plugin" "github.com/yandex/pandora/lib/tag" - "go.uber.org/zap" ) func AddHooks() { @@ -65,7 +66,7 @@ func parseConf(t reflect.Type, data interface{}) (name string, fillConf func(con if PluginNameKey == strings.ToLower(key) { strVal, ok := val.(string) if !ok { - err = stackerr.Newf("%s has non-string value %s", PluginNameKey, val) + err = errors.Errorf("%s has non-string value %s", PluginNameKey, val) return } names = append(names, strVal) @@ -73,11 +74,11 @@ func parseConf(t reflect.Type, data interface{}) (name string, fillConf func(con } } if len(names) == 0 { - err = stackerr.Newf("plugin %s expected", PluginNameKey) + err = errors.Errorf("plugin %s expected", PluginNameKey) return } if len(names) > 1 { - err = stackerr.Newf("too many %s keys", PluginNameKey) + err = errors.Errorf("too many %s keys", PluginNameKey) return } name = names[0] @@ -108,14 +109,14 @@ func toStringKeyMap(data interface{}) (out map[string]interface{}, err error) { } untypedKeyData, ok := data.(map[interface{}]interface{}) if !ok { - err = stackerr.Newf("unexpected config type %T: should be map[string or interface{}]interface{}", data) + err = errors.Errorf("unexpected config type %T: should be map[string or interface{}]interface{}", data) return } out = make(map[string]interface{}, len(untypedKeyData)) for key, val := range untypedKeyData { strKey, ok := key.(string) if !ok { - err = stackerr.Newf("unexpected key type %T: %v", key, key) + err = errors.Errorf("unexpected key type %T: %v", key, key) } out[strKey] = val } diff --git a/glide.lock b/glide.lock index b33db36b4..9bf34c575 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,12 @@ -hash: a8b2fdb8271a2c1bd63bd71b95e59c4016ff29336a21dd6fbabc3c4c5ca9f069 -updated: 2017-08-23T14:20:45.659155574+03:00 +hash: b06d35c760b8a60202cc52695b1dd3d3ac96acdbbb9a8cd52c0cba9a05880733 +updated: 2017-08-25T10:25:48.281437192+03:00 imports: - name: github.com/amahi/spdy version: 31da8b754faf6833fa95192c69bdb10518e9fb7b - name: github.com/asaskevich/govalidator - version: fdf19785fd3558d619ef81212f5edf1d6c2a5911 + version: 15028e809df8c71964e8efa6c11e81d5c0262302 - name: github.com/c2h5oh/datasize - version: ac35299842b215b2719d674ea369162c13ba4dc1 + version: 54516c931ae99c3c74637b9ea2390cf9a6327f26 - name: github.com/davecgh/go-spew version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 subpackages: @@ -16,11 +16,11 @@ imports: - name: github.com/facebookgo/stackerr version: c2fcf88613f4fc155efcb81619424db6cbb5bce0 - name: github.com/fatih/structs - version: a720dfa8df582c51dee1b36feabb906bde1588bd + version: 7e5a8eef611ee84dd359503f3969f80df4c50723 - name: github.com/fsnotify/fsnotify - version: fd9ec7deca8bf46ecd2a795baaacf2b3a9be1197 + version: 4da3e2cfbabc9f751898f250b49f2439785783a1 - name: github.com/hashicorp/hcl - version: eb6f65b2d77ed5078887f960ff570fbddbbeb49d + version: 392dba7d905ed5d04a5794ba89f558b27e2ba1ca subpackages: - hcl/ast - hcl/parser @@ -31,11 +31,12 @@ imports: - json/scanner - json/token - name: github.com/magiconair/properties - version: b3b15ef068fd0b17ddf408a23669f20811d194d2 + version: be5ece7dd465ab0765a9682137865547526d1dfb - name: github.com/mitchellh/mapstructure - version: ed105d635dfa9ea7133f7c79f1eb36203fc3a156 + version: e88fb6b4946b282e0d5196ac35b09d256e09e9d2 + repo: https://github.com/skipor/mapstructure - name: github.com/onsi/ginkgo - version: f24729c07644086231e82a4bc3a1cc2feff06e71 + version: 8382b23d18dbaaff8e5f7e83784c53ebb8ec2f47 subpackages: - config - extensions/table @@ -56,7 +57,7 @@ imports: - reporters/stenographer/support/go-isatty - types - name: github.com/onsi/gomega - version: f9b2e15b86f3d6dd556700ffbaa4feee86745028 + version: c893efa28eb45626cdaa76c9f653b62488858837 subpackages: - format - gbytes @@ -73,10 +74,8 @@ imports: - matchers/support/goraph/node - matchers/support/goraph/util - types -- name: github.com/pelletier/go-buffruneio - version: df1e16fde7fc330a0ca68167c23bf7ed6ac31d6d - name: github.com/pelletier/go-toml - version: a1f048ba24490f9b0674a67e1ce995d685cddf4a + version: 9c1b4e331f1e3d98e72600677699fbe212cd6d16 - name: github.com/pkg/errors version: c605e284fe17294bda444b34710735b29d1a9d90 - name: github.com/pmezard/go-difflib @@ -84,34 +83,34 @@ imports: subpackages: - difflib - name: github.com/pquerna/ffjson - version: c24ab9baf274473906fbdba5fccb594aec4b9ff1 + version: 9a5203b7a07166f217f5f8177d5b16177acad3b2 subpackages: - fflib/v1 - fflib/v1/internal - name: github.com/spf13/afero - version: 72b31426848c6ef12a7a8e216708cb0d1530f074 + version: 9be650865eab0c12963d8753212f4f9c66cdcf12 subpackages: - mem - name: github.com/spf13/cast - version: 56a7ecbeb18dde53c6db4bd96b541fd9741b8d44 + version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 - name: github.com/spf13/jwalterweatherman - version: fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66 + version: 0efa5202c04663c757d84f90f5219c1250baf94f - name: github.com/spf13/pflag - version: 51268031d79952516489a9f8aa34e9709b98d946 + version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 - name: github.com/spf13/viper - version: 5ed0fc31f7f453625df314d8e66b9791e8d13003 + version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 - name: github.com/stretchr/objx version: cbeaeb16a013161a98496fad62933b1d21786672 - name: github.com/stretchr/testify - version: 2402e8e7a02fc811447d11f881aa9746cdc57983 + version: 890a5c3458b43e6104ff5da8dfa139d013d77544 subpackages: - assert - mock - require - name: github.com/uber-go/atomic - version: 0506d69f5564c56e25797bf7183c28921d4c6360 + version: 70bd1261d36be490ebd22a62b385a3c5d23b6240 - name: go.uber.org/atomic - version: 4e336646b2ef9fc6e47be8e21594178f98e5ebcf + version: 70bd1261d36be490ebd22a62b385a3c5d23b6240 - name: go.uber.org/multierr version: 3c4937480c32f4c13a875a1829af76c98ca3d40a - name: go.uber.org/zap @@ -123,7 +122,7 @@ imports: - internal/exit - zapcore - name: golang.org/x/net - version: 30f03014b4f11f9f238fdeca8cca734746a24fbb + version: 57efc9c3d9f91fb3277f8da1cff370539c4d3dc5 subpackages: - html - html/atom @@ -133,11 +132,11 @@ imports: - idna - lex/httplex - name: golang.org/x/sys - version: d75a52659825e75fff6158388dddc6a5b04f9ba5 + version: 07c182904dbd53199946ba614a412c61d3c548f5 subpackages: - unix - name: golang.org/x/text - version: 11dbc599981ccdf4fb18802a28392a8bcf7a9395 + version: ac87088df8ef557f1e32cd00ed0b6fbc3f7ddafb subpackages: - encoding - encoding/charmap @@ -158,9 +157,9 @@ imports: - unicode/bidi - unicode/norm - name: gopkg.in/go-playground/validator.v8 - version: 5f57d2222ad794d0dffb07e664ea05e2ee07d60c + version: 5f1438d3fca68893a817e4a66806cea46a9e4ebf - name: gopkg.in/yaml.v2 - version: a5b47d31c556af34a302ce5d659e6fea44d90de0 + version: eb3733d160e74a9c7e442f435eb3bea458e1d19f testImports: - name: github.com/ghodss/yaml version: 0ca9ea5df5451ffdf184b4428c902747c2c11cd7 diff --git a/glide.yaml b/glide.yaml index cee747239..a1b405639 100644 --- a/glide.yaml +++ b/glide.yaml @@ -6,6 +6,7 @@ import: - package: github.com/facebookgo/stackerr - package: github.com/fatih/structs - package: github.com/mitchellh/mapstructure + repo: https://github.com/skipor/mapstructure # Need until fix https://github.com/mitchellh/mapstructure/issues/70 - package: github.com/pquerna/ffjson subpackages: - fflib/v1 diff --git a/lib/testutil/ginkgo.go b/lib/testutil/ginkgo.go index 0c7b840cc..b60339da6 100644 --- a/lib/testutil/ginkgo.go +++ b/lib/testutil/ginkgo.go @@ -6,7 +6,11 @@ package testutil import ( - "github.com/onsi/ginkgo" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/spf13/viper" "github.com/stretchr/testify/mock" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -22,7 +26,7 @@ func ReplaceGlobalLogger() *zap.Logger { func NewLogger() *zap.Logger { conf := zap.NewDevelopmentConfig() enc := zapcore.NewConsoleEncoder(conf.EncoderConfig) - core := zapcore.NewCore(enc, zapcore.AddSync(ginkgo.GinkgoWriter), zap.DebugLevel) + core := zapcore.NewCore(enc, zapcore.AddSync(GinkgoWriter), zap.DebugLevel) log := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zap.DPanicLevel)) return log } @@ -34,10 +38,18 @@ type Mock interface { func AssertExpectations(mocks ...Mock) { for _, m := range mocks { - m.AssertExpectations(ginkgo.GinkgoT(1)) + m.AssertExpectations(GinkgoT(1)) } } func AssertNotCalled(mock Mock, methodName string) { - mock.AssertNotCalled(ginkgo.GinkgoT(1), methodName) + mock.AssertNotCalled(GinkgoT(1), methodName) +} + +func ParseYAML(data string) map[string]interface{} { + v := viper.New() + v.SetConfigType("yaml") + err := v.ReadConfig(strings.NewReader(data)) + Expect(err).NotTo(HaveOccurred()) + return v.AllSettings() }