Merge pull request #70 from yandex/feature/40

Support NewFactory constructors in plugin pkg. Target address pre resolve as bonus.

README.md

@@ -2,7 +2,7 @@
 
 [![Join the chat at https://gitter.im/yandex/pandora](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/yandex/pandora?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Build Status](https://travis-ci.org/yandex/pandora.svg)](https://travis-ci.org/yandex/pandora)
-[![Coverage Status](https://coveralls.io/repos/yandex/pandora/badge.svg?branch=master&service=github)](https://coveralls.io/github/yandex/pandora?branch=master)
+[![Coverage Status](https://coveralls.io/repos/yandex/pandora/badge.svg?branch=develop&service=github)](https://coveralls.io/github/yandex/pandora?branch=develop)
 
 A load generator in Go language.
 

components/example/import/import_suite_test.go

@@ -0,0 +1,20 @@
+package example
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/yandex/pandora/lib/testutil"
+)
+
+func TestImport(t *testing.T) {
+	testutil.RunSuite(t, "Import Suite")
+}
+
+var _ = Describe("import", func() {
+	It("not panics", func() {
+		Expect(Import).NotTo(Panic())
+	})
+})

components/phttp/client.go

@@ -6,7 +6,6 @@
 package phttp
 
 import (
-	"context"
 	"crypto/tls"
 	"net"
 	"net/http"
@@ -17,6 +16,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/yandex/pandora/core/config"
+	"github.com/yandex/pandora/lib/netutil"
 )
 
 //go:generate mockery -name=Client -case=underscore -inpkg -testonly
@@ -43,6 +43,8 @@ func NewDefaultClientConfig() ClientConfig {
 // DialerConfig can be mapped on net.Dialer.
 // Set net.Dialer for details.
 type DialerConfig struct {
+	DNSCache bool `config:"dns-cache" map:"-"`
+
 	Timeout   time.Duration `config:"timeout"`
 	DualStack bool          `config:"dual-stack"`
 
@@ -54,28 +56,20 @@ type DialerConfig struct {
 
 func NewDefaultDialerConfig() DialerConfig {
 	return DialerConfig{
+		DNSCache:  true,
+		DualStack: true,
 		Timeout:   3 * time.Second,
 		KeepAlive: 120 * time.Second,
 	}
 }
 
-type Dialer interface {
-	DialContext(ctx context.Context, network, address string) (net.Conn, error)
-}
-
-type DialerFunc func(ctx context.Context, network, address string) (net.Conn, error)
-
-func (f DialerFunc) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
-	return f(ctx, network, address)
-}
-
-func NewDialer(conf DialerConfig) *net.Dialer {
+func NewDialer(conf DialerConfig) netutil.Dialer {
 	d := &net.Dialer{}
-	err := config.Map(d, conf)
-	if err != nil {
-		zap.L().Panic("Dialer config map fail", zap.Error(err))
+	config.Map(d, conf)
+	if !conf.DNSCache {
+		return d
 	}
-	return d
+	return netutil.NewDNSCachingDialer(d, netutil.DefaultDNSCache)
 }
 
 // TransportConfig can be mapped on http.Transport.
@@ -101,21 +95,18 @@ func NewDefaultTransportConfig() TransportConfig {
 	}
 }
 
-func NewTransport(conf TransportConfig, dial DialerFunc) *http.Transport {
+func NewTransport(conf TransportConfig, dial netutil.DialerFunc) *http.Transport {
 	tr := &http.Transport{}
 	tr.TLSClientConfig = &tls.Config{
 		InsecureSkipVerify: true,                 // We should not spend time for this stuff.
 		NextProtos:         []string{"http/1.1"}, // Disable HTTP/2. Use HTTP/2 transport explicitly, if needed.
 	}
-	err := config.Map(tr, conf)
-	if err != nil {
-		zap.L().Panic("Transport config map fail", zap.Error(err))
-	}
+	config.Map(tr, conf)
 	tr.DialContext = dial
 	return tr
 }
 
-func NewHTTP2Transport(conf TransportConfig, dial DialerFunc) *http.Transport {
+func NewHTTP2Transport(conf TransportConfig, dial netutil.DialerFunc) *http.Transport {
 	tr := NewTransport(conf, dial)
 	err := http2.ConfigureTransport(tr)
 	if err != nil {

components/phttp/connect.go

@@ -15,6 +15,7 @@ import (
 	"net/url"
 
 	"github.com/pkg/errors"
+	"github.com/yandex/pandora/lib/netutil"
 )
 
 type ConnectGunConfig struct {
@@ -78,7 +79,7 @@ func newConnectClient(conf ConnectGunConfig) Client {
 	return newClient(transport, conf.Client.Redirect)
 }
 
-func newConnectDialFunc(target string, connectSSL bool, dialer Dialer) DialerFunc {
+func newConnectDialFunc(target string, connectSSL bool, dialer netutil.Dialer) netutil.DialerFunc {
 	return func(ctx context.Context, network, address string) (conn net.Conn, err error) {
 		// TODO(skipor): make connect sample.
 		// TODO(skipor): make httptrace callbacks called correctly.

components/phttp/http.go

@@ -36,7 +36,7 @@ func NewHTTPGun(conf HTTPGunConfig) *HTTPGun {
 // NewHTTP2Gun return simple HTTP/2 gun that can shoot sequentially through one connection.
 func NewHTTP2Gun(conf HTTP2GunConfig) (*HTTPGun, error) {
 	if !conf.Gun.SSL {
-		// Open issue on github if you need this feature.
+		// Open issue on github if you really need this feature.
 		return nil, errors.New("HTTP/2.0 over TCP is not supported. Please leave SSL option true by default.")
 	}
 	transport := NewHTTP2Transport(conf.Client.Transport, NewDialer(conf.Client.Dialer).DialContext)

components/phttp/import/import.go

@@ -6,14 +6,18 @@
 package phttp
 
 import (
+	"net"
+
 	"github.com/spf13/afero"
+	"go.uber.org/zap"
 
 	. "github.com/yandex/pandora/components/phttp"
 	"github.com/yandex/pandora/components/phttp/ammo/simple/jsonline"
 	"github.com/yandex/pandora/components/phttp/ammo/simple/raw"
 	"github.com/yandex/pandora/components/phttp/ammo/simple/uri"
 	"github.com/yandex/pandora/core"
 	"github.com/yandex/pandora/core/register"
+	"github.com/yandex/pandora/lib/netutil"
 )
 
 func Import(fs afero.Fs) {
@@ -29,16 +33,56 @@ func Import(fs afero.Fs) {
 		return raw.NewProvider(fs, conf)
 	})
 
-	register.Gun("http", func(conf HTTPGunConfig) core.Gun {
-		return WrapGun(NewHTTPGun(conf))
+	register.Gun("http", func(conf HTTPGunConfig) func() core.Gun {
+		preResolveTargetAddr(&conf.Client, &conf.Gun.Target)
+		return func() core.Gun { return WrapGun(NewHTTPGun(conf)) }
 	}, NewDefaultHTTPGunConfig)
 
-	register.Gun("http2", func(conf HTTP2GunConfig) (core.Gun, error) {
-		gun, err := NewHTTP2Gun(conf)
-		return WrapGun(gun), err
+	register.Gun("http2", func(conf HTTP2GunConfig) func() (core.Gun, error) {
+		preResolveTargetAddr(&conf.Client, &conf.Gun.Target)
+		return func() (core.Gun, error) {
+			gun, err := NewHTTP2Gun(conf)
+			return WrapGun(gun), err
+		}
 	}, NewDefaultHTTP2GunConfig)
 
-	register.Gun("connect", func(conf ConnectGunConfig) core.Gun {
-		return WrapGun(NewConnectGun(conf))
+	register.Gun("connect", func(conf ConnectGunConfig) func() core.Gun {
+		preResolveTargetAddr(&conf.Client, &conf.Target)
+		return func() core.Gun {
+			return WrapGun(NewConnectGun(conf))
+		}
 	}, NewDefaultConnectGunConfig)
 }
+
+// DNS resolve optimisation.
+// When DNSCache turned off - do nothing extra, host will be resolved on every shoot.
+// When using resolved target, don't use DNS caching logic - it is useless.
+// If we can resolve accessible target addr - use it as target, not use caching.
+// Otherwise just use DNS cache - we should not fail shooting, we should try to
+// connect on every shoot. DNS cache will save resolved addr after first successful connect.
+func preResolveTargetAddr(clientConf *ClientConfig, target *string) (err error) {
+	if !clientConf.Dialer.DNSCache {
+		return
+	}
+	if endpointIsResolved(*target) {
+		clientConf.Dialer.DNSCache = false
+		return
+	}
+	resolved, err := netutil.LookupReachable(*target)
+	if err != nil {
+		zap.L().Warn("DNS target pre resolve failed",
+			zap.String("target", *target), zap.Error(err))
+		return
+	}
+	clientConf.Dialer.DNSCache = false
+	*target = resolved
+	return
+}
+
+func endpointIsResolved(endpoint string) bool {
+	host, _, err := net.SplitHostPort(endpoint)
+	if err != nil {
+		return false
+	}
+	return net.ParseIP(host) != nil
+}

components/phttp/import/import_suite_test.go

@@ -0,0 +1,72 @@
+package phttp
+
+import (
+	"net"
+	"strconv"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/spf13/afero"
+
+	. "github.com/yandex/pandora/components/phttp"
+	"github.com/yandex/pandora/lib/testutil"
+)
+
+func TestImport(t *testing.T) {
+	testutil.RunSuite(t, "phttp Import Suite")
+}
+
+var _ = Describe("import", func() {
+	It("not panics", func() {
+		Expect(func() {
+			Import(afero.NewOsFs())
+		}).NotTo(Panic())
+	})
+})
+
+var _ = Describe("preResolveTargetAddr", func() {
+	It("host target", func() {
+		conf := &ClientConfig{}
+		conf.Dialer.DNSCache = true
+
+		listener, err := net.ListenTCP("tcp4", nil)
+		defer listener.Close()
+		Expect(err).NotTo(HaveOccurred())
+
+		port := strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)
+		target := "localhost:" + port
+		expectedResolved := "127.0.0.1:" + port
+
+		err = preResolveTargetAddr(conf, &target)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(conf.Dialer.DNSCache).To(BeFalse())
+
+		Expect(target).To(Equal(expectedResolved))
+	})
+
+	It("ip target", func() {
+		conf := &ClientConfig{}
+		conf.Dialer.DNSCache = true
+
+		const addr = "127.0.0.1:80"
+		target := addr
+		err := preResolveTargetAddr(conf, &target)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(conf.Dialer.DNSCache).To(BeFalse())
+		Expect(target).To(Equal(addr))
+	})
+
+	It("failed", func() {
+		conf := &ClientConfig{}
+		conf.Dialer.DNSCache = true
+
+		const addr = "localhost:54321"
+		target := addr
+		err := preResolveTargetAddr(conf, &target)
+		Expect(err).To(HaveOccurred())
+		Expect(conf.Dialer.DNSCache).To(BeTrue())
+		Expect(target).To(Equal(addr))
+	})
+
+})

core/config/config.go

@@ -54,19 +54,22 @@ func AddKindHook(hook KindHook) (_ struct{}) {
 // Example: you need to configure only some subset fields of struct Multi,
 // in such case you can from this subset of fields struct Single, decode config
 // into it, and map it on Multi.
-func Map(dst, src interface{}) error {
+func Map(dst, src interface{}) {
 	conf := &mapstructure.DecoderConfig{
 		ErrorUnused: true,
 		ZeroFields:  true,
 		Result:      dst,
 	}
 	d, err := mapstructure.NewDecoder(conf)
 	if err != nil {
-		return err
+		panic(err)
 	}
 	s := structs.New(src)
 	s.TagName = "map"
-	return d.Decode(s.Map())
+	err = d.Decode(s.Map())
+	if err != nil {
+		panic(err)
+	}
 }
 
 func newDecoderConfig(result interface{}) *mapstructure.DecoderConfig {

core/config/config_test.go

@@ -151,13 +151,11 @@ type SingleString struct {
 
 func TestMapFlat(t *testing.T) {
 	a := &MultiStrings{}
-	err := Map(a, &SingleString{B: "b"})
-	require.NoError(t, err)
+	Map(a, &SingleString{B: "b"})
 	assert.Equal(t, &MultiStrings{B: "b"}, a)
 
 	a = &MultiStrings{A: "a", B: "not b"}
-	err = Map(a, &SingleString{B: "b"})
-	require.NoError(t, err)
+	Map(a, &SingleString{B: "b"})
 	assert.Equal(t, &MultiStrings{A: "a", B: "b"}, a)
 }
 
@@ -170,8 +168,7 @@ func TestMapRecursive(t *testing.T) {
 		MultiStrings
 	}
 	n := &N{MultiStrings: MultiStrings{B: "b"}, A: "a"}
-	err := Map(n, &M{MultiStrings: MultiStrings{A: "a"}})
-	require.NoError(t, err)
+	Map(n, &M{MultiStrings: MultiStrings{A: "a"}})
 	assert.Equal(t, &N{A: "a", MultiStrings: MultiStrings{A: "a"}}, n)
 }
 
@@ -184,8 +181,7 @@ func TestMapTagged(t *testing.T) {
 		SomeOtherFieldName MultiStrings `map:"MultiStrings"`
 	}
 	n := &N{MultiStrings: MultiStrings{B: "b"}, A: "a"}
-	err := Map(n, &M{SomeOtherFieldName: MultiStrings{A: "a"}})
-	require.NoError(t, err)
+	Map(n, &M{SomeOtherFieldName: MultiStrings{A: "a"}})
 	assert.Equal(t, &N{A: "a", MultiStrings: MultiStrings{A: "a"}}, n)
 }
 

core/import/import_suite_test.go

@@ -6,7 +6,6 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/onsi/gomega/format"
 	"github.com/spf13/afero"
 
 	"github.com/yandex/pandora/core"
@@ -16,13 +15,8 @@ import (
 )
 
 func TestImport(t *testing.T) {
-	format.UseStringerRepresentation = true
-	RegisterFailHandler(Fail)
-
-	testutil.ReplaceGlobalLogger()
 	Import(afero.NewOsFs())
-
-	RunSpecs(t, "Import Suite")
+	testutil.RunSuite(t, "Import Suite")
 }
 
 var _ = Describe("plugin decode", func() {