Merge pull request #173 from yandex/dev

v0.5.15

.mapping.json

@@ -53,9 +53,6 @@
   "components/providers/http/decoders/ammo/raw_ammo.go":"load/projects/pandora/components/providers/http/decoders/ammo/raw_ammo.go",
   "components/providers/http/decoders/decoder.go":"load/projects/pandora/components/providers/http/decoders/decoder.go",
   "components/providers/http/decoders/jsonline.go":"load/projects/pandora/components/providers/http/decoders/jsonline.go",
-  "components/providers/http/decoders/jsonline/data.go":"load/projects/pandora/components/providers/http/decoders/jsonline/data.go",
-  "components/providers/http/decoders/jsonline/data_ffjson.go":"load/projects/pandora/components/providers/http/decoders/jsonline/data_ffjson.go",
-  "components/providers/http/decoders/jsonline/data_test.go":"load/projects/pandora/components/providers/http/decoders/jsonline/data_test.go",
   "components/providers/http/decoders/jsonline_test.go":"load/projects/pandora/components/providers/http/decoders/jsonline_test.go",
   "components/providers/http/decoders/mock_decoder.go":"load/projects/pandora/components/providers/http/decoders/mock_decoder.go",
   "components/providers/http/decoders/raw.go":"load/projects/pandora/components/providers/http/decoders/raw.go",
@@ -256,8 +253,8 @@
   "examples/debug_and_profiling.yaml":"load/projects/pandora/examples/debug_and_profiling.yaml",
   "examples/http.jsonline":"load/projects/pandora/examples/http.jsonline",
   "examples/http.yaml":"load/projects/pandora/examples/http.yaml",
-  "go.mod":"gomod/go.mod",
-  "go.sum":"gomod/go.sum",
+  "go.mod":"load/projects/pandora/gomod/go.mod",
+  "go.sum":"load/projects/pandora/gomod/go.sum",
   "gomod/go.mod":"load/projects/pandora/gomod/go.mod",
   "gomod/go.sum":"load/projects/pandora/gomod/go.sum",
   "labels.json":"load/projects/pandora/labels.json",

README.md

@@ -46,7 +46,7 @@ Or use Pandora with [Yandex.Tank](https://yandextank.readthedocs.io/en/latest/co
 [Overload](https://overload.yandex.net).
 
 ### Documentation
-[Documentation](documentation/eng/index.md)
+[Documentation](https://yandex.github.io/pandora/)
 
 ### Old Documentation
 [ReadTheDocs](https://yandexpandora.readthedocs.io/)

cli/cli.go

@@ -24,7 +24,7 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-const Version = "0.5.14"
+const Version = "0.5.15"
 const defaultConfigFile = "load"
 const stdinConfigSelector = "-"
 

components/providers/http/decoders/decoder.go

@@ -71,7 +71,7 @@ func NewDecoder(conf config.Config, file io.ReadSeeker) (d Decoder, err error) {
 
 	switch conf.Decoder {
 	case config.DecoderJSONLine:
-		d = newJsonlineDecoder(file, conf, decodedConfigHeaders)
+		d, err = newJsonlineDecoder(file, conf, decodedConfigHeaders)
 	case config.DecoderRaw:
 		d = newRawDecoder(file, conf, decodedConfigHeaders)
 	case config.DecoderURI:

components/providers/http/decoders/jsonline.go

@@ -3,40 +3,72 @@ package decoders
 import (
 	"bufio"
 	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"net/http"
-	"strings"
 	"sync"
 
 	"github.com/yandex/pandora/components/providers/http/config"
 	"github.com/yandex/pandora/components/providers/http/decoders/ammo"
-	"github.com/yandex/pandora/components/providers/http/decoders/jsonline"
 	"github.com/yandex/pandora/core"
 	"golang.org/x/xerrors"
 )
 
-func newJsonlineDecoder(file io.ReadSeeker, cfg config.Config, decodedConfigHeaders http.Header) *jsonlineDecoder {
+func newJsonlineDecoder(file io.ReadSeeker, cfg config.Config, decodedConfigHeaders http.Header) (*jsonlineDecoder, error) {
 	scanner := bufio.NewScanner(file)
 	if cfg.MaxAmmoSize != 0 {
 		var buffer []byte
 		scanner.Buffer(buffer, cfg.MaxAmmoSize)
 	}
-	return &jsonlineDecoder{
+
+	isArray, err := isArray(file)
+	if err != nil {
+		return nil, err
+	}
+
+	decoder := &jsonlineDecoder{
 		protoDecoder: protoDecoder{
 			file:                 file,
 			config:               cfg,
 			decodedConfigHeaders: decodedConfigHeaders,
 		},
 		scanner: scanner,
 		pool:    &sync.Pool{New: func() any { return &ammo.Ammo{} }},
+		decoder: json.NewDecoder(file),
 	}
+	if isArray {
+		ammos, err := decoder.readArray()
+		if err != nil {
+			return decoder, fmt.Errorf("cant read json array: %w", err)
+		}
+		decoder.ammos = ammos
+	}
+	return decoder, nil
+}
+
+func isArray(r io.ReadSeeker) (bool, error) {
+	d := json.NewDecoder(r)
+	t, err := d.Token()
+	if err != nil {
+		return false, err
+	}
+	delim, ok := t.(json.Delim)
+	if !ok {
+		return false, errors.New("invalid json token")
+	}
+	_, err = r.Seek(0, io.SeekStart)
+	return delim.String() == "[", err
 }
 
 type jsonlineDecoder struct {
 	protoDecoder
 	scanner *bufio.Scanner
 	line    uint
 	pool    *sync.Pool
+	decoder *json.Decoder
+	ammos   []DecodedAmmo
 }
 
 func (d *jsonlineDecoder) Release(a core.Ammo) {
@@ -50,36 +82,57 @@ func (d *jsonlineDecoder) LoadAmmo(ctx context.Context) ([]DecodedAmmo, error) {
 	return d.protoDecoder.LoadAmmo(ctx, d.Scan)
 }
 
+type entity struct {
+	// Host defines Host header to send.
+	// Request endpoint is defied by gun config.
+	Host   string `json:"host"`
+	Method string `json:"method"`
+	URI    string `json:"uri"`
+	// Headers defines headers to send.
+	// NOTE: Host header will be silently ignored.
+	Headers map[string]string `json:"headers"`
+	Tag     string            `json:"tag"`
+	// Body should be string, doublequotes should be escaped for json body
+	Body string `json:"body"`
+}
+
 func (d *jsonlineDecoder) Scan(ctx context.Context) (DecodedAmmo, error) {
 	if d.config.Limit != 0 && d.ammoNum >= d.config.Limit {
 		return nil, ErrAmmoLimit
 	}
+	if d.ammos != nil {
+		return d.scanAmmos()
+	}
 	for {
 		if d.config.Passes != 0 && d.passNum >= d.config.Passes {
 			return nil, ErrPassLimit
 		}
-
-		for d.scanner.Scan() {
-			d.line++
-			data := d.scanner.Bytes()
-			if len(strings.TrimSpace(string(data))) == 0 {
-				continue
+		var da entity
+		err := d.decoder.Decode(&da)
+		if err != nil {
+			if err != io.EOF {
+				return nil, xerrors.Errorf("failed to decode ammo at line: %v; with err: %w", d.line+1, err)
 			}
+			// go to next pass
+		} else {
+			d.line++
 			d.ammoNum++
-			method, url, header, tag, body, err := jsonline.DecodeAmmo(data, d.decodedConfigHeaders)
-			if err != nil {
-				if !d.config.ContinueOnError {
-					return nil, xerrors.Errorf("failed to decode ammo at line: %v; data: %q, with err: %w", d.line+1, data, err)
-				}
-				// TODO: add log message about error
-				continue // skipping ammo
+
+			header := d.decodedConfigHeaders.Clone()
+			for k, v := range da.Headers {
+				header.Set(k, v)
+			}
+			url := "http://" + da.Host + da.URI // schema will be rewrite in gun
+			var body []byte
+			if da.Body != "" {
+				body = []byte(da.Body)
 			}
 			a := d.pool.Get().(*ammo.Ammo)
-			err = a.Setup(method, url, body, header, tag)
+			err = a.Setup(da.Method, url, body, header, da.Tag)
 			return a, err
 		}
 
-		err := d.scanner.Err()
+		err = d.scanner.Err()
 		if err != nil {
 			return nil, err
 		}
@@ -93,10 +146,51 @@ func (d *jsonlineDecoder) Scan(ctx context.Context) (DecodedAmmo, error) {
 		if err != nil {
 			return nil, err
 		}
-		d.scanner = bufio.NewScanner(d.file)
-		if d.config.MaxAmmoSize != 0 {
-			var buffer []byte
-			d.scanner.Buffer(buffer, d.config.MaxAmmoSize)
+		d.decoder = json.NewDecoder(d.file)
+	}
+}
+
+func (d *jsonlineDecoder) readArray() ([]DecodedAmmo, error) {
+	var data []entity
+	err := d.decoder.Decode(&data)
+	if err != nil {
+		return nil, fmt.Errorf("cant readArray, err: %w", err)
+	}
+	result := make([]DecodedAmmo, len(data))
+	for i, datum := range data {
+		header := d.decodedConfigHeaders.Clone()
+		for k, v := range datum.Headers {
+			header.Set(k, v)
+		}
+		url := "http://" + datum.Host + datum.URI // schema will be rewrite in gun
+		var body []byte
+		if datum.Body != "" {
+			body = []byte(datum.Body)
+		}
+		a := d.pool.Get().(*ammo.Ammo)
+		err = a.Setup(datum.Method, url, body, header, datum.Tag)
+		if err != nil {
+			return nil, fmt.Errorf("cant readArray, err: %w", err)
 		}
+		result[i] = a
+	}
+
+	return result, nil
+}
+
+func (d *jsonlineDecoder) scanAmmos() (DecodedAmmo, error) {
+	length := len(d.ammos)
+	if length == 0 {
+		return nil, ErrNoAmmo
+	}
+	if d.config.Passes != 0 && d.passNum >= d.config.Passes {
+		return nil, ErrPassLimit
+	}
+	i := int(d.ammoNum) % length
+	a := d.ammos[i]
+	if d.ammoNum > 0 && i == length-1 {
+		d.passNum++
 	}
+	d.ammoNum++
+	return a, nil
 }