From c5d1f1742c767de5127582d8e483a7f7bacffd9a Mon Sep 17 00:00:00 2001 From: Anis Eleuch Date: Thu, 11 Jul 2024 15:24:08 +0100 Subject: [PATCH 1/4] get: Add --range-size to have a random but a defined range pattern (#302) --range-size will create a random offset but with a fixed range --- cli/get.go | 17 ++++++++++++++++- pkg/bench/get.go | 15 +++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cli/get.go b/cli/get.go index 0e5f6fc..318edde 100644 --- a/cli/get.go +++ b/cli/get.go @@ -53,6 +53,10 @@ var getFlags = []cli.Flag{ Name: "range", Usage: "Do ranged get operations. Will request with random offset and length.", }, + cli.StringFlag{ + Name: "range-size", + Usage: "Use a fixed range size while doing random range offsets, --range is implied", + }, cli.IntFlag{ Name: "versions", Value: 1, @@ -81,11 +85,22 @@ FLAGS: // mainGet is the entry point for get command. func mainGet(ctx *cli.Context) error { checkGetSyntax(ctx) + + var rangeSize int64 + if rs := ctx.String("range-size"); rs != "" { + s, err := toSize(rs) + if err != nil { + return err + } + rangeSize = int64(s) + } + sse := newSSE(ctx) b := bench.Get{ Common: getCommon(ctx, newGenSource(ctx, "obj.size")), Versions: ctx.Int("versions"), - RandomRanges: ctx.Bool("range"), + RandomRanges: ctx.Bool("range") || ctx.IsSet("range-size"), + RangeSize: rangeSize, CreateObjects: ctx.Int("objects"), GetOpts: minio.GetObjectOptions{ServerSideEncryption: sse}, ListExisting: ctx.Bool("list-existing"), diff --git a/pkg/bench/get.go b/pkg/bench/get.go index e8c6ad3..3a1345d 100644 --- a/pkg/bench/get.go +++ b/pkg/bench/get.go @@ -43,6 +43,7 @@ type Get struct { CreateObjects int Versions int RandomRanges bool + RangeSize int64 ListExisting bool ListFlat bool } @@ -288,10 +289,16 @@ func (g *Get) Start(ctx context.Context, wait chan struct{}) (Operations, error) } if g.RandomRanges && op.Size > 2 { - // Randomize length similar to --obj.randsize - size := generator.GetExpRandSize(rng, 0, op.Size-2) - start := rng.Int63n(op.Size - size) - end := start + size + var start, end int64 + if g.RangeSize <= 0 { + // Randomize length similar to --obj.randsize + size := generator.GetExpRandSize(rng, 0, op.Size-2) + start = rng.Int63n(op.Size - size) + end = start + size + } else { + start = rng.Int63n(op.Size - g.RangeSize) + end = start + g.RangeSize - 1 + } op.Size = end - start + 1 opts.SetRange(start, end) } From b3452c34f08a0c0055cd6d75bfae85dae18efccc Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 26 Jul 2024 05:54:22 -0700 Subject: [PATCH 2/4] Add YAML configuration option (#325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit You can use an on-disk YAML configuration file as an alternative configuration option. See [yml-samples](https://github.com/minio/warp/tree/master/yml-samples) for a collection of configuration files for each benchmark type. To run a benchmark, use `λ warp run `. Values can be injected from the command line using one or multiple `-var VarName=Value`. These values can be referenced inside YAML files with `{{.VarName}}`. Go [text templates](https://pkg.go.dev/text/template) are used for this. --- .github/workflows/release.yml | 6 +- README.md | 37 ++-- cli/analyze.go | 6 + cli/benchmark.go | 2 + cli/cli.go | 3 +- cli/client.go | 2 +- cli/delete.go | 3 + cli/flags.go | 21 ++- cli/get.go | 3 + cli/mixed.go | 2 +- cli/run.go | 307 ++++++++++++++++++++++++++++++++++ go.mod | 1 + pkg/aggregate/aggregate.go | 12 ++ pkg/aggregate/requests.go | 19 ++- pkg/bench/multipart.go | 13 +- pkg/bench/ops.go | 59 +++++++ pkg/generator/options.go | 2 +- yml-samples/delete.yml | 197 ++++++++++++++++++++++ yml-samples/get.yml | 204 ++++++++++++++++++++++ yml-samples/list.yml | 192 +++++++++++++++++++++ yml-samples/mixed.yml | 193 +++++++++++++++++++++ yml-samples/multipart.yml | 181 ++++++++++++++++++++ yml-samples/put.yml | 181 ++++++++++++++++++++ yml-samples/stat.yml | 188 +++++++++++++++++++++ yml-samples/versioned.yml | 193 +++++++++++++++++++++ yml-samples/zip.yml | 180 ++++++++++++++++++++ 26 files changed, 2174 insertions(+), 33 deletions(-) create mode 100644 cli/run.go create mode 100644 yml-samples/delete.yml create mode 100644 yml-samples/get.yml create mode 100644 yml-samples/list.yml create mode 100644 yml-samples/mixed.yml create mode 100644 yml-samples/multipart.yml create mode 100644 yml-samples/put.yml create mode 100644 yml-samples/stat.yml create mode 100644 yml-samples/versioned.yml create mode 100644 yml-samples/zip.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5db3085..c5ba8e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,21 +27,21 @@ jobs: args: release --clean --skip=publish --snapshot - name: Upload Win64 Binaries - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 if: success() with: name: Warp-Snapshot-Build-Win64 path: dist/warp_windows_amd64_v1 - name: Upload Linux Binaries - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 if: success() with: name: Warp-Snapshot-Build-Linux-amd64 path: dist/warp_linux_amd64_v1 - name: Upload MacOS Binaries - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 if: success() with: name: Warp-Snapshot-Build-MacOSX-amd64 diff --git a/README.md b/README.md index ba2cbce..804acb6 100644 --- a/README.md +++ b/README.md @@ -9,20 +9,20 @@ S3 benchmarking tool. ## Build with source -Warp require minimum version is `go1.22`, please ensure you have compatible version for this build. +Warp requires minimum Go `go1.21`, please ensure you have compatible version for this build. You can follow easy step below to build project - Clone project ``` -git clone https://github.com/minio/warp.git +λ git clone https://github.com/minio/warp.git ``` - Change directory and build ``` -cd warp && go build +λ cd warp && go build ``` - To run a test, please run ``` -./warp [options] +λ ./warp [options] ``` # Configuration @@ -49,15 +49,28 @@ If your server is incompatible with [AWS v4 signatures](https://docs.aws.amazon. # Usage -`warp command [options]` +`λ warp command [options]` Example running a mixed type benchmark against 8 servers named `s3-server-1` to `s3-server-8` on port 9000 with the provided keys: -`warp mixed --host=s3-server{1...8}:9000 --access-key=minio --secret-key=minio123 --autoterm` +`λ warp mixed --host=s3-server{1...8}:9000 --access-key=minio --secret-key=minio123 --autoterm` This will run the benchmark for up to 5 minutes and print the results. +## YAML configuration + +As an alternative configuration option you can use an on-disk YAML configuration file. + +See [yml-samples](https://github.com/minio/warp/tree/master/yml-samples) for a collection of +configuration files for each benchmark type. + +To run a benchmark use `λ warp run `. + +Values can be injected from the commandline using one or multiple `-var VarName=Value`. +These values can be referenced inside YAML files with `{{.VarName}}`. +Go [text templates](https://pkg.go.dev/text/template) are used for this. + # Benchmarks All benchmarks operate concurrently. By default, 20 operations will run concurrently. @@ -110,7 +123,7 @@ WARNING: Never run warp clients on a publicly exposed port. Clients have the pot Clients are started with ``` -warp client [listenaddress:port] +λ warp client [listenaddress:port] ``` `warp client` Only accepts an optional host/ip to listen on, but otherwise no specific parameters. @@ -141,7 +154,7 @@ If no host port is specified the default is added. Example: ``` -warp get --duration=3m --warp-client=client-{1...10} --host=minio-server-{1...16} --access-key=minio --secret-key=minio123 +λ warp get --duration=3m --warp-client=client-{1...10} --host=minio-server-{1...16} --access-key=minio --secret-key=minio123 ``` Note that parameters apply to *each* client. @@ -309,8 +322,6 @@ will attempt to run `--concurrent` concurrent downloads. The analysis will include the upload stats as `PUT` operations and the `GET` operations. - - ``` Operation: GET * Average: 94.10 MiB/s, 9866.97 obj/s @@ -407,7 +418,7 @@ Since the object size is of little importance, only objects per second is report Example: ``` -$ warp stat --autoterm +λ warp stat --autoterm [...] ------------------- Operation: STAT @@ -735,7 +746,7 @@ These are the data fields exported: | `ops_started` | Operations started within segment | | `ops_ended` | Operations ended within the segment | | `errors` | Errors logged on operations ending within the segment | -| `mb_per_sec` | MiB/s of operations within the segment (*distributed*) | +| `mb_per_sec` | MiB/s of operations within the segment (*distributed*) | | `ops_ended_per_sec` | Operations that ended within the segment per second | | `objs_per_sec` | Objects per second processed in the segment (*distributed*) | | `start_time` | Absolute start time of the segment | @@ -783,7 +794,7 @@ The usual analysis parameters can be applied to define segment lengths. ## Merging Benchmarks -It is possible to merge runs from several clients using the `warp merge (file1) (file2) [additional files...]` command. +It is possible to merge runs from several clients using the `λ warp merge (file1) (file2) [additional files...]` command. The command will output a combined data file with all data that overlap in time. diff --git a/cli/analyze.go b/cli/analyze.go index 80bedcf..073b286 100644 --- a/cli/analyze.go +++ b/cli/analyze.go @@ -424,6 +424,12 @@ func writeSegs(ctx *cli.Context, wrSegs io.Writer, ops bench.Operations, allThre // Write segments per endpoint eps := ops.SortSplitByEndpoint() + if len(eps) == 1 { + cl := ops.SortSplitByClient() + if len(cl) > 1 { + eps = cl + } + } epsSorted := stringKeysSorted(eps) if details && len(eps) > 1 { for _, ep := range epsSorted { diff --git a/cli/benchmark.go b/cli/benchmark.go index e44ab15..51cf323 100644 --- a/cli/benchmark.go +++ b/cli/benchmark.go @@ -515,6 +515,8 @@ func checkBenchmark(ctx *cli.Context) { madmin.ProfilerBlock, madmin.ProfilerMutex, madmin.ProfilerTrace, + madmin.ProfilerCPUIO, + madmin.ProfilerThreads, } _, err := parseInfluxURL(ctx) diff --git a/cli/cli.go b/cli/cli.go index 4c34c88..8decd14 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -104,6 +104,7 @@ func init() { cmpCmd, mergeCmd, clientCmd, + runCmd, } appCmds = append(append(appCmds, a...), b...) benchCmds = a @@ -214,7 +215,7 @@ func registerApp(name string, appCmds []cli.Command) *cli.App { app.Commands = commands app.Author = "MinIO, Inc." app.Version = pkg.Version + " - " + pkg.ShortCommitID - app.Copyright = "(c) 2020-2023 MinIO, Inc." + app.Copyright = "(c) 2020-2024 MinIO, Inc." app.Compiled, _ = time.Parse(time.RFC3339, pkg.ReleaseTime) app.Flags = append(app.Flags, profileFlags...) app.Flags = append(app.Flags, globalFlags...) diff --git a/cli/client.go b/cli/client.go index d423d2f..b90bdb9 100644 --- a/cli/client.go +++ b/cli/client.go @@ -187,7 +187,7 @@ func clientTransport(ctx *cli.Context) http.RoundTripper { }).DialContext, MaxIdleConnsPerHost: ctx.Int("concurrent"), WriteBufferSize: ctx.Int("sndbuf"), // Configure beyond 4KiB default buffer size. - ReadBufferSize: ctx.Int("sndbuf"), // Configure beyond 4KiB default buffer size. + ReadBufferSize: ctx.Int("rcvbuf"), // Configure beyond 4KiB default buffer size. IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 15 * time.Second, ExpectContinueTimeout: 10 * time.Second, diff --git a/cli/delete.go b/cli/delete.go index f5d486b..afc5686 100644 --- a/cli/delete.go +++ b/cli/delete.go @@ -80,6 +80,9 @@ func mainDelete(ctx *cli.Context) error { ListFlat: ctx.Bool("list-flat"), ListPrefix: ctx.String("prefix"), } + if b.ListExisting && !ctx.IsSet("objects") { + b.CreateObjects = 0 + } return runBench(ctx, &b) } diff --git a/cli/flags.go b/cli/flags.go index 7da08db..31259e1 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -105,10 +105,10 @@ var globalWG sync.WaitGroup // Set global states. NOTE: It is deliberately kept monolithic to ensure we dont miss out any flags. func setGlobalsFromContext(ctx *cli.Context) error { - quiet := ctx.IsSet("quiet") - debug := ctx.IsSet("debug") - json := ctx.IsSet("json") - noColor := ctx.IsSet("no-color") + quiet := ctx.Bool("quiet") + debug := ctx.Bool("debug") + json := ctx.Bool("json") + noColor := ctx.Bool("no-color") setGlobals(quiet, debug, json, noColor) return nil } @@ -209,9 +209,16 @@ var ioFlags = []cli.Flag{ Usage: "Run this many concurrent operations per warp client", }, cli.IntFlag{ - Name: "sndbuf", - Value: 32 * 1024, // 32KiB up from 4KiB default - Usage: "specify custom read/write socket buffer size in bytes", + Name: "sndbuf", + Value: 32 * 1024, // 32KiB up from 4KiB default + Usage: "specify custom write socket buffer size in bytes", + Hidden: true, + }, + cli.IntFlag{ + Name: "rcvbuf", + Value: 32 * 1024, // 32KiB up from 4KiB default + Usage: "specify custom read socket buffer size in bytes", + Hidden: true, }, cli.BoolFlag{ Name: "noprefix", diff --git a/cli/get.go b/cli/get.go index 318edde..624cea7 100644 --- a/cli/get.go +++ b/cli/get.go @@ -107,6 +107,9 @@ func mainGet(ctx *cli.Context) error { ListFlat: ctx.Bool("list-flat"), ListPrefix: ctx.String("prefix"), } + if b.ListExisting && !ctx.IsSet("objects") { + b.CreateObjects = 0 + } return runBench(ctx, &b) } diff --git a/cli/mixed.go b/cli/mixed.go index f40ad72..0d082d5 100644 --- a/cli/mixed.go +++ b/cli/mixed.go @@ -55,7 +55,7 @@ var mixedFlags = []cli.Flag{ }, cli.Float64Flag{ Name: "delete-distrib", - Usage: "The amount of DELETE operations. Must be at least the same as PUT.", + Usage: "The amount of DELETE operations. Must be same or lower than -put-distrib", Value: 10, }, } diff --git a/cli/run.go b/cli/run.go new file mode 100644 index 0000000..2407edd --- /dev/null +++ b/cli/run.go @@ -0,0 +1,307 @@ +/* + * Warp (C) 2019-2024 MinIO, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package cli + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "strings" + "text/template" + + "github.com/minio/cli" + "github.com/minio/mc/pkg/probe" + "gopkg.in/yaml.v3" +) + +var runCmd = cli.Command{ + Name: "run", + Usage: "run benchmark defined in YAML file", + Action: mainExec, + Before: setGlobalsFromContext, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + Usage: "enable debug logging before executing", + Hidden: true, + }, + cli.StringSliceFlag{ + Name: "var", + Usage: "Set variables for template replacement. Can be used multiple times. Example: ObjSize=1KB", + }, + }, + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + + Execute the benchmark as defined in YAML file. +USAGE: + {{.HelpName}} + -> see https://github.com/minio/warp#run + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}`, +} + +// mainExec is the entry point for exe command. +func mainExec(ctx *cli.Context) error { + var yFile []byte + switch ctx.NArg() { + case 1: + b, err := os.ReadFile(ctx.Args()[0]) + if err != nil { + fatal(probe.NewError(err), "error reading input file") + } + yFile = b + default: + fatal(errInvalidArgument(), "No YAML file specified") + } + + // Do template replacements + if vars := ctx.StringSlice("var"); len(vars) > 0 { + replacements := make(map[string]string) + for _, v := range vars { + idx := strings.Index(v, "=") + if idx <= 0 { + fatal(probe.NewError(fmt.Errorf("unable to parse %q", v)), "unable to parse --var") + } + name := v[:idx] + replacements[name] = v[idx+1:] + } + t, err := template.New("benchmark").Parse(string(yFile)) + if err != nil { + fatal(probe.NewError(err), "error parsing template") + } + dst := new(bytes.Buffer) + err = t.ExecuteTemplate(dst, "benchmark", replacements) + if err != nil { + fatal(probe.NewError(err), "error parsing template") + } + yFile = dst.Bytes() + } + + // Unmarshal into map. + var doc map[string]any + err := yaml.Unmarshal(yFile, &doc) + if err != nil { + fatal(probe.NewError(err), "error parsing YAML file") + } + doc, ok := doc["warp"].(map[string]any) + if !ok { + fatal(probe.NewError(err), "Expected top level 'warp' element could not be found") + } + switch ver := mustGetString(doc, "api"); ver { + case "v1": + default: + fatal(probe.NewError(fmt.Errorf("unsupported api: %s", ver)), "Incompatible API version") + } + delete(doc, "api") + op := mustGetString(doc, "benchmark") + var benchCmd *cli.Command + for i, cmd := range benchCmds { + if cmd.Name == op { + benchCmd = &benchCmds[i] + break + } + } + if benchCmd == nil { + fatal(probe.NewError(fmt.Errorf("unknown benchmark: %s", op)), "Unknown benchmark") + } + delete(doc, "benchmark") + + // Rename input fields to commandline params: + rename := map[string]string{ + "sse-c-encrypt": "encrypt", + "analyze.verbose": "analyze.v", + "obj.part-size": "part.size", + "analyze.skip-duration": "analyze.skip", + "analyze.segment-duration": "analyze.dur", + "analyze.filter-op": "analyze.op", + "server-profile": "serverprof", + "bench-data": "benchdata", + "autoterm.enabled": "autoterm", + "obj.versions": "versions", + "obj.rand-size": "obj.randsize", + "no-prefix": "noprefix", + "no-clear": "noclear", + "distribution.get": "get-distrib", + "distribution.stat": "stat-distrib", + "distribution.put": "put-distrib", + "distribution.delete": "delete-distrib", + "obj.parts": "parts", + } + + // Allow some fields to be string lists + commaListed := map[string]bool{ + "server-profile": true, + "remote.host": true, + "warp-client": true, + } + + var prefixStack []string + var currDept []string + flags := map[string]string{} + setFlag := func(key string, value any) { + name := strings.Join(append(prefixStack, key), ".") + ogName := strings.Join(append(currDept, key), ".") + + if alt := rename[name]; alt != "" { + name = alt + } + var flag cli.Flag + for _, f := range benchCmd.Flags { + if strings.Split(f.GetName(), ",")[0] == name { + flag = f + break + } + } + if value == nil { + if globalDebug { + fmt.Printf("%s => --%s=(default)\n", ogName, name) + } + // Ignore unset values + return + } + if flag == nil { + section := strings.Join(currDept, ".") + fatal(probe.NewError(fmt.Errorf("unknown key: %q in section %q", name, section)), "Unknown benchmark flag") + } + + var err error + var setFlag []byte + switch flag.(type) { + case cli.BoolFlag: + if v, ok := value.(bool); !ok { + err = fmt.Errorf("value of %s must be a bool", ogName) + } else { + setFlag, err = json.Marshal(v) + } + case cli.StringFlag, cli.DurationFlag: + var wasList bool + if commaListed[ogName] { + if v, ok := value.([]any); ok { + wasList = true + var all []string + for i, v := range v { + value, ok := v.(string) + if !ok { + err = fmt.Errorf("value of %s item %d must be a string", ogName, i+1) + break + } + all = append(all, value) + } + setFlag = []byte(strings.Join(all, ",")) + } + } + if !wasList { + if v, ok := value.(string); !ok { + err = fmt.Errorf("value of %s must be a string, was %T", ogName, value) + } else { + setFlag = []byte(v) + } + } + case cli.IntFlag, cli.Float64Flag, cli.UintFlag, cli.Uint64Flag: + switch v := value.(type) { + case float64, int: + setFlag, err = json.Marshal(v) + default: + err = fmt.Errorf("value of %s must be a number, was %T", ogName, value) + } + + default: + err = fmt.Errorf("unknown flag type %T for key %s", flag, ogName) + } + if err != nil { + fatal(probe.NewError(err), "error parsing config") + return + } + if _, ok := flags[name]; ok { + fatal(probe.NewError(fmt.Errorf("duplicate benchmark flag: %s", ogName)), "duplicate benchmark flag") + } + flags[name] = string(setFlag) + if globalDebug { + fmt.Printf("%s => --%s=%s\n", ogName, name, string(setFlag)) + } + } + parseDoc(doc, &prefixStack, &currDept, setFlag) + + // Reconstruct command + app := registerApp("warp", benchCmds) + fs, err := flagSet(benchCmd.Name, benchCmd.Flags, nil) + if err != nil { + fatal(probe.NewError(err), "error setting flags") + } + ctx2 := cli.NewContext(app, fs, ctx) + ctx2.Command = *benchCmd + for k, v := range flags { + err := ctx2.Set(k, v) + if err != nil { + err := fmt.Errorf("parsing parameters (%v:%v): %w", k, v, err) + fatal(probe.NewError(err), "error setting flags") + } + } + + return runCommand(ctx2, benchCmd) +} + +func parseDoc(doc map[string]any, prefixStack, printStack *[]string, setFlag func(key string, value any)) { + push := func(stack *[]string, s string) func() { + v := *stack + v = append(v, s) + *stack = v + return func() { + v := *stack + v = v[:len(v)-1] + *stack = v + } + } + for k, v := range doc { + switch k { + case "analyze", "obj", "autoterm", "distribution": + // These automatically adds the prefix to the flag name. + pop := push(prefixStack, k) + pop2 := push(printStack, k) + for k, v := range v.(map[string]any) { + setFlag(k, v) + } + pop() + pop2() + + case "io", "remote", "params", "advanced": + // These are just added to the top level. + pop := push(printStack, k) + parseDoc(v.(map[string]any), prefixStack, printStack, setFlag) + pop() + default: + setFlag(k, v) + } + } +} + +func mustGetString(m map[string]any, key string) string { + v, ok := m[key] + if !ok { + fatal(probe.NewError(fmt.Errorf("value of '%s' not found", key)), "Missing key") + } + val, ok := v.(string) + if !ok { + fatal(probe.NewError(fmt.Errorf("value of '%s' must be a string", key)), "Invalid type") + } + return val +} diff --git a/go.mod b/go.mod index 39374cf..23a100c 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/secure-io/sio-go v0.3.1 golang.org/x/net v0.25.0 golang.org/x/time v0.5.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/pkg/aggregate/aggregate.go b/pkg/aggregate/aggregate.go index 083f2b5..53e79b6 100644 --- a/pkg/aggregate/aggregate.go +++ b/pkg/aggregate/aggregate.go @@ -134,6 +134,12 @@ func Aggregate(o bench.Operations, opts Options) Aggregated { } eps := o.SortSplitByEndpoint() + if len(eps) == 1 { + cl := ops.SortSplitByClient() + if len(cl) > 1 { + eps = cl + } + } a.MixedThroughputByHost = make(map[string]Throughput, len(eps)) var wg sync.WaitGroup var mu sync.Mutex @@ -229,6 +235,12 @@ func Aggregate(o bench.Operations, opts Options) Aggregated { } eps := allOps.SortSplitByEndpoint() + if len(eps) == 1 { + cl := ops.SortSplitByClient() + if len(cl) > 1 { + eps = cl + } + } a.ThroughputByHost = make(map[string]Throughput, len(eps)) var epMu sync.Mutex var epWg sync.WaitGroup diff --git a/pkg/aggregate/requests.go b/pkg/aggregate/requests.go index 9e285dd..bddfb7a 100644 --- a/pkg/aggregate/requests.go +++ b/pkg/aggregate/requests.go @@ -233,13 +233,21 @@ func RequestAnalysisSingleSized(o bench.Operations, allThreads bool) *SingleSize res.fillFirstLast(o) res.HostNames = o.Endpoints() res.ByHost = RequestAnalysisHostsSingleSized(o) - + if len(res.HostNames) != len(res.ByHost) { + res.HostNames = o.ClientIDs() + } return &res } // RequestAnalysisHostsSingleSized performs host analysis where all objects have equal size. func RequestAnalysisHostsSingleSized(o bench.Operations) map[string]SingleSizedRequests { eps := o.SortSplitByEndpoint() + if len(eps) == 1 { + cl := o.SortSplitByClient() + if len(cl) > 1 { + eps = cl + } + } res := make(map[string]SingleSizedRequests, len(eps)) var wg sync.WaitGroup var mu sync.Mutex @@ -276,6 +284,9 @@ func RequestAnalysisMultiSized(o bench.Operations, allThreads bool) *MultiSizedR res.fill(active) res.ByHost = RequestAnalysisHostsMultiSized(active) res.HostNames = active.Endpoints() + if len(res.HostNames) != len(res.ByHost) { + res.HostNames = o.ClientIDs() + } return &res } @@ -283,6 +294,12 @@ func RequestAnalysisMultiSized(o bench.Operations, allThreads bool) *MultiSizedR func RequestAnalysisHostsMultiSized(o bench.Operations) map[string]RequestSizeRange { start, end := o.TimeRange() eps := o.SortSplitByEndpoint() + if len(eps) == 1 { + cl := o.SortSplitByClient() + if len(cl) > 1 { + eps = cl + } + } res := make(map[string]RequestSizeRange, len(eps)) var wg sync.WaitGroup var mu sync.Mutex diff --git a/pkg/bench/multipart.go b/pkg/bench/multipart.go index 43ca7ac..6906570 100644 --- a/pkg/bench/multipart.go +++ b/pkg/bench/multipart.go @@ -80,8 +80,11 @@ func (g *Multipart) Prepare(ctx context.Context) error { var wg sync.WaitGroup wg.Add(g.Concurrency) g.addCollector() - objs := splitObjs(g.CreateParts, g.Concurrency) - + obj := make(chan int, g.CreateParts) + for i := 0; i < g.CreateParts; i++ { + obj <- i + g.PartStart + } + close(obj) rcv := g.Collector.rcv var groupErr error var mu sync.Mutex @@ -89,8 +92,8 @@ func (g *Multipart) Prepare(ctx context.Context) error { if g.Custom == nil { g.Custom = make(map[string]string, g.CreateParts) } - for i, obj := range objs { - go func(i int, obj []struct{}) { + for i := 0; i < g.Concurrency; i++ { + go func(i int) { defer wg.Done() src := g.Source() opts := g.PutOpts @@ -161,7 +164,7 @@ func (g *Multipart) Prepare(ctx context.Context) error { mu.Unlock() rcv <- op } - }(i, obj) + }(i) } wg.Wait() return groupErr diff --git a/pkg/bench/ops.go b/pkg/bench/ops.go index 2236e7a..5eb7891 100644 --- a/pkg/bench/ops.go +++ b/pkg/bench/ops.go @@ -222,6 +222,25 @@ func (o Operations) SortByEndpoint() { }) } +// SortByClient will sort the operations by client. +// Earliest operations first. +func (o Operations) SortByClient() { + if sort.SliceIsSorted(o, func(i, j int) bool { + if o[i].ClientID == o[j].ClientID { + return o[i].Start.Before(o[j].Start) + } + return o[i].ClientID < o[j].ClientID + }) { + return + } + sort.Slice(o, func(i, j int) bool { + if o[i].ClientID == o[j].ClientID { + return o[i].Start.Before(o[j].Start) + } + return o[i].ClientID < o[j].ClientID + }) +} + // SortByOpType will sort the operations by operation type. // Earliest operations first. func (o Operations) SortByOpType() { @@ -366,6 +385,30 @@ func (o Operations) SortSplitByEndpoint() map[string]Operations { return dst } +// SortSplitByClient will sort operations by endpoint and split by host. +func (o Operations) SortSplitByClient() map[string]Operations { + clients := o.Clients() + o.SortByClient() + dst := make(map[string]Operations, clients) + cl := "" + start := 0 + for i, op := range o { + if op.ClientID == cl { + continue + } + if cl != "" { + dst[cl] = o[start:i] + } + cl = op.ClientID + start = i + } + if cl != "" { + dst[cl] = o[start:] + } + + return dst +} + // SortSplitByOpType will sort operations by op + start time and split by op. func (o Operations) SortSplitByOpType() map[string]Operations { o.SortByOpType() @@ -848,6 +891,22 @@ func (o Operations) Endpoints() []string { return dst } +func (o Operations) ClientIDs() []string { + if len(o) == 0 { + return nil + } + found := make(map[string]struct{}, 1) + for _, op := range o { + found[op.ClientID] = struct{}{} + } + dst := make([]string, 0, len(found)) + for k := range found { + dst = append(dst, k) + } + sort.Strings(dst) + return dst +} + // Errors returns the errors found. func (o Operations) Errors() []string { if len(o) == 0 { diff --git a/pkg/generator/options.go b/pkg/generator/options.go index 749c982..24d200b 100644 --- a/pkg/generator/options.go +++ b/pkg/generator/options.go @@ -99,7 +99,7 @@ func WithSize(n int64) Option { // WithRandomSize will randomize the size from 1 byte to the total size set. func WithRandomSize(b bool) Option { return func(o *Options) error { - if o.totalSize > 0 && o.totalSize < 256 { + if b && o.totalSize > 0 && o.totalSize < 256 { return errors.New("WithRandomSize: Random sized objects should be at least 256 bytes") } o.randSize = b diff --git a/yml-samples/delete.yml b/yml-samples/delete.yml new file mode 100644 index 0000000..777382c --- /dev/null +++ b/yml-samples/delete.yml @@ -0,0 +1,197 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: delete + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 5m + + # Concurrent operations to run per warp instance. + concurrent: 8 + + # The number of objects to upload before starting the benchmark. + # Upload enough objects to ensure that any remote caching is bypassed. + objects: 32000 + + # Number of DELETE operations per batch. + batch: 100 + + # Properties of uploaded objects. + obj: + # Size of each uploaded object + size: 1KiB + + # Randomize the size of each object within certain constraints. + # See https://github.com/minio/warp?tab=readme-ov-file#random-file-sizes + rand-size: false + + # Force specific size of each multipart part. + # Must be '5MB' or bigger. + part-size: + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Instead of preparing the bench by PUTing some objects, + # only use objects already in the bucket. + # If 'objects' is set > 0 this will limit the number of objects. + # Does not perform any deletes before or after benchmark. + list-existing: false + + # When using list-existing, do not use recursive listing + list-flat: false + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: diff --git a/yml-samples/get.yml b/yml-samples/get.yml new file mode 100644 index 0000000..cb5aa58 --- /dev/null +++ b/yml-samples/get.yml @@ -0,0 +1,204 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: get + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 1m + + # Concurrent operations to run per warp instance. + concurrent: 8 + + # The number of objects to upload before starting the benchmark. + # Upload enough objects to ensure that any remote caching is bypassed. + objects: 1000 + + # Properties of uploaded objects. + obj: + # Size of each uploaded object + size: 100KiB + + # Number of versions to upload of each object + versions: 2 + + # Randomize the size of each object within certain constraints. + # See https://github.com/minio/warp?tab=readme-ov-file#random-file-sizes + rand-size: false + + # Force specific size of each multipart part. + # Must be '5MB' or bigger. + part-size: + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Do RANGE get operations. Will request with random offset and length. + range: false + + # Use a fixed range size while doing random range offsets. + # Sizes can be '1KB', '2MB', etc. 'range' is implied. + range-size: + + # Instead of preparing the bench by PUTing some objects, + # only use objects already in the bucket. + # If 'objects' is set > 0 this will limit the number of objects. + # Does not perform any deletes before or after benchmark. + list-existing: false + + # When using list-existing, do not use recursive listing + list-flat: false + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: diff --git a/yml-samples/list.yml b/yml-samples/list.yml new file mode 100644 index 0000000..4921a5b --- /dev/null +++ b/yml-samples/list.yml @@ -0,0 +1,192 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: list + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 1m + + # Concurrent operations to run per warp instance. + concurrent: 8 + + # The number of objects to upload before starting the benchmark. + # Upload enough objects to ensure that any remote caching is bypassed. + objects: 10000 + + # Enable extended MinIO ListObjects with metadata, + # by default this benchmarking uses ListObjectsV2 API + metadata: false + + # Properties of uploaded objects. + obj: + # Size of each uploaded object + size: 1B + + # Number of versions to upload of each object + versions: 2 + + # Randomize the size of each object within certain constraints. + # See https://github.com/minio/warp?tab=readme-ov-file#random-file-sizes + rand-size: false + + # Force specific size of each multipart part. + # Must be '5MB' or bigger. + part-size: + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: diff --git a/yml-samples/mixed.yml b/yml-samples/mixed.yml new file mode 100644 index 0000000..7459953 --- /dev/null +++ b/yml-samples/mixed.yml @@ -0,0 +1,193 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: mixed + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 1m + + # Concurrent operations to run per warp instance. + concurrent: 8 + + # The number of objects to upload before starting the benchmark. + # Upload enough objects to ensure that any remote caching is bypassed. + objects: 1000 + + # Adjust the distribution of each operation type + # The final distribution will be determined by the fraction of each value of the total. + distribution: + get: 45.0 + stat: 30.0 + put: 15.0 + delete: 10.0 # Must be same or lower than 'put'. + + # Properties of uploaded objects. + obj: + # Size of each uploaded object + size: 100KiB + + # Randomize the size of each object within certain constraints. + # See https://github.com/minio/warp?tab=readme-ov-file#random-file-sizes + rand-size: false + + # Force specific size of each multipart part. + # Must be '5MB' or bigger. + part-size: + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: diff --git a/yml-samples/multipart.yml b/yml-samples/multipart.yml new file mode 100644 index 0000000..1bfcec7 --- /dev/null +++ b/yml-samples/multipart.yml @@ -0,0 +1,181 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: multipart + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 1m + + # Concurrent operations to run per warp instance. + concurrent: 8 + + # Properties of uploaded object. + obj: + # Object name + name: 'warp-multipart.bin' + + # Parts to add per warp client + parts: 16 + + # Size of each multipart part. + # Size of each part. Can be a number or MiB/GiB. + # Must be greater than or equal to 5MiB. + part-size: 5MiB + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: diff --git a/yml-samples/put.yml b/yml-samples/put.yml new file mode 100644 index 0000000..d9d8417 --- /dev/null +++ b/yml-samples/put.yml @@ -0,0 +1,181 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: put + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 1m + + # Concurrent operations to run per warp instance. + concurrent: 8 + + # Properties of uploaded objects. + obj: + # Size of each uploaded object + size: 100KiB + + # Randomize the size of each object within certain constraints. + # See https://github.com/minio/warp?tab=readme-ov-file#random-file-sizes + rand-size: false + + # Force specific size of each multipart part. + # Must be '5MB' or bigger. + part-size: + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: diff --git a/yml-samples/stat.yml b/yml-samples/stat.yml new file mode 100644 index 0000000..dd12332 --- /dev/null +++ b/yml-samples/stat.yml @@ -0,0 +1,188 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: stat + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 1m + + # Concurrent operations to run per warp instance. + concurrent: 16 + + # The number of objects to upload before starting the benchmark. + # Upload enough objects to ensure that any remote caching is bypassed. + objects: 5000 + + # Properties of uploaded objects. + obj: + # Size of each uploaded object + size: 1B + + # Number of versions to upload of each object + versions: 2 + + # Randomize the size of each object within certain constraints. + # See https://github.com/minio/warp?tab=readme-ov-file#random-file-sizes + rand-size: false + + # Force specific size of each multipart part. + # Must be '5MB' or bigger. + part-size: + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: diff --git a/yml-samples/versioned.yml b/yml-samples/versioned.yml new file mode 100644 index 0000000..5cd4ac5 --- /dev/null +++ b/yml-samples/versioned.yml @@ -0,0 +1,193 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: versioned + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 1m + + # Concurrent operations to run per warp instance. + concurrent: 8 + + # The number of objects to upload before starting the benchmark. + # Upload enough objects to ensure that any remote caching is bypassed. + objects: 1000 + + # Adjust the distribution of each operation type + # The final distribution will be determined by the fraction of each value of the total. + distribution: + get: 45.0 + stat: 30.0 + put: 15.0 + delete: 10.0 # Must be same or lower than 'put'. + + # Properties of uploaded objects. + obj: + # Size of each uploaded object + size: 100KiB + + # Randomize the size of each object within certain constraints. + # See https://github.com/minio/warp?tab=readme-ov-file#random-file-sizes + rand-size: false + + # Force specific size of each multipart part. + # Must be '5MB' or bigger. + part-size: + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: diff --git a/yml-samples/zip.yml b/yml-samples/zip.yml new file mode 100644 index 0000000..12d9637 --- /dev/null +++ b/yml-samples/zip.yml @@ -0,0 +1,180 @@ +warp: + api: v1 + + # Benchmark to run. + # Corresponds to warp [benchmark] command. + benchmark: zip + + # Do not print any output. + quiet: false + + # Disable terminal color output. + no-color: false + + # Print results and errors as JSON. + json: false + + # Output benchmark+profile data to this file. + # By default a unique filename is generated. + bench-data: + + # Connect to warp clients and run benchmarks there. + # See https://github.com/minio/warp?tab=readme-ov-file#distributed-benchmarking + # Can be a single value or a list. + warp-client: + + # Run MinIO server profiling during benchmark; + # possible values are 'cpu', 'cpuio', 'mem', 'block', 'mutex', 'threads' and 'trace'. + # Can be single value or a list. + server-profile: + + # Remote host parameters and connection info. + remote: + # Specify custom region + region: us-east-1 + + # Access key and Secret key + access-key: 'Q3AM3UQ867SPQQA43P2F' + secret-key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' + + # Specify one or more hosts. + # The benchmark will be run against all hosts concurrently. + # Multiple servers can be specified with elipsis notation; + # for example '10.0.0.{1...10}:9000' specifies 10 hosts. + # See more at https://github.com/minio/warp?tab=readme-ov-file#multiple-hosts + host: + - 'play.min.io' + + # Use TLS for calls. + tls: true + + # Allow TLS with unverified certificates. + insecure: false + + # Stream benchmark statistics to Influx DB instance. + # See more at https://github.com/minio/warp?tab=readme-ov-file#influxdb-output + influxdb: '' + + # Bucket to use for benchmark data. + # + # CAREFUL: ALL DATA WILL BE DELETED IN BUCKET! + # + # By default, 'warp-benchmark-bucket' will be created or used. + bucket: + + # params specifies the benchmark parameters. + # The fields here depend on the benchmark type. + params: + # Duration to run the benchmark. + # Use 's' and 'm' to specify seconds and minutes. + duration: 1m + + # Concurrent operations to run per warp instance. + concurrent: 8 + + # Number of files to upload in the zip file + files: 10000 + + # Properties of uploaded objects. + obj: + # Size of each uploaded object inside the zip file. + size: 1KiB + + # Randomize the size of each object within certain constraints. + # See https://github.com/minio/warp?tab=readme-ov-file#random-file-sizes + rand-size: false + + # Use automatic termination when traffic stabilizes. + # Can not be used with distributed warp setup. + # See https://github.com/minio/warp?tab=readme-ov-file#automatic-termination + autoterm: + enabled: false + dur: 10s + pct: 7.5 + + # Do not clear bucket before or after running benchmarks. + no-clear: false + + # Leave benchmark data. Do not run cleanup after benchmark. + # Bucket will still be cleaned prior to benchmark. + keep-data: false + + + # The io section specifies custom IO properties for uploaded objects. + io: + # Use a custom prefix + prefix: + + # Do not use separate prefix for each thread + no-prefix: false + + # Add MD5 sum to uploads + md5: false + + # Disable multipart uploads + disable-multipart: false + + # Disable calculating sha256 on client side for uploads + disable-sha256-payload: false + + # Server-side sse-s3 encrypt/decrypt objects + sse-s3-encrypt: false + + # Encrypt/decrypt objects (using server-side encryption with random keys) + sse-c-encrypt: false + + # Override storage class. + # Default storage class will be used unless specified. + storage-class: + + analyze: + # Display additional analysis data. + verbose: false + # Only output for this host. + host: '' + # Only output for this operation. Can be 'GET', 'PUT', 'DELETE', etc. + filter-op: '' + # Split analysis into durations of this length. + # Can be '1s', '5s', '1m', etc. + segment-duration: + # Output aggregated data as to file. + out: + # Additional time duration to skip when analyzing data. + skip-duration: + # Max operations to load for analysis. + limit: + # Skip this number of operations before starting analysis. + offset: + + advanced: + # Stress test only and discard output. + stress: false + + # Print requests. + debug: false + + # Disable HTTP Keep-Alive + disable-http-keepalive: false + + # Enable HTTP2 support if server supports it + http2: false + + # Rate limit each instance to this number of requests per second + rps-limit: + + # Host selection algorithm. + # Can be 'weighed' or 'roundrobin' + host-select: weighed + + # "Resolve the host(s) ip(s) (including multiple A/AAAA records). + # This can break SSL certificates, use --insecure if so + resolve-host: false + + # Specify custom write socket buffer size in bytes + sndbuf: 32768 + + # Specify custom read socket buffer size in bytes + rcvbuf: 32768 + + # When running benchmarks open a webserver to fetch results remotely, eg: localhost:7762 + serve: From 22eed4761de3fe73bed25d1538d591fd68734a9e Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 5 Aug 2024 08:28:52 -0700 Subject: [PATCH 3/4] multipart: Respect disable-sha256-payload on upload (#331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When preparing, forward the `--disable-sha256-payload` parameter. Requires https://github.com/minio/minio-go/pull/1988 Fixes #329 Without `--disable-sha256-payload` on non TLS host: ``` 127.0.0.1:9001 [REQUEST s3.PutObjectPart] [2024-08-05T11:38:38.387] [Client IP: 127.0.0.1] 127.0.0.1:9001 PUT /warp-benchmark-bucket/warp-multipart.bin?partNumber=8&uploadId=YTQ0NDc3MTktNWViMS00YWY4LWI0NmMtYTgyMjMyYzE5YjAyLjlhNjU3OWVlLWY3ODctNGZhMi1hMTBmLTkyNDIwOWU0NDMxZngxNzIyODUwNzE4MzcxOTA1MjAw 127.0.0.1:9001 Proto: HTTP/1.1 127.0.0.1:9001 Host: 127.0.0.1:9001 127.0.0.1:9001 User-Agent: MinIO (windows; amd64) minio-go/v7.0.75 warp/(dev) 127.0.0.1:9001 X-Amz-Content-Sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD 127.0.0.1:9001 X-Amz-Date: 20240805T093838Z 127.0.0.1:9001 X-Amz-Decoded-Content-Length: 5242880 127.0.0.1:9001 Authorization: AWS4-HMAC-SHA256 Credential=minio/20240805/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=75c66253396134912f0ba16503254c07c3485fc89d4c08c14be16f4d20742071 127.0.0.1:9001 Content-Length: 5250166 127.0.0.1:9001 127.0.0.1:9001 [RESPONSE] [2024-08-05T11:38:38.500] [ Duration 112.643ms TTFB 0s ↑ 5.0 MiB ↓ 0 B ] 127.0.0.1:9001 200 OK ``` With `--disable-sha256-payload` on non TLS host: ``` 127.0.0.1:9001 [REQUEST s3.PutObjectPart] [2024-08-05T11:40:02.698] [Client IP: 127.0.0.1] 127.0.0.1:9001 PUT /warp-benchmark-bucket/warp-multipart.bin?partNumber=6&uploadId=YTQ0NDc3MTktNWViMS00YWY4LWI0NmMtYTgyMjMyYzE5YjAyLmJlZGNjZjc2LWJjZWEtNDBhOC05YmU3LTE5YjFiYThhNTRjNHgxNzIyODUwODAyNjgyNzk2OTAw 127.0.0.1:9001 Proto: HTTP/1.1 127.0.0.1:9001 Host: 127.0.0.1:9001 127.0.0.1:9001 X-Amz-Content-Sha256: UNSIGNED-PAYLOAD 127.0.0.1:9001 X-Amz-Date: 20240805T094002Z 127.0.0.1:9001 Authorization: AWS4-HMAC-SHA256 Credential=minio/20240805/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=378eeccc3e2e530193ebe31015aa1b58cd47f57bc611064051c50e768c8ca949 127.0.0.1:9001 Content-Length: 5242880 127.0.0.1:9001 User-Agent: MinIO (windows; amd64) minio-go/v7.0.75 warp/(dev) 127.0.0.1:9001 127.0.0.1:9001 [RESPONSE] [2024-08-05T11:40:02.787] [ Duration 88.98ms TTFB 0s ↑ 5.0 MiB ↓ 0 B ] 127.0.0.1:9001 200 OK ``` --- go.mod | 18 ++++++++++-------- go.sum | 35 ++++++++++++++++++----------------- pkg/bench/multipart.go | 3 ++- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 23a100c..2b59f2b 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.17.0 github.com/influxdata/influxdb-client-go/v2 v2.13.0 - github.com/klauspost/compress v1.17.8 + github.com/klauspost/compress v1.17.9 github.com/minio/cli v1.24.2 github.com/minio/madmin-go/v3 v3.0.51 github.com/minio/mc v0.0.0-20240513163111-b46cf3c417f6 @@ -18,7 +18,7 @@ require ( github.com/minio/websocket v1.6.0 github.com/posener/complete v1.2.3 github.com/secure-io/sio-go v0.3.1 - golang.org/x/net v0.25.0 + golang.org/x/net v0.26.0 golang.org/x/time v0.5.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -26,15 +26,16 @@ require ( require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/goccy/go-json v0.10.3 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -64,10 +65,11 @@ require ( github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - golang.org/x/crypto v0.23.0 // indirect + golang.org/x/crypto v0.24.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.34.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect ) + +replace github.com/minio/minio-go/v7 => github.com/klauspost/minio-go/v7 v7.0.0-20240805091306-15b5b5441312 diff --git a/go.sum b/go.sum index 998b4ed..e9c0926 100644 --- a/go.sum +++ b/go.sum @@ -19,11 +19,14 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -46,11 +49,13 @@ github.com/influxdata/influxdb-client-go/v2 v2.13.0/go.mod h1:k+spCbt9hcvqvUiz0s github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/minio-go/v7 v7.0.0-20240805091306-15b5b5441312 h1:x62ZNK7R4pHRVgmAX8hRvzwImwF03hmTRrbd4Unmjnc= +github.com/klauspost/minio-go/v7 v7.0.0-20240805091306-15b5b5441312/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -92,8 +97,6 @@ github.com/minio/mc v0.0.0-20240513163111-b46cf3c417f6 h1:/ILVfsE2Xz9F1oi5oCPgLe github.com/minio/mc v0.0.0-20240513163111-b46cf3c417f6/go.mod h1:AHsPnIxIslIZ4jlX27Hq/ut5FQ7kpxLavdt3Q0XGpT8= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g= -github.com/minio/minio-go/v7 v7.0.70/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA= github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ= github.com/minio/pkg/v2 v2.0.19 h1:r187/k/oVH9H0DDwvLY5WipkJaZ4CLd4KI3KgIUExR0= @@ -170,8 +173,8 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -181,8 +184,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -210,8 +213,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -224,8 +227,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -239,8 +242,6 @@ google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/bench/multipart.go b/pkg/bench/multipart.go index 6906570..6dade97 100644 --- a/pkg/bench/multipart.go +++ b/pkg/bench/multipart.go @@ -129,7 +129,8 @@ func (g *Multipart) Prepare(ctx context.Context) error { opts.ContentType = obj.ContentType mpopts := minio.PutObjectPartOptions{ - SSE: g.Common.PutOpts.ServerSideEncryption, + SSE: g.Common.PutOpts.ServerSideEncryption, + DisableContentSha256: g.PutOpts.DisableContentSha256, } op.Start = time.Now() res, err := core.PutObjectPart(ctx, g.Bucket, obj.Name, g.UploadID, partN, obj.Reader, obj.Size, mpopts) From 61422c5e280c6a9ea95507cf6d25c65bc8b4fd8a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 5 Aug 2024 09:03:01 -0700 Subject: [PATCH 4/4] update go.mod pointing to minio-go/v7 repo --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2b59f2b..fcb13eb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/minio/madmin-go/v3 v3.0.51 github.com/minio/mc v0.0.0-20240513163111-b46cf3c417f6 github.com/minio/md5-simd v1.1.2 - github.com/minio/minio-go/v7 v7.0.70 + github.com/minio/minio-go/v7 v7.0.75-0.20240805152911-fd0e50784915 github.com/minio/pkg/v2 v2.0.19 github.com/minio/websocket v1.6.0 github.com/posener/complete v1.2.3 @@ -71,5 +71,3 @@ require ( golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.34.1 // indirect ) - -replace github.com/minio/minio-go/v7 => github.com/klauspost/minio-go/v7 v7.0.0-20240805091306-15b5b5441312 diff --git a/go.sum b/go.sum index e9c0926..0e86b52 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,6 @@ github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/minio-go/v7 v7.0.0-20240805091306-15b5b5441312 h1:x62ZNK7R4pHRVgmAX8hRvzwImwF03hmTRrbd4Unmjnc= -github.com/klauspost/minio-go/v7 v7.0.0-20240805091306-15b5b5441312/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -97,6 +95,8 @@ github.com/minio/mc v0.0.0-20240513163111-b46cf3c417f6 h1:/ILVfsE2Xz9F1oi5oCPgLe github.com/minio/mc v0.0.0-20240513163111-b46cf3c417f6/go.mod h1:AHsPnIxIslIZ4jlX27Hq/ut5FQ7kpxLavdt3Q0XGpT8= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.75-0.20240805152911-fd0e50784915 h1:qRjzKsHt8d+gwgHFKYkbHefvp+DzL8LZi1BN0DvOLXs= +github.com/minio/minio-go/v7 v7.0.75-0.20240805152911-fd0e50784915/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA= github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ= github.com/minio/pkg/v2 v2.0.19 h1:r187/k/oVH9H0DDwvLY5WipkJaZ4CLd4KI3KgIUExR0=