From 8729c8cbba2b0c1128f3dc090b2603926751ce97 Mon Sep 17 00:00:00 2001 From: Marc Lopez Rubio Date: Wed, 20 Mar 2024 08:49:48 +0100 Subject: [PATCH] Replace `compress/gzip` with `klauspost/compress` (#139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the stdlib `compress/gzip` with `klauspost/compress/gzip`. It is slightly faster (by using less CPU) and results in the same allocs. That is assuming default gzip compression (6). Using an extraneous application to benchmark the change, I was able to observe `10-11%` more throughput (measured in events per second) and lower CPU usage. Using the same container with 2vCPU set and GOMAXPROCS=2 (200% max): - baseline: 39-40K eps - 186%-190% CPU Usage. - klauspost: 43-44.5K eps - 159% CPU usage. - `+10-11%` throughput. - `-15%` CPU usage. Microbenchmarks also point show decreased ns/op on my machine, although it's not a perfect benchmark, since it's not entirely idle, but there's improvements, even for benchmarks with a higher p. ``` $ benchstat main.txt klauspost.txt goos: darwin goarch: arm64 pkg: github.com/elastic/go-docappender │ main.txt │ klauspost.txt │ │ sec/op │ sec/op vs base │ Appender/NoCompression-4 394.8n ± 11% 389.4n ± 17% ~ (p=0.084 n=6) Appender/NoCompressionScaling-4 353.8n ± 13% 388.2n ± 11% ~ (p=0.589 n=6) Appender/BestSpeed-4 654.5n ± 36% 572.4n ± 33% ~ (p=0.065 n=6) Appender/BestSpeedScaling-4 546.7n ± 25% 577.8n ± 30% ~ (p=0.818 n=6) Appender/DefaultCompression-4 1143.5n ± 18% 615.1n ± 43% -46.21% (p=0.002 n=6) Appender/DefaultCompressionScaling-4 997.9n ± 16% 640.7n ± 33% -35.80% (p=0.002 n=6) Appender/BestCompression-4 1.064µ ± 9% 1.098µ ± 15% ~ (p=0.965 n=6) Appender/BestCompressionScaling-4 1.056µ ± 9% 1.014µ ± 12% ~ (p=0.368 n=6) AppenderError-4 996.3n ± 1% 997.3n ± 1% ~ (p=0.937 n=6) geomean 736.6n 654.0n -11.22% ``` --------- Signed-off-by: Marc Lopez Rubio --- bulk_indexer.go | 2 +- go.mod | 1 + go.sum | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bulk_indexer.go b/bulk_indexer.go index 249c5c1..4d55bb2 100644 --- a/bulk_indexer.go +++ b/bulk_indexer.go @@ -19,7 +19,6 @@ package docappender import ( "bytes" - "compress/gzip" "context" "fmt" "io" @@ -28,6 +27,7 @@ import ( "strings" "unsafe" + "github.com/klauspost/compress/gzip" "go.elastic.co/fastjson" "github.com/elastic/go-elasticsearch/v8" diff --git a/go.mod b/go.mod index b915905..222834b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/elastic/go-elasticsearch/v8 v8.12.1 github.com/json-iterator/go v1.1.12 + github.com/klauspost/compress v1.17.7 github.com/stretchr/testify v1.9.0 go.elastic.co/apm/module/apmelasticsearch/v2 v2.5.0 go.elastic.co/apm/module/apmzap/v2 v2.5.0 diff --git a/go.sum b/go.sum index dd8982a..2d2f632 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=