Merge pull request #142 from ClickHouse/feat/zstd-reduce-memory-consu…

…mption feat(compress): reduce ZSTD encoder memory consumption
ClickHouse · Jun 27, 2022 · 3b0bd0d · 3b0bd0d
2 parents 92362cb + 0647f4d
commit 3b0bd0d
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 2 deletions.
diff --git a/internal/compress/reader.go b/internal/compress/reader.go
@@ -85,7 +85,10 @@ func (r *Reader) readBlock() error {
 		if r.zstd == nil {
 			// Lazily initializing to prevent spawning goroutines in NewReader.
 			// See https://github.com/golang/go/issues/47056#issuecomment-997436820
-			zstdReader, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1))
+			zstdReader, err := zstd.NewReader(nil,
+				zstd.WithDecoderConcurrency(1),
+				zstd.WithDecoderLowmem(true),
+			)
 			if err != nil {
 				return errors.Wrap(err, "zstd")
 			}

diff --git a/internal/compress/writer.go b/internal/compress/writer.go
@@ -56,8 +56,16 @@ func (w *Writer) Compress(m Method, buf []byte) error {
 }
 
 func NewWriter() *Writer {
+	w, err := zstd.NewWriter(nil,
+		zstd.WithEncoderLevel(zstd.SpeedDefault),
+		zstd.WithEncoderConcurrency(1),
+		zstd.WithLowerEncoderMem(true),
+	)
+	if err != nil {
+		panic(err)
+	}
 	return &Writer{
 		lz4:  &lz4.Compressor{},
-		zstd: &zstd.Encoder{},
+		zstd: w,
 	}
 }