Skip to content

Commit

Permalink
Use max bytes to allocate WAL compressed buffer (#12928)
Browse files Browse the repository at this point in the history
* use max bytes to allocate WAL compressed buffer

* calculate max bytes with lz4 compressor
  • Loading branch information
shuwenwei authored Jul 15, 2024
1 parent 0fcd638 commit ac110bc
Showing 1 changed file with 9 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALFlushListener;
import org.apache.iotdb.db.utils.MmapUtil;

import org.apache.tsfile.compress.ICompressor;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -149,14 +151,19 @@ private void allocateBuffers() {
try {
workingBuffer = ByteBuffer.allocateDirect(ONE_THIRD_WAL_BUFFER_SIZE);
idleBuffer = ByteBuffer.allocateDirect(ONE_THIRD_WAL_BUFFER_SIZE);
compressedByteBuffer = ByteBuffer.allocateDirect(ONE_THIRD_WAL_BUFFER_SIZE);
compressedByteBuffer =
ByteBuffer.allocateDirect(getCompressedByteBufferSize(ONE_THIRD_WAL_BUFFER_SIZE));
} catch (OutOfMemoryError e) {
logger.error("Fail to allocate wal node-{}'s buffer because out of memory.", identifier, e);
close();
throw e;
}
}

private int getCompressedByteBufferSize(int size) {
return ICompressor.getCompressor(CompressionType.LZ4).getMaxBytesForCompression(size);
}

@Override
protected File rollLogWriter(long searchIndex, WALFileStatus fileStatus) throws IOException {
File file = super.rollLogWriter(searchIndex, fileStatus);
Expand All @@ -175,7 +182,7 @@ public void setBufferSize(int size) {
MmapUtil.clean(compressedByteBuffer);
workingBuffer = ByteBuffer.allocateDirect(capacity);
idleBuffer = ByteBuffer.allocateDirect(capacity);
compressedByteBuffer = ByteBuffer.allocateDirect(capacity);
compressedByteBuffer = ByteBuffer.allocateDirect(getCompressedByteBufferSize(capacity));
currentWALFileWriter.setCompressedByteBuffer(compressedByteBuffer);
} catch (OutOfMemoryError e) {
logger.error("Fail to allocate wal node-{}'s buffer because out of memory.", identifier, e);
Expand Down

0 comments on commit ac110bc

Please sign in to comment.