From 0a4da548b2021fc734bfa58d15d18d450a37754d Mon Sep 17 00:00:00 2001 From: shuwenwei <55970239+shuwenwei@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:27:17 +0800 Subject: [PATCH] Fix fillMeasurements NPE (#313) * fix npe * modify bitmap size calculation * fix example --- .../java/org/apache/tsfile/utils/BitMap.java | 14 ++-- .../java/org/apache/tsfile/TsFileRead.java | 7 +- .../TsFileWriteAlignedWithTSRecord.java | 6 +- .../tsfile/TsFileWriteWithTSRecord.java | 6 +- .../apache/tsfile/TsFileWriteWithTablet.java | 13 ++-- .../block/SingleDeviceTsBlockReader.java | 46 ++++++------ .../tsfile/read/query/ResultSetTest.java | 70 +++++++++++++++++++ .../org/apache/tsfile/utils/BitMapTest.java | 4 +- 8 files changed, 125 insertions(+), 41 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/utils/BitMap.java b/java/common/src/main/java/org/apache/tsfile/utils/BitMap.java index 61fc4b56b..8a4c10b4e 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/BitMap.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/BitMap.java @@ -42,7 +42,7 @@ public class BitMap { /** Initialize a BitMap with given size. */ public BitMap(int size) { this.size = size; - bits = new byte[size / Byte.SIZE + 1]; + bits = new byte[getSizeOfBytes(size)]; Arrays.fill(bits, (byte) 0); } @@ -245,11 +245,17 @@ public BitMap getRegion(int positionOffset, int length) { return newBitMap; } - public int getTruncatedSize(int size) { - return size / Byte.SIZE + (size % Byte.SIZE == 0 ? 0 : 1); + public static int getSizeOfBytes(int size) { + // Regardless of whether it is divisible here, add 1 byte. + // Should not modify this place, as many codes are already using the same method to calculate + // bitmap size. + // Precise calculation of size may cause those codes to throw IndexOutOfBounds or + // BufferUnderFlow + // exceptions. + return size / Byte.SIZE + 1; } public byte[] getTruncatedByteArray(int size) { - return Arrays.copyOf(this.bits, getTruncatedSize(size)); + return Arrays.copyOf(this.bits, getSizeOfBytes(size)); } } diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileRead.java b/java/examples/src/main/java/org/apache/tsfile/TsFileRead.java index 64ef6f99e..5b520fc6c 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileRead.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileRead.java @@ -19,6 +19,7 @@ package org.apache.tsfile; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TsFileReader; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Path; @@ -97,7 +98,8 @@ public static void main(String[] args) throws IOException { // value filter : device_1.sensor_2 <= 20, should select 1 2 4 6 7 IExpression valueFilter = - new SingleSeriesExpression(new Path(DEVICE_1, SENSOR_2, true), ValueFilterApi.ltEq(20L)); + new SingleSeriesExpression( + new Path(DEVICE_1, SENSOR_2, true), ValueFilterApi.ltEq(1, 20L, TSDataType.INT64)); queryAndPrint(paths, readTsFile, valueFilter); // time filter : 4 <= time <= 10, value filter : device_1.sensor_3 >= 20, should select 4 7 8 @@ -106,7 +108,8 @@ public static void main(String[] args) throws IOException { new GlobalTimeExpression(TimeFilterApi.gtEq(4L)), new GlobalTimeExpression(TimeFilterApi.ltEq(10L))); valueFilter = - new SingleSeriesExpression(new Path(DEVICE_1, SENSOR_3, true), ValueFilterApi.gtEq(20L)); + new SingleSeriesExpression( + new Path(DEVICE_1, SENSOR_3, true), ValueFilterApi.gtEq(2, 20L, TSDataType.INT64)); IExpression finalFilter = BinaryExpression.and(timeFilter, valueFilter); queryAndPrint(paths, readTsFile, finalFilter); } diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java index f7d1bc7b4..3e4c74486 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java @@ -91,18 +91,18 @@ private static void writeAligned( throws IOException, WriteProcessException { for (long time = startTime; time < rowSize + startTime; time++) { // construct TsRecord - TSRecord tsRecord = new TSRecord(time, deviceId); + TSRecord tsRecord = new TSRecord(deviceId, time); for (IMeasurementSchema schema : schemas) { tsRecord.addTuple( DataPoint.getDataPoint( schema.getType(), - schema.getMeasurementId(), + schema.getMeasurementName(), Objects.requireNonNull(DataGenerator.generate(schema.getType(), (int) startValue)) .toString())); startValue++; } // write - tsFileWriter.writeAligned(tsRecord); + tsFileWriter.writeRecord(tsRecord); } } } diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java index 79f9fd527..42e244247 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java @@ -87,18 +87,18 @@ private static void write( throws IOException, WriteProcessException { for (long time = startTime; time < rowSize + startTime; time++) { // construct TsRecord - TSRecord tsRecord = new TSRecord(time, deviceId); + TSRecord tsRecord = new TSRecord(deviceId, time); for (IMeasurementSchema schema : schemas) { tsRecord.addTuple( DataPoint.getDataPoint( schema.getType(), - schema.getMeasurementId(), + schema.getMeasurementName(), Objects.requireNonNull(DataGenerator.generate(schema.getType(), (int) startValue)) .toString())); startValue++; } // write - tsFileWriter.write(tsRecord); + tsFileWriter.writeRecord(tsRecord); } } } diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java index 3ab95a187..bb0293a52 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java @@ -26,6 +26,7 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; @@ -91,23 +92,23 @@ private static void writeWithTablet( long sensorNum = schemas.size(); for (long r = 0; r < rowNum; r++, startValue++) { - int row = tablet.rowSize++; + int row = tablet.getRowSize(); timestamps[row] = startTime++; for (int i = 0; i < sensorNum; i++) { tablet.addValue( - schemas.get(i).getMeasurementId(), + schemas.get(i).getMeasurementName(), row, DataGenerator.generate(schemas.get(i).getType(), (int) r)); } // write - if (tablet.rowSize == tablet.getMaxRowNumber()) { - tsFileWriter.write(tablet); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + tsFileWriter.writeTree(tablet); tablet.reset(); } } // write - if (tablet.rowSize != 0) { - tsFileWriter.write(tablet); + if (tablet.getRowSize() != 0) { + tsFileWriter.writeTree(tablet); tablet.reset(); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java index a1fc5b29b..6c7a43b62 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java @@ -369,27 +369,31 @@ void fillInto(TsBlock block, int blockRowNum) { final TsPrimitiveType value = vector[i]; final List columnPositions = posInResult.get(i); for (Integer pos : columnPositions) { - switch (value.getDataType()) { - case TEXT: - block.getColumn(pos).getBinaries()[blockRowNum] = value.getBinary(); - break; - case INT32: - block.getColumn(pos).getInts()[blockRowNum] = value.getInt(); - break; - case INT64: - block.getColumn(pos).getLongs()[blockRowNum] = value.getLong(); - break; - case BOOLEAN: - block.getColumn(pos).getBooleans()[blockRowNum] = value.getBoolean(); - break; - case FLOAT: - block.getColumn(pos).getFloats()[blockRowNum] = value.getFloat(); - break; - case DOUBLE: - block.getColumn(pos).getDoubles()[blockRowNum] = value.getDouble(); - break; - default: - throw new IllegalArgumentException("Unsupported data type: " + value.getDataType()); + if (value != null) { + switch (value.getDataType()) { + case TEXT: + block.getColumn(pos).getBinaries()[blockRowNum] = value.getBinary(); + break; + case INT32: + block.getColumn(pos).getInts()[blockRowNum] = value.getInt(); + break; + case INT64: + block.getColumn(pos).getLongs()[blockRowNum] = value.getLong(); + break; + case BOOLEAN: + block.getColumn(pos).getBooleans()[blockRowNum] = value.getBoolean(); + break; + case FLOAT: + block.getColumn(pos).getFloats()[blockRowNum] = value.getFloat(); + break; + case DOUBLE: + block.getColumn(pos).getDoubles()[blockRowNum] = value.getDouble(); + break; + default: + throw new IllegalArgumentException("Unsupported data type: " + value.getDataType()); + } + } else { + block.getColumn(pos).setNull(blockRowNum, blockRowNum + 1); } block.getColumn(pos).setPositionCount(blockRowNum + 1); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java index 2bc65d200..bf8273bbd 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java @@ -137,4 +137,74 @@ public void testQueryTable() throws Exception { Assert.assertTrue(resultSet.isNull(5)); } } + + @Test + public void testQueryTableWithPartialNullValueInChunk() throws Exception { + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new MeasurementSchema("id1", TSDataType.STRING), + new MeasurementSchema("id2", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.BOOLEAN)), + Arrays.asList( + Tablet.ColumnCategory.ID, + Tablet.ColumnCategory.ID, + Tablet.ColumnCategory.MEASUREMENT, + Tablet.ColumnCategory.MEASUREMENT)); + Tablet tablet = + new Tablet( + Arrays.asList("id1", "id2", "s1", "s2"), + Arrays.asList( + TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN), + 1024); + tablet.addTimestamp(0, 0); + tablet.addValue("id1", 0, "id_field1"); + tablet.addValue("id2", 0, "id_field2"); + tablet.addValue("s1", 0, true); + tablet.addValue("s2", 0, false); + + tablet.addTimestamp(1, 1); + tablet.addValue("id1", 1, "id_field1"); + tablet.addValue("id2", 1, "id_field2"); + tablet.addValue("s2", 1, false); + + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(tsfile).tableSchema(tableSchema).build()) { + writer.write(tablet); + } + + try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile); + ResultSet resultSet = + tsFileReader.query("t1", Arrays.asList("id1", "id2", "s2", "s1"), 0, 2); ) { + // id1 id2 s2 s1 + ResultSetMetadata resultSetMetadata = resultSet.getMetadata(); + // Time id1 id2 s2 s1 + Assert.assertEquals("Time", resultSetMetadata.getColumnName(1)); + Assert.assertEquals(TSDataType.INT64, resultSetMetadata.getColumnType(1)); + Assert.assertEquals("id1", resultSetMetadata.getColumnName(2)); + Assert.assertEquals(TSDataType.STRING, resultSetMetadata.getColumnType(2)); + Assert.assertEquals("id2", resultSetMetadata.getColumnName(3)); + Assert.assertEquals(TSDataType.STRING, resultSetMetadata.getColumnType(3)); + Assert.assertEquals("s2", resultSetMetadata.getColumnName(4)); + Assert.assertEquals(TSDataType.BOOLEAN, resultSetMetadata.getColumnType(4)); + Assert.assertEquals("s1", resultSetMetadata.getColumnName(5)); + Assert.assertEquals(TSDataType.BOOLEAN, resultSetMetadata.getColumnType(5)); + + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(0, resultSet.getLong(1)); + Assert.assertEquals("id_field1", resultSet.getString(2)); + Assert.assertEquals("id_field2", resultSet.getString(3)); + Assert.assertFalse(resultSet.getBoolean(4)); + Assert.assertTrue(resultSet.getBoolean(5)); + + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(1, resultSet.getLong(1)); + Assert.assertEquals("id_field1", resultSet.getString(2)); + Assert.assertEquals("id_field2", resultSet.getString(3)); + Assert.assertTrue(resultSet.isNull("s1")); + Assert.assertFalse(resultSet.getBoolean("s2")); + } + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/BitMapTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/BitMapTest.java index 03b3630dc..3aca8e60e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/BitMapTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/BitMapTest.java @@ -90,7 +90,7 @@ public void testIsAllUnmarkedInRange() { public void testGetTruncatedByteArray() { BitMap bitMap = new BitMap(16); assertArrayEquals(new byte[2], bitMap.getTruncatedByteArray(13)); - assertArrayEquals(new byte[2], bitMap.getTruncatedByteArray(16)); + assertArrayEquals(new byte[3], bitMap.getTruncatedByteArray(16)); bitMap.mark(3); byte[] truncatedArray = bitMap.getTruncatedByteArray(12); @@ -100,7 +100,7 @@ public void testGetTruncatedByteArray() { assertEquals((byte) 0b00000000, truncatedArray[1]); truncatedArray = bitMap.getTruncatedByteArray(8); - assertEquals(1, truncatedArray.length); + assertEquals(2, truncatedArray.length); assertEquals((byte) 0b00001000, truncatedArray[0]); }