Skip to content

Commit

Permalink
Fix fillMeasurements NPE (#313)
Browse files Browse the repository at this point in the history 
* fix npe

* modify bitmap size calculation

* fix example
  • Loading branch information
@shuwenwei
shuwenwei authored Nov 28, 2024
1 parent 027c7c3 commit 0a4da54
Show file tree
Hide file tree
Showing 8 changed files with 125 additions and 41 deletions.
14 changes: 10 additions & 4 deletions java/common/src/main/java/org/apache/tsfile/utils/BitMap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public class BitMap {
/** Initialize a BitMap with given size. */
public BitMap(int size) {
this.size = size;
bits = new byte[size / Byte.SIZE + 1];
bits = new byte[getSizeOfBytes(size)];
Arrays.fill(bits, (byte) 0);
}

Expand Down Expand Up @@ -245,11 +245,17 @@ public BitMap getRegion(int positionOffset, int length) {
return newBitMap;
}

public int getTruncatedSize(int size) {
return size / Byte.SIZE + (size % Byte.SIZE == 0 ? 0 : 1);
public static int getSizeOfBytes(int size) {
// Regardless of whether it is divisible here, add 1 byte.
// Should not modify this place, as many codes are already using the same method to calculate
// bitmap size.
// Precise calculation of size may cause those codes to throw IndexOutOfBounds or
// BufferUnderFlow
// exceptions.
return size / Byte.SIZE + 1;
}

public byte[] getTruncatedByteArray(int size) {
return Arrays.copyOf(this.bits, getTruncatedSize(size));
return Arrays.copyOf(this.bits, getSizeOfBytes(size));
}
}
7 changes: 5 additions & 2 deletions java/examples/src/main/java/org/apache/tsfile/TsFileRead.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.tsfile;

import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.TsFileReader;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.read.common.Path;
Expand Down Expand Up @@ -97,7 +98,8 @@ public static void main(String[] args) throws IOException {

// value filter : device_1.sensor_2 <= 20, should select 1 2 4 6 7
IExpression valueFilter =
new SingleSeriesExpression(new Path(DEVICE_1, SENSOR_2, true), ValueFilterApi.ltEq(20L));
new SingleSeriesExpression(
new Path(DEVICE_1, SENSOR_2, true), ValueFilterApi.ltEq(1, 20L, TSDataType.INT64));
queryAndPrint(paths, readTsFile, valueFilter);

// time filter : 4 <= time <= 10, value filter : device_1.sensor_3 >= 20, should select 4 7 8
Expand All @@ -106,7 +108,8 @@ public static void main(String[] args) throws IOException {
new GlobalTimeExpression(TimeFilterApi.gtEq(4L)),
new GlobalTimeExpression(TimeFilterApi.ltEq(10L)));
valueFilter =
new SingleSeriesExpression(new Path(DEVICE_1, SENSOR_3, true), ValueFilterApi.gtEq(20L));
new SingleSeriesExpression(
new Path(DEVICE_1, SENSOR_3, true), ValueFilterApi.gtEq(2, 20L, TSDataType.INT64));
IExpression finalFilter = BinaryExpression.and(timeFilter, valueFilter);
queryAndPrint(paths, readTsFile, finalFilter);
}
Expand Down
6 changes: 3 additions & 3 deletions java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,18 +91,18 @@ private static void writeAligned(
throws IOException, WriteProcessException {
for (long time = startTime; time < rowSize + startTime; time++) {
// construct TsRecord
TSRecord tsRecord = new TSRecord(time, deviceId);
TSRecord tsRecord = new TSRecord(deviceId, time);
for (IMeasurementSchema schema : schemas) {
tsRecord.addTuple(
DataPoint.getDataPoint(
schema.getType(),
schema.getMeasurementId(),
schema.getMeasurementName(),
Objects.requireNonNull(DataGenerator.generate(schema.getType(), (int) startValue))
.toString()));
startValue++;
}
// write
tsFileWriter.writeAligned(tsRecord);
tsFileWriter.writeRecord(tsRecord);
}
}
}
6 changes: 3 additions & 3 deletions java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,18 +87,18 @@ private static void write(
throws IOException, WriteProcessException {
for (long time = startTime; time < rowSize + startTime; time++) {
// construct TsRecord
TSRecord tsRecord = new TSRecord(time, deviceId);
TSRecord tsRecord = new TSRecord(deviceId, time);
for (IMeasurementSchema schema : schemas) {
tsRecord.addTuple(
DataPoint.getDataPoint(
schema.getType(),
schema.getMeasurementId(),
schema.getMeasurementName(),
Objects.requireNonNull(DataGenerator.generate(schema.getType(), (int) startValue))
.toString()));
startValue++;
}
// write
tsFileWriter.write(tsRecord);
tsFileWriter.writeRecord(tsRecord);
}
}
}
13 changes: 7 additions & 6 deletions java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;

import org.slf4j.Logger;
Expand Down Expand Up @@ -91,23 +92,23 @@ private static void writeWithTablet(
long sensorNum = schemas.size();

for (long r = 0; r < rowNum; r++, startValue++) {
int row = tablet.rowSize++;
int row = tablet.getRowSize();
timestamps[row] = startTime++;
for (int i = 0; i < sensorNum; i++) {
tablet.addValue(
schemas.get(i).getMeasurementId(),
schemas.get(i).getMeasurementName(),
row,
DataGenerator.generate(schemas.get(i).getType(), (int) r));
}
// write
if (tablet.rowSize == tablet.getMaxRowNumber()) {
tsFileWriter.write(tablet);
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
tsFileWriter.writeTree(tablet);
tablet.reset();
}
}
// write
if (tablet.rowSize != 0) {
tsFileWriter.write(tablet);
if (tablet.getRowSize() != 0) {
tsFileWriter.writeTree(tablet);
tablet.reset();
}
}
Expand Down
46 changes: 25 additions & 21 deletions java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,27 +369,31 @@ void fillInto(TsBlock block, int blockRowNum) {
final TsPrimitiveType value = vector[i];
final List<Integer> columnPositions = posInResult.get(i);
for (Integer pos : columnPositions) {
switch (value.getDataType()) {
case TEXT:
block.getColumn(pos).getBinaries()[blockRowNum] = value.getBinary();
break;
case INT32:
block.getColumn(pos).getInts()[blockRowNum] = value.getInt();
break;
case INT64:
block.getColumn(pos).getLongs()[blockRowNum] = value.getLong();
break;
case BOOLEAN:
block.getColumn(pos).getBooleans()[blockRowNum] = value.getBoolean();
break;
case FLOAT:
block.getColumn(pos).getFloats()[blockRowNum] = value.getFloat();
break;
case DOUBLE:
block.getColumn(pos).getDoubles()[blockRowNum] = value.getDouble();
break;
default:
throw new IllegalArgumentException("Unsupported data type: " + value.getDataType());
if (value != null) {
switch (value.getDataType()) {
case TEXT:
block.getColumn(pos).getBinaries()[blockRowNum] = value.getBinary();
break;
case INT32:
block.getColumn(pos).getInts()[blockRowNum] = value.getInt();
break;
case INT64:
block.getColumn(pos).getLongs()[blockRowNum] = value.getLong();
break;
case BOOLEAN:
block.getColumn(pos).getBooleans()[blockRowNum] = value.getBoolean();
break;
case FLOAT:
block.getColumn(pos).getFloats()[blockRowNum] = value.getFloat();
break;
case DOUBLE:
block.getColumn(pos).getDoubles()[blockRowNum] = value.getDouble();
break;
default:
throw new IllegalArgumentException("Unsupported data type: " + value.getDataType());
}
} else {
block.getColumn(pos).setNull(blockRowNum, blockRowNum + 1);
}
block.getColumn(pos).setPositionCount(blockRowNum + 1);
}
Expand Down
70 changes: 70 additions & 0 deletions java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,4 +137,74 @@ public void testQueryTable() throws Exception {
Assert.assertTrue(resultSet.isNull(5));
}
}

@Test
public void testQueryTableWithPartialNullValueInChunk() throws Exception {
TableSchema tableSchema =
new TableSchema(
"t1",
Arrays.asList(
new MeasurementSchema("id1", TSDataType.STRING),
new MeasurementSchema("id2", TSDataType.STRING),
new MeasurementSchema("s1", TSDataType.BOOLEAN),
new MeasurementSchema("s2", TSDataType.BOOLEAN)),
Arrays.asList(
Tablet.ColumnCategory.ID,
Tablet.ColumnCategory.ID,
Tablet.ColumnCategory.MEASUREMENT,
Tablet.ColumnCategory.MEASUREMENT));
Tablet tablet =
new Tablet(
Arrays.asList("id1", "id2", "s1", "s2"),
Arrays.asList(
TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN),
1024);
tablet.addTimestamp(0, 0);
tablet.addValue("id1", 0, "id_field1");
tablet.addValue("id2", 0, "id_field2");
tablet.addValue("s1", 0, true);
tablet.addValue("s2", 0, false);

tablet.addTimestamp(1, 1);
tablet.addValue("id1", 1, "id_field1");
tablet.addValue("id2", 1, "id_field2");
tablet.addValue("s2", 1, false);

try (ITsFileWriter writer =
new TsFileWriterBuilder().file(tsfile).tableSchema(tableSchema).build()) {
writer.write(tablet);
}

try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile);
ResultSet resultSet =
tsFileReader.query("t1", Arrays.asList("id1", "id2", "s2", "s1"), 0, 2); ) {
// id1 id2 s2 s1
ResultSetMetadata resultSetMetadata = resultSet.getMetadata();
// Time id1 id2 s2 s1
Assert.assertEquals("Time", resultSetMetadata.getColumnName(1));
Assert.assertEquals(TSDataType.INT64, resultSetMetadata.getColumnType(1));
Assert.assertEquals("id1", resultSetMetadata.getColumnName(2));
Assert.assertEquals(TSDataType.STRING, resultSetMetadata.getColumnType(2));
Assert.assertEquals("id2", resultSetMetadata.getColumnName(3));
Assert.assertEquals(TSDataType.STRING, resultSetMetadata.getColumnType(3));
Assert.assertEquals("s2", resultSetMetadata.getColumnName(4));
Assert.assertEquals(TSDataType.BOOLEAN, resultSetMetadata.getColumnType(4));
Assert.assertEquals("s1", resultSetMetadata.getColumnName(5));
Assert.assertEquals(TSDataType.BOOLEAN, resultSetMetadata.getColumnType(5));

Assert.assertTrue(resultSet.next());
Assert.assertEquals(0, resultSet.getLong(1));
Assert.assertEquals("id_field1", resultSet.getString(2));
Assert.assertEquals("id_field2", resultSet.getString(3));
Assert.assertFalse(resultSet.getBoolean(4));
Assert.assertTrue(resultSet.getBoolean(5));

Assert.assertTrue(resultSet.next());
Assert.assertEquals(1, resultSet.getLong(1));
Assert.assertEquals("id_field1", resultSet.getString(2));
Assert.assertEquals("id_field2", resultSet.getString(3));
Assert.assertTrue(resultSet.isNull("s1"));
Assert.assertFalse(resultSet.getBoolean("s2"));
}
}
}
4 changes: 2 additions & 2 deletions java/tsfile/src/test/java/org/apache/tsfile/utils/BitMapTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public void testIsAllUnmarkedInRange() {
public void testGetTruncatedByteArray() {
BitMap bitMap = new BitMap(16);
assertArrayEquals(new byte[2], bitMap.getTruncatedByteArray(13));
assertArrayEquals(new byte[2], bitMap.getTruncatedByteArray(16));
assertArrayEquals(new byte[3], bitMap.getTruncatedByteArray(16));

bitMap.mark(3);
byte[] truncatedArray = bitMap.getTruncatedByteArray(12);
Expand All @@ -100,7 +100,7 @@ public void testGetTruncatedByteArray() {
assertEquals((byte) 0b00000000, truncatedArray[1]);

truncatedArray = bitMap.getTruncatedByteArray(8);
assertEquals(1, truncatedArray.length);
assertEquals(2, truncatedArray.length);

assertEquals((byte) 0b00001000, truncatedArray[0]);
}
Expand Down

0 comments on commit 0a4da54

Please sign in to comment.