Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Column#getTsDataType return actual data type #314

Closed
wants to merge 1 commit into from
Closed

Column#getTsDataType return actual data type #314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,13 +116,17 @@ private TsBlockBuilder(int initialExpectedEntries, int maxTsBlockBytes, List<TSD
case DATE:
valueColumnBuilders[i] =
new IntColumnBuilder(
tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries);
tsBlockBuilderStatus.createColumnBuilderStatus(),
initialExpectedEntries,
types.get(i));
break;
case INT64:
case TIMESTAMP:
valueColumnBuilders[i] =
new LongColumnBuilder(
tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries);
tsBlockBuilderStatus.createColumnBuilderStatus(),
initialExpectedEntries,
types.get(i));
break;
case FLOAT:
valueColumnBuilders[i] =
Expand All @@ -139,7 +143,9 @@ private TsBlockBuilder(int initialExpectedEntries, int maxTsBlockBytes, List<TSD
case STRING:
valueColumnBuilders[i] =
new BinaryColumnBuilder(
tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries);
tsBlockBuilderStatus.createColumnBuilderStatus(),
initialExpectedEntries,
types.get(i));
break;
default:
throw new IllegalArgumentException("Unknown data type: " + types.get(i));
Expand Down Expand Up @@ -188,13 +194,17 @@ public void buildValueColumnBuilders(List<TSDataType> types) {
case DATE:
valueColumnBuilders[i] =
new IntColumnBuilder(
tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries);
tsBlockBuilderStatus.createColumnBuilderStatus(),
initialExpectedEntries,
types.get(i));
break;
case INT64:
case TIMESTAMP:
valueColumnBuilders[i] =
new LongColumnBuilder(
tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries);
tsBlockBuilderStatus.createColumnBuilderStatus(),
initialExpectedEntries,
types.get(i));
break;
case FLOAT:
valueColumnBuilders[i] =
Expand All @@ -211,7 +221,9 @@ public void buildValueColumnBuilders(List<TSDataType> types) {
case STRING:
valueColumnBuilders[i] =
new BinaryColumnBuilder(
tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries);
tsBlockBuilderStatus.createColumnBuilderStatus(),
initialExpectedEntries,
types.get(i));
break;
default:
throw new IllegalArgumentException("Unknown data type: " + types.get(i));
Expand Down
2 changes: 1 addition & 1 deletion ...le/src/main/java/org/apache/tsfile/read/common/block/column/BinaryArrayColumnEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun
}
}
}
return new BinaryColumn(0, positionCount, nullIndicators, values);
return new BinaryColumn(0, positionCount, nullIndicators, values, dataType);
}

@Override
Expand Down
28 changes: 18 additions & 10 deletions java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,23 @@ public class BinaryColumn implements Column {
private final Binary[] values;

private final long retainedSizeInBytes;
private final TSDataType dataType;

public BinaryColumn(int initialCapacity) {
this(0, 0, null, new Binary[initialCapacity]);
public BinaryColumn(int initialCapacity, TSDataType dataType) {
this(0, 0, null, new Binary[initialCapacity], dataType);
}

public BinaryColumn(int positionCount, Optional<boolean[]> valueIsNull, Binary[] values) {
this(0, positionCount, valueIsNull.orElse(null), values);
public BinaryColumn(
int positionCount, Optional<boolean[]> valueIsNull, Binary[] values, TSDataType dataType) {
this(0, positionCount, valueIsNull.orElse(null), values, dataType);
}

BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values) {
BinaryColumn(
int arrayOffset,
int positionCount,
boolean[] valueIsNull,
Binary[] values,
TSDataType dataType) {
if (arrayOffset < 0) {
throw new IllegalArgumentException("arrayOffset is negative");
}
Expand All @@ -76,11 +83,12 @@ public BinaryColumn(int positionCount, Optional<boolean[]> valueIsNull, Binary[]
// TODO we need to sum up all the Binary's retainedSize here
retainedSizeInBytes =
INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOfObjectArray(positionCount);
this.dataType = dataType;
}

@Override
public TSDataType getDataType() {
return TSDataType.TEXT;
return dataType;
}

@Override
Expand Down Expand Up @@ -141,7 +149,7 @@ public long getRetainedSizeInBytes() {
@Override
public Column getRegion(int positionOffset, int length) {
checkValidRegion(getPositionCount(), positionOffset, length);
return new BinaryColumn(positionOffset + arrayOffset, length, valueIsNull, values);
return new BinaryColumn(positionOffset + arrayOffset, length, valueIsNull, values, dataType);
}

@Override
Expand All @@ -154,7 +162,7 @@ public Column getRegionCopy(int positionOffset, int length) {
valueIsNull != null ? Arrays.copyOfRange(valueIsNull, from, to) : null;
Binary[] valuesCopy = Arrays.copyOfRange(values, from, to);

return new BinaryColumn(0, length, valueIsNullCopy, valuesCopy);
return new BinaryColumn(0, length, valueIsNullCopy, valuesCopy, dataType);
}

@Override
Expand All @@ -163,7 +171,7 @@ public Column subColumn(int fromIndex) {
throw new IllegalArgumentException("fromIndex is not valid");
}
return new BinaryColumn(
arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values);
arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values, dataType);
}

@Override
Expand All @@ -178,7 +186,7 @@ public Column subColumnCopy(int fromIndex) {
Binary[] valuesCopy = Arrays.copyOfRange(values, from, positionCount);

int length = positionCount - fromIndex;
return new BinaryColumn(0, length, valueIsNullCopy, valuesCopy);
return new BinaryColumn(0, length, valueIsNullCopy, valuesCopy, dataType);
}

@Override
Expand Down
28 changes: 21 additions & 7 deletions .../tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumnBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,13 @@ public class BinaryColumnBuilder implements ColumnBuilder {
RamUsageEstimator.shallowSizeOfInstance(BinaryColumnBuilder.class);

private final ColumnBuilderStatus columnBuilderStatus;
public static final BinaryColumn NULL_VALUE_BLOCK =
new BinaryColumn(0, 1, new boolean[] {true}, new Binary[1]);
private final TSDataType dataType;
public static final BinaryColumn NULL_STRING_BLOCK =
new BinaryColumn(0, 1, new boolean[] {true}, new Binary[1], TSDataType.STRING);
public static final BinaryColumn NULL_TEXT_BLOCK =
new BinaryColumn(0, 1, new boolean[] {true}, new Binary[1], TSDataType.TEXT);
public static final BinaryColumn NULL_BLOB_BLOCK =
new BinaryColumn(0, 1, new boolean[] {true}, new Binary[1], TSDataType.BLOB);

private boolean initialized;
private final int initialEntryCount;
Expand All @@ -57,9 +62,11 @@ public class BinaryColumnBuilder implements ColumnBuilder {

private long arraysRetainedSizeInBytes;

public BinaryColumnBuilder(ColumnBuilderStatus columnBuilderStatus, int expectedEntries) {
public BinaryColumnBuilder(
ColumnBuilderStatus columnBuilderStatus, int expectedEntries, TSDataType dataType) {
this.initialEntryCount = max(expectedEntries, 1);
this.columnBuilderStatus = columnBuilderStatus;
this.dataType = dataType;
updateArraysDataSize();
}

Expand Down Expand Up @@ -112,14 +119,20 @@ public ColumnBuilder appendNull() {
@Override
public Column build() {
if (!hasNonNullValue) {
return new RunLengthEncodedColumn(NULL_VALUE_BLOCK, positionCount);
if (dataType == TSDataType.STRING) {
return new RunLengthEncodedColumn(NULL_STRING_BLOCK, positionCount);
} else if (dataType == TSDataType.TEXT) {
return new RunLengthEncodedColumn(NULL_TEXT_BLOCK, positionCount);
} else if (dataType == TSDataType.BLOB) {
return new RunLengthEncodedColumn(NULL_BLOB_BLOCK, positionCount);
}
}
return new BinaryColumn(0, positionCount, hasNullValue ? valueIsNull : null, values);
return new BinaryColumn(0, positionCount, hasNullValue ? valueIsNull : null, values, dataType);
}

@Override
public TSDataType getDataType() {
return TSDataType.TEXT;
return dataType;
}

@Override
Expand All @@ -135,7 +148,8 @@ public long getRetainedSizeInBytes() {
@Override
public ColumnBuilder newColumnBuilderLike(ColumnBuilderStatus columnBuilderStatus) {
// TODO we should take retain size into account here
return new BinaryColumnBuilder(columnBuilderStatus, calculateBlockResetSize(positionCount));
return new BinaryColumnBuilder(
columnBuilderStatus, calculateBlockResetSize(positionCount), dataType);
}

private void growCapacity() {
Expand Down
6 changes: 3 additions & 3 deletions java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,18 @@ public static Column create(TSDataType dataType, int initialCapacity) {
switch (dataType) {
case INT64:
case TIMESTAMP:
return new LongColumn(initialCapacity);
return new LongColumn(initialCapacity, dataType);
case DOUBLE:
return new DoubleColumn(initialCapacity);
case FLOAT:
return new FloatColumn(initialCapacity);
case TEXT:
case STRING:
case BLOB:
return new BinaryColumn(initialCapacity);
return new BinaryColumn(initialCapacity, dataType);
case INT32:
case DATE:
return new IntColumn(initialCapacity);
return new IntColumn(initialCapacity, dataType);
case BOOLEAN:
return new BooleanColumn(initialCapacity);
default:
Expand Down
2 changes: 1 addition & 1 deletion ...ile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun
}
}
}
return new IntColumn(0, positionCount, nullIndicators, intValues);
return new IntColumn(0, positionCount, nullIndicators, intValues, dataType);
case FLOAT:
float[] floatValues = new float[positionCount];
if (nullIndicators == null) {
Expand Down
2 changes: 1 addition & 1 deletion ...ile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun
}
}
}
return new LongColumn(0, positionCount, nullIndicators, values);
return new LongColumn(0, positionCount, nullIndicators, values, dataType);
case DOUBLE:
double[] doubleValues = new double[positionCount];
if (nullIndicators == null) {
Expand Down
29 changes: 19 additions & 10 deletions java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,23 @@ public class IntColumn implements Column {
private final int[] values;

private final long retainedSizeInBytes;
private final TSDataType dataType;

public IntColumn(int initialCapacity) {
this(0, 0, null, new int[initialCapacity]);
public IntColumn(int initialCapacity, TSDataType dataType) {
this(0, 0, null, new int[initialCapacity], dataType);
}

public IntColumn(int positionCount, Optional<boolean[]> valueIsNull, int[] values) {
this(0, positionCount, valueIsNull.orElse(null), values);
public IntColumn(
int positionCount, Optional<boolean[]> valueIsNull, int[] values, TSDataType dataType) {
this(0, positionCount, valueIsNull.orElse(null), values, dataType);
}

IntColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, int[] values) {
IntColumn(
int arrayOffset,
int positionCount,
boolean[] valueIsNull,
int[] values,
TSDataType dataType) {
if (arrayOffset < 0) {
throw new IllegalArgumentException("arrayOffset is negative");
}
Expand All @@ -75,11 +82,12 @@ public IntColumn(int positionCount, Optional<boolean[]> valueIsNull, int[] value

retainedSizeInBytes =
INSTANCE_SIZE + sizeOfIntArray(positionCount) + sizeOfBooleanArray(positionCount);
this.dataType = dataType;
}

@Override
public TSDataType getDataType() {
return TSDataType.INT32;
return dataType;
}

@Override
Expand Down Expand Up @@ -140,7 +148,7 @@ public long getRetainedSizeInBytes() {
@Override
public Column getRegion(int positionOffset, int length) {
checkValidRegion(getPositionCount(), positionOffset, length);
return new IntColumn(positionOffset + arrayOffset, length, valueIsNull, values);
return new IntColumn(positionOffset + arrayOffset, length, valueIsNull, values, dataType);
}

@Override
Expand All @@ -153,15 +161,16 @@ public Column getRegionCopy(int positionOffset, int length) {
valueIsNull != null ? Arrays.copyOfRange(valueIsNull, from, to) : null;
int[] valuesCopy = Arrays.copyOfRange(values, from, to);

return new IntColumn(0, length, valueIsNullCopy, valuesCopy);
return new IntColumn(0, length, valueIsNullCopy, valuesCopy, dataType);
}

@Override
public Column subColumn(int fromIndex) {
if (fromIndex > positionCount) {
throw new IllegalArgumentException("fromIndex is not valid");
}
return new IntColumn(arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values);
return new IntColumn(
arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values, dataType);
}

@Override
Expand All @@ -176,7 +185,7 @@ public Column subColumnCopy(int fromIndex) {
int[] valuesCopy = Arrays.copyOfRange(values, from, positionCount);

int length = positionCount - fromIndex;
return new IntColumn(0, length, valueIsNullCopy, valuesCopy);
return new IntColumn(0, length, valueIsNullCopy, valuesCopy, dataType);
}

@Override
Expand Down
Loading
Loading