Skip to content

Commit

Permalink
add validation in table schema constructor (#324)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shuwenwei
shuwenwei authored Dec 9, 2024
1 parent fcb444c commit a6680a4
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 17 deletions.
57 changes: 40 additions & 17 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,25 +75,34 @@ public TableSchema(
measurementSchema.getProps()))
.collect(Collectors.toList());
this.columnCategories = columnCategories;
this.updatable = false;
}

public TableSchema(
String tableName,
List<IMeasurementSchema> columnSchemas,
List<ColumnCategory> columnCategories) {
this.tableName = tableName.toLowerCase();
this.measurementSchemas =
columnSchemas.stream()
.map(
measurementSchema ->
new MeasurementSchema(
measurementSchema.getMeasurementName().toLowerCase(),
measurementSchema.getType(),
measurementSchema.getEncodingType(),
measurementSchema.getCompressor(),
measurementSchema.getProps()))
.collect(Collectors.toList());
this.measurementSchemas = new ArrayList<>(columnSchemas.size());
this.columnPosIndex = new HashMap<>(columnSchemas.size());
for (int i = 0; i < columnSchemas.size(); i++) {
IMeasurementSchema columnSchema = columnSchemas.get(i);
String measurementName = columnSchema.getMeasurementName().toLowerCase();
this.measurementSchemas.add(
new MeasurementSchema(
measurementName,
columnSchema.getType(),
columnSchema.getEncodingType(),
columnSchema.getCompressor(),
columnSchema.getProps()));
columnPosIndex.put(measurementName, i);
}
if (measurementSchemas.size() != columnPosIndex.size()) {
throw new IllegalArgumentException(
"Each column name in the table should be unique(case insensitive).");
}
this.columnCategories = columnCategories;
this.updatable = false;
}

public TableSchema(
Expand All @@ -103,24 +112,38 @@ public TableSchema(
List<ColumnCategory> categoryList) {
this.tableName = tableName.toLowerCase();
this.measurementSchemas = new ArrayList<>(columnNameList.size());
this.columnPosIndex = new HashMap<>(columnNameList.size());
for (int i = 0; i < columnNameList.size(); i++) {
measurementSchemas.add(
new MeasurementSchema(columnNameList.get(i).toLowerCase(), dataTypeList.get(i)));
String columnName = columnNameList.get(i).toLowerCase();
measurementSchemas.add(new MeasurementSchema(columnName, dataTypeList.get(i)));
columnPosIndex.put(columnName, i);
}
if (columnNameList.size() != columnPosIndex.size()) {
throw new IllegalArgumentException(
"Each column name in the table should be unique(case insensitive).");
}
this.columnCategories = categoryList;
this.updatable = false;
}

@TsFileApi
public TableSchema(String tableName, List<ColumnSchema> columnSchemaList) {
this.tableName = tableName.toLowerCase();
this.measurementSchemas = new ArrayList<>(columnSchemaList.size());
this.columnCategories = new ArrayList<>(columnSchemaList.size());
for (ColumnSchema columnSchema : columnSchemaList) {
this.measurementSchemas.add(
new MeasurementSchema(
columnSchema.getColumnName().toLowerCase(), columnSchema.getDataType()));
this.columnPosIndex = new HashMap<>(columnSchemaList.size());
for (int i = 0; i < columnSchemaList.size(); i++) {
ColumnSchema columnSchema = columnSchemaList.get(i);
String columnName = columnSchema.getColumnName().toLowerCase();
this.measurementSchemas.add(new MeasurementSchema(columnName, columnSchema.getDataType()));
this.columnCategories.add(columnSchema.getColumnCategory());
this.columnPosIndex.put(columnName, i);
}
if (columnSchemaList.size() != columnPosIndex.size()) {
throw new IllegalArgumentException(
"Each column name in the table should be unique(case insensitive).");
}
this.updatable = false;
}

public Map<String, Integer> getColumnPosIndex() {
Expand Down
42 changes: 42 additions & 0 deletions java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,12 @@ private void addValueOfDataType(
case STRING:
case BLOB:
{
if (value != null && !(value instanceof Binary) && !(value instanceof String)) {
throw new IllegalArgumentException(
String.format(
"Expected value of type Binary for data type %s, but got %s",
dataType, value.getClass().getName()));
}
final Binary[] sensor = (Binary[]) values[indexOfSchema];
if (value instanceof Binary) {
sensor[rowIndex] = (Binary) value;
Expand All @@ -307,37 +313,73 @@ private void addValueOfDataType(
}
case FLOAT:
{
if (value != null && !(value instanceof Float)) {
throw new IllegalArgumentException(
String.format(
"Expected value of type Float for data type %s, but got %s",
dataType, value.getClass().getName()));
}
final float[] sensor = (float[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (float) value : Float.MIN_VALUE;
break;
}
case INT32:
{
if (value != null && !(value instanceof Integer)) {
throw new IllegalArgumentException(
String.format(
"Expected value of type Integer for data type %s, but got %s",
dataType, value.getClass().getName()));
}
final int[] sensor = (int[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE;
break;
}
case DATE:
{
if (value != null && !(value instanceof LocalDate)) {
throw new IllegalArgumentException(
String.format(
"Expected value of type LocalDate for data type %s, but got %s",
dataType, value.getClass().getName()));
}
final LocalDate[] sensor = (LocalDate[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE;
break;
}
case INT64:
case TIMESTAMP:
{
if (value != null && !(value instanceof Long)) {
throw new IllegalArgumentException(
String.format(
"Expected value of type Long for data type %s, but got %s",
dataType, value.getClass().getName()));
}
final long[] sensor = (long[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (long) value : Long.MIN_VALUE;
break;
}
case DOUBLE:
{
if (value != null && !(value instanceof Double)) {
throw new IllegalArgumentException(
String.format(
"Expected value of type Double for data type %s, but got %s",
dataType, value.getClass().getName()));
}
final double[] sensor = (double[]) values[indexOfSchema];
sensor[rowIndex] = value != null ? (double) value : Double.MIN_VALUE;
break;
}
case BOOLEAN:
{
if (value != null && !(value instanceof Boolean)) {
throw new IllegalArgumentException(
String.format(
"Expected value of type Boolean for data type %s, but got %s",
dataType, value.getClass().getName()));
}
final boolean[] sensor = (boolean[]) values[indexOfSchema];
sensor[rowIndex] = value != null && (boolean) value;
break;
Expand Down
16 changes: 16 additions & 0 deletions java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,14 @@
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;

import org.junit.Assert;
import org.junit.Test;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import static org.apache.tsfile.write.record.Tablet.ColumnCategory.ID;
Expand Down Expand Up @@ -121,4 +123,18 @@ public void testLogicalTableSchema() throws IOException {
assertEquals(measurementSchemaCnt + 2, deserialized.getColumnSchemas().size());
}
}

@Test
public void testConstructTableSchemaWithDuplicateColumnName() {
try {
new TableSchema(
"t1",
Arrays.asList("id1", "ID1", "id2", "s1"),
Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.STRING, TSDataType.STRING),
Arrays.asList(ID, ID, MEASUREMENT, MEASUREMENT));
} catch (IllegalArgumentException e) {
return;
}
Assert.fail();
}
}
37 changes: 37 additions & 0 deletions java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,4 +241,41 @@ public void testSerializationAndDeSerializationNull() {
fail();
}
}

@Test
public void testWriteWrongType() {
final String deviceId = "root.sg";
final List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
measurementSchemas.add(new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s2", TSDataType.FLOAT, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s3", TSDataType.DOUBLE, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s4", TSDataType.BOOLEAN, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s5", TSDataType.TEXT, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s6", TSDataType.STRING, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s7", TSDataType.BLOB, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s8", TSDataType.TIMESTAMP, TSEncoding.PLAIN));
measurementSchemas.add(new MeasurementSchema("s9", TSDataType.DATE, TSEncoding.PLAIN));

Tablet tablet = new Tablet(deviceId, measurementSchemas);
addValueWithException(tablet, "s0", 0, 1L);
addValueWithException(tablet, "s1", 0, 1);
addValueWithException(tablet, "s2", 0, 0.1d);
addValueWithException(tablet, "s3", 0, 0.1f);
addValueWithException(tablet, "s3", 0, "1");
addValueWithException(tablet, "s5", 0, 1L);
addValueWithException(tablet, "s6", 0, 1L);
addValueWithException(tablet, "s7", 0, 1L);
addValueWithException(tablet, "s8", 0, "str");
addValueWithException(tablet, "s9", 0, 1L);
}

private void addValueWithException(Tablet tablet, String column, int rowIndex, Object value) {
try {
tablet.addValue(column, rowIndex, value);
} catch (IllegalArgumentException e) {
return;
}
Assert.fail();
}
}

0 comments on commit a6680a4

Please sign in to comment.