Skip to content

Commit

Permalink
[To dev/1.3]Script adaptation to 4 new data types (#14296) (#14630)
Browse files Browse the repository at this point in the history 
* Script adaptation to 4 new data types

---------

Co-authored-by: 2b3c511 <[email protected]>
  • Loading branch information
@2b3c511
2b3c511 and 2b3c511 authored Jan 7, 2025
1 parent 5df241c commit 2669666
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 11 deletions.
32 changes: 28 additions & 4 deletions iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,15 @@
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.utils.Binary;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -176,7 +178,6 @@ public abstract class AbstractDataTool {
protected static Boolean aligned;
protected static Session session;
protected static final LongAdder loadFileSuccessfulNum = new LongAdder();
protected static final Map<String, TSDataType> TYPE_INFER_KEY_DICT = new HashMap<>();

protected static final String DATATYPE_BOOLEAN = "boolean";
protected static final String DATATYPE_INT = "int";
Expand All @@ -188,10 +189,13 @@ public abstract class AbstractDataTool {
protected static final String DATATYPE_BLOB = "blob";
protected static final String DATATYPE_NAN = "NaN";
protected static final String DATATYPE_TEXT = "text";
protected static final String DATATYPE_STRING = "string";

protected static final String DATATYPE_NULL = "null";
protected static int batchPointSize = 100_000;

protected static final Map<String, TSDataType> TYPE_INFER_KEY_DICT = new HashMap<>();

static {
TYPE_INFER_KEY_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN);
TYPE_INFER_KEY_DICT.put(DATATYPE_INT, TSDataType.FLOAT);
Expand All @@ -200,8 +204,9 @@ public abstract class AbstractDataTool {
TYPE_INFER_KEY_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE);
TYPE_INFER_KEY_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP);
TYPE_INFER_KEY_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP);
TYPE_INFER_KEY_DICT.put(DATATYPE_BLOB, TSDataType.TEXT);
TYPE_INFER_KEY_DICT.put(DATATYPE_BLOB, TSDataType.BLOB);
TYPE_INFER_KEY_DICT.put(DATATYPE_NAN, TSDataType.DOUBLE);
TYPE_INFER_KEY_DICT.put(DATATYPE_STRING, TSDataType.STRING);
}

protected static final Map<String, TSDataType> TYPE_INFER_VALUE_DICT = new HashMap<>();
Expand All @@ -213,9 +218,10 @@ public abstract class AbstractDataTool {
TYPE_INFER_VALUE_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT);
TYPE_INFER_VALUE_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE);
TYPE_INFER_VALUE_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP);
TYPE_INFER_VALUE_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP);
TYPE_INFER_VALUE_DICT.put(DATATYPE_BLOB, TSDataType.TEXT);
TYPE_INFER_VALUE_DICT.put(DATATYPE_DATE, TSDataType.DATE);
TYPE_INFER_VALUE_DICT.put(DATATYPE_BLOB, TSDataType.BLOB);
TYPE_INFER_VALUE_DICT.put(DATATYPE_TEXT, TSDataType.TEXT);
TYPE_INFER_VALUE_DICT.put(DATATYPE_STRING, TSDataType.STRING);
}

private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
Expand Down Expand Up @@ -689,6 +695,8 @@ private static TSDataType typeInfer(String strValue) {
// "NaN" is returned if the NaN Literal is given in Parser
} else if (DATATYPE_NAN.equals(strValue)) {
return TYPE_INFER_KEY_DICT.get(DATATYPE_NAN);
} else if (isBlob(strValue)) {
return TYPE_INFER_KEY_DICT.get(DATATYPE_BLOB);
} else if (strValue.length() <= 512) {
return STRING;
} else {
Expand All @@ -708,6 +716,10 @@ static boolean isNumber(String s) {
return true;
}

private static boolean isBlob(String s) {
return s.length() >= 3 && s.startsWith("X'") && s.endsWith("'");
}

private static boolean isBoolean(String s) {
return s.equalsIgnoreCase(SqlConstant.BOOLEAN_TRUE)
|| s.equalsIgnoreCase(SqlConstant.BOOLEAN_FALSE);
Expand Down Expand Up @@ -745,8 +757,11 @@ private static Object typeTrans(String value, TSDataType type) {
case DOUBLE:
return Double.parseDouble(value);
case TIMESTAMP:
return Long.parseLong(value);
case DATE:
return LocalDate.parse(value);
case BLOB:
return new Binary(parseHexStringToByteArray(value.replaceFirst("0x", "")));
default:
return null;
}
Expand All @@ -755,6 +770,15 @@ private static Object typeTrans(String value, TSDataType type) {
}
}

private static byte[] parseHexStringToByteArray(String hexString) {
byte[] bytes = new byte[hexString.length() / 2];
for (int i = 0; i < hexString.length(); i += 2) {
int value = Integer.parseInt(hexString.substring(i, i + 2), 16);
bytes[i / 2] = (byte) value;
}
return bytes;
}

private static long parseTimestamp(String str) {
long timestamp;
try {
Expand Down
43 changes: 38 additions & 5 deletions iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.utils.BytesUtils;
import org.jline.reader.LineReader;

import java.io.BufferedReader;
Expand All @@ -54,6 +55,7 @@
import java.io.FileWriter;
import java.io.IOException;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
Expand Down Expand Up @@ -651,7 +653,7 @@ public static void writeCsvFile(
while (i++ < linesPerFile) {
if (sessionDataSet.hasNext()) {
RowRecord rowRecord = sessionDataSet.next();
if (rowRecord.getTimestamp() != 0) {
if ("Time".equals(headers.get(0))) {
csvPrinterWrapper.print(timeTrans(rowRecord.getTimestamp()));
}
rowRecord
Expand All @@ -660,10 +662,24 @@ public static void writeCsvFile(
field -> {
String fieldStringValue = field.getStringValue();
if (!"null".equals(field.getStringValue())) {
if ((field.getDataType() == TSDataType.TEXT
|| field.getDataType() == TSDataType.STRING)
final TSDataType dataType = field.getDataType();
if ((dataType == TSDataType.TEXT || dataType == TSDataType.STRING)
&& !fieldStringValue.startsWith("root.")) {
fieldStringValue = "\"" + fieldStringValue + "\"";
} else if (dataType == TSDataType.BLOB) {
final byte[] v = field.getBinaryV().getValues();
if (v == null) {
fieldStringValue = null;
} else {
fieldStringValue = BytesUtils.parseBlobByteArrayToString(v);
}
} else if (dataType == TSDataType.DATE) {
final LocalDate dateV = field.getDateV();
if (dateV == null) {
fieldStringValue = null;
} else {
fieldStringValue = dateV.toString();
}
}
csvPrinterWrapper.print(fieldStringValue);
} else {
Expand Down Expand Up @@ -757,8 +773,25 @@ public static void writeSqlFile(
headersTemp.remove(seriesList.get(index));
continue;
}
if ("TEXT".equalsIgnoreCase(timeseriesList.get(3).getStringValue())) {
values.add("\"" + value + "\"");
final String dataType = timeseriesList.get(3).getStringValue();
if (TSDataType.TEXT.name().equalsIgnoreCase(dataType)
|| TSDataType.STRING.name().equalsIgnoreCase(dataType)) {
values.add("\'" + value + "\'");
} else if (TSDataType.BLOB.name().equalsIgnoreCase(dataType)) {
final byte[] v = fields.get(index).getBinaryV().getValues();
if (v == null) {
values.add(null);
} else {
values.add(
BytesUtils.parseBlobByteArrayToString(v).replaceFirst("0x", "X'") + "'");
}
} else if (TSDataType.DATE.name().equalsIgnoreCase(dataType)) {
final LocalDate dateV = fields.get(index).getDateV();
if (dateV == null) {
values.add(null);
} else {
values.add("'" + dateV.toString() + "'");
}
} else {
values.add(value);
}
Expand Down
24 changes: 22 additions & 2 deletions iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,13 +402,33 @@ private static void applyTypeInferArgs(String key, String value) throws ArgsErro
if (key.equals(DATATYPE_NAN)
&& !(value.equals(DATATYPE_FLOAT)
|| value.equals(DATATYPE_DOUBLE)
|| value.equals(DATATYPE_TEXT))) {
|| value.equals(DATATYPE_TEXT)
|| value.equals(DATATYPE_STRING))) {
throw new ArgsErrorException("NaN can not convert to " + value);
}
if (key.equals(DATATYPE_BOOLEAN)
&& !(value.equals(DATATYPE_BOOLEAN) || value.equals(DATATYPE_TEXT))) {
&& !(value.equals(DATATYPE_BOOLEAN)
|| value.equals(DATATYPE_TEXT)
|| value.equals(DATATYPE_STRING))) {
throw new ArgsErrorException("Boolean can not convert to " + value);
}
if (key.equals(DATATYPE_DATE)
&& !(value.equals(DATATYPE_DATE)
|| value.equals(DATATYPE_TEXT)
|| value.equals(DATATYPE_STRING))) {
throw new ArgsErrorException("Date can not convert to " + value);
}
if (key.equals(DATATYPE_TIMESTAMP)
&& !(value.equals(DATATYPE_TIMESTAMP)
|| value.equals(DATATYPE_TEXT)
|| value.equals(DATATYPE_STRING)
|| value.equals(DATATYPE_DOUBLE)
|| value.equals(DATATYPE_LONG))) {
throw new ArgsErrorException("Timestamp can not convert to " + value);
}
if (key.equals(DATATYPE_BLOB) && !(value.equals(DATATYPE_BLOB))) {
throw new ArgsErrorException("Blob can not convert to " + value);
}
final TSDataType srcType = TYPE_INFER_VALUE_DICT.get(key);
final TSDataType dstType = TYPE_INFER_VALUE_DICT.get(value);
if (dstType.getType() < srcType.getType()) {
Expand Down

0 comments on commit 2669666

Please sign in to comment.