diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 3d9e40b77..30ae509bf 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -6,6 +6,11 @@ # 被测试的数据库，目前的格式为{name}{-version}{-insert mode}(注意-号)其全部参考值参见README文件，注意版本需要匹配 # DB_SWITCH=IoTDB-130-SESSION_BY_TABLET +# IoTDB需指定sql_dialect, 并且一个IoTDB只能指定一种。支持tree、table,默认值为tree。 +# sql_dialect等于table时，要满足：device数量 >= table数量 >= database数量 +# sql_dialect等于tree时，要满足：device数量 >= database数量 +# IoTDB_DIALECT_MODE=tree + # 主机列表，如果有多个主机则使用英文逗号进行分割 # HOST=127.0.0.1 @@ -70,15 +75,12 @@ # 如果 BENCHMARK_CLUSTER=true, 则为当前Benchmark的编号 # BENCHMARK_INDEX=0 -# 在集群模式下：是否所有数据库都可见，如果可见则可以向其他数据库节点发送数据 -# IS_ALL_NODES_VISIBLE=false - ################ Benchmark：双写模式 #################### -# 双写模式仅支持不同数据库之间进行比较，不支持同一个数据库不同版本进行双写 +# 双写模式仅支持不同数据库之间进行比较，不支持同一个数据库不同版本进行双写，不支持iotdb树模型与表模型的比较。 # IS_DOUBLE_WRITE=false # 另一个写入的数据库，目前的格式为{name}{-version}{-insert mode}(注意-号)其全部参考值参见README文件 -# ANOTHER_DB_SWITCH=IoTDB-100-SESSION_BY_TABLET +# ANOTHER_DB_SWITCH=IoTDB-130-SESSION_BY_TABLET # 另一个写入的数据库的主机 # ANOTHER_HOST=127.0.0.1 @@ -130,6 +132,9 @@ # 存储组名称前缀 # GROUP_NAME_PREFIX=g_ +# IoTDB 表模型下(IoTDB_DIALECT_MODE=table)，表名称前缀 +# IoTDB_TABLE_NAME_PREFIX=table_ + # 设备名称前缀 # DEVICE_NAME_PREFIX=d_ @@ -167,6 +172,9 @@ # 存储组的数量，对IoTDB而言为database的数量 # GROUP_NUMBER=1 +# IoTDB 表模型下(IoTDB_DIALECT_MODE=table)，表的数量 +# IoTDB_TABLE_NUMBER=1 + # IoTDB核心线程池数量 # IOTDB_SESSION_POOL_SIZE=50 @@ -298,7 +306,7 @@ ######################################################## ################### 操作信息整体配置 ##################### -# 各操作的比例,按照顺序为 写入:Q1:Q2:Q3:Q4:Q5:Q6:Q7:Q8:Q9:Q10, 请注意使用英文冒号。比例中的每一项是整数。 +# 各操作的比例,按照顺序为 写入:Q1:Q2:Q3:Q4:Q5:Q6:Q7:Q8:Q9:Q10:Q11, 请注意使用英文冒号。比例中的每一项是整数。 # Qi表示的查询如下: # Q1 精确点查询 select v1... from data where time = ? and device in ? # Q2 范围查询（只限制起止时间）select v1... from data where time > ? and time < ? and device in ? @@ -311,6 +319,7 @@ # Q9 倒序范围查询（只限制起止时间）select v1... from data where time > ? and time < ? and device in ? order by time desc # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc # Q11 分组聚合查询，倒序；目前仅支持iotdb、tdengine-3.0、influxdb v1 +# IoTDB V1表模型支持 Q1 Q2 Q3 Q9 Q10 # OPERATION_PROPORTION=1:0:0:0:0:0:0:0:0:0:0:0 # 最长等待写时间，单位毫秒，即如果整个写操作在指定时间内没有返回，则终止此操作 @@ -463,4 +472,5 @@ # 测试过程中测试进度日志的输出间隔，单位为秒 # LOG_PRINT_INTERVAL=5 # 测试过程中当前测试结果日志的输出间隔，单位为秒 +# 设为0时，benchmark 仅在结束前输出一次测试结果日志 # RESULT_PRINT_INTERVAL=3600 diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index d38ecd1a6..3a19188a9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -20,6 +20,7 @@ package cn.edu.tsinghua.iot.benchmark.conf; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.function.FunctionParam; import cn.edu.tsinghua.iot.benchmark.function.FunctionXml; @@ -60,6 +61,9 @@ public class Config { /** Total number of operations that each client process */ private long LOOP = 100; + /** iotdb supports table model and tree model */ + private SQLDialect IoTDB_DIALECT_MODE = SQLDialect.TREE; + /** * The running mode of benchmark 1. testWithDefaultPath: Conventional test mode, supporting mixed * loads of multiple read and write operations 2. writeWithRealDataSet: Write the real data set @@ -84,8 +88,6 @@ public class Config { private int BENCHMARK_INDEX = 0; /** Calculated in this way: FIRST_DEVICE_INDEX = BENCHMARK_INDEX * DEVICE_NUMBER */ private int FIRST_DEVICE_INDEX = 0; - /** 是否都可见，如果可见就可以向其他node发送 Whether access all nodes, rather than just one coordinator */ - private boolean IS_ALL_NODES_VISIBLE = false; // 初始化：被测数据库配置 private DBConfig dbConfig = new DBConfig(); @@ -188,6 +190,8 @@ public class Config { */ private int CLIENT_NUMBER = 20; + /** name prefix of table */ + private String IoTDB_TABLE_NAME_PREFIX = "table_"; /** name prefix of group */ private String GROUP_NAME_PREFIX = "g_"; /** name prefix of device */ @@ -222,6 +226,8 @@ public class Config { private String SG_STRATEGY = "mod"; /** The number of storage group, must less than or equal to number of devices */ private int GROUP_NUMBER = 1; + /** The number of table, In the tree model, it is equal to group_number */ + private int IoTDB_TABLE_NUMBER = 1; /** The size of IoTDB core session pool */ private int IOTDB_SESSION_POOL_SIZE = 50; /** Whether to use templates */ @@ -627,6 +633,14 @@ public void setLOOP(long LOOP) { this.LOOP = LOOP; } + public SQLDialect getIoTDB_DIALECT_MODE() { + return IoTDB_DIALECT_MODE; + } + + public void setIoTDB_DIALECT_MODE(SQLDialect ioTDB_DIALECT_MODE) { + IoTDB_DIALECT_MODE = ioTDB_DIALECT_MODE; + } + public BenchmarkMode getBENCHMARK_WORK_MODE() { return BENCHMARK_WORK_MODE; } @@ -683,14 +697,6 @@ public void setFIRST_DEVICE_INDEX(int FIRST_DEVICE_INDEX) { this.FIRST_DEVICE_INDEX = FIRST_DEVICE_INDEX; } - public boolean isIS_ALL_NODES_VISIBLE() { - return IS_ALL_NODES_VISIBLE; - } - - public void setIS_ALL_NODES_VISIBLE(boolean IS_ALL_NODES_VISIBLE) { - this.IS_ALL_NODES_VISIBLE = IS_ALL_NODES_VISIBLE; - } - public String getKAFKA_LOCATION() { return KAFKA_LOCATION; } @@ -939,6 +945,14 @@ public void setTAG_VALUE_CARDINALITY(List TAG_VALUE_CARDINALITY) { this.TAG_VALUE_CARDINALITY = TAG_VALUE_CARDINALITY; } + public String getIoTDB_TABLE_NAME_PREFIX() { + return IoTDB_TABLE_NAME_PREFIX; + } + + public void setIoTDB_TABLE_NAME_PREFIX(String IoTDB_TABLE_NAME_PREFIX) { + this.IoTDB_TABLE_NAME_PREFIX = IoTDB_TABLE_NAME_PREFIX; + } + public String getGROUP_NAME_PREFIX() { return GROUP_NAME_PREFIX; } @@ -1035,6 +1049,14 @@ public void setGROUP_NUMBER(int GROUP_NUMBER) { this.GROUP_NUMBER = GROUP_NUMBER; } + public int getIoTDB_TABLE_NUMBER() { + return IoTDB_TABLE_NUMBER; + } + + public void setIoTDB_TABLE_NUMBER(int ioTDB_TABLE_NUMBER) { + IoTDB_TABLE_NUMBER = ioTDB_TABLE_NUMBER; + } + public int getIOTDB_SESSION_POOL_SIZE() { return IOTDB_SESSION_POOL_SIZE; } @@ -1766,6 +1788,7 @@ public ConfigProperties getShowConfigProperties() { ConfigProperties configProperties = new ConfigProperties(); configProperties.addProperty("Test Mode", "BENCHMARK_WORK_MODE", this.BENCHMARK_WORK_MODE); + configProperties.addProperty("Test Mode", "IoTDB_DIALECT_MODE", this.getIoTDB_DIALECT_MODE()); configProperties.addProperty( "Database Connection Information", "DOUBLE_WRITE", this.IS_DOUBLE_WRITE); @@ -1778,6 +1801,7 @@ public ConfigProperties getShowConfigProperties() { this.ANOTHER_DBConfig.getMainConfig()); } configProperties.addProperty("Data Mode", "GROUP_NUMBER", this.GROUP_NUMBER); + configProperties.addProperty("Data Mode", "IoTDB_TABLE_NUMBER", this.IoTDB_TABLE_NUMBER); configProperties.addProperty("Data Mode", "DEVICE_NUMBER", this.DEVICE_NUMBER); configProperties.addProperty("Data Mode", "REAL_INSERT_RATE", this.REAL_INSERT_RATE); configProperties.addProperty("Data Mode", "SENSOR_NUMBER", this.SENSOR_NUMBER); @@ -1813,7 +1837,15 @@ public ConfigProperties getShowConfigProperties() { + "/" + this.ENCODING_DOUBLE + "/" - + this.ENCODING_TEXT); + + this.ENCODING_TEXT + + "/" + + this.ENCODING_STRING + + "/" + + this.ENCODING_BLOB + + "/" + + this.ENCODING_TIMESTAMP + + "/" + + this.ENCODING_DATE); configProperties.addProperty("Data Amount", "COMPRESSOR", this.COMPRESSOR); if (hasQuery()) { configProperties.addProperty("Query Param", "QUERY_DEVICE_NUM", this.QUERY_DEVICE_NUM); @@ -1836,8 +1868,6 @@ public ConfigProperties getShowConfigProperties() { if (this.BENCHMARK_CLUSTER) { configProperties.addProperty("Other Param", "BENCHMARK_INDEX", this.BENCHMARK_INDEX); configProperties.addProperty("Other Param", "FIRST_DEVICE_INDEX", this.FIRST_DEVICE_INDEX); - configProperties.addProperty( - "Other Param", "IS_ALL_NODES_VISIBLE", this.IS_ALL_NODES_VISIBLE); } if (this.TEMPLATE) { configProperties.addProperty("Other Param", "TEMPLATE", this.TEMPLATE); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 76dd73318..80cbb3fdf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -19,6 +19,7 @@ package cn.edu.tsinghua.iot.benchmark.conf; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.mode.enums.BenchmarkMode; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBSwitch; @@ -40,6 +41,7 @@ import java.util.stream.Collectors; import static cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode.INSERT_USE_SESSION_RECORDS; +import static cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode.INSERT_USE_SESSION_TABLET; public class ConfigDescriptor { private static final Logger LOGGER = LoggerFactory.getLogger(ConfigDescriptor.class); @@ -86,6 +88,9 @@ private void loadProps() { Properties properties = new Properties(); try { properties.load(inputStream); + config.setIoTDB_DIALECT_MODE( + SQLDialect.getSQLDialect( + properties.getProperty("IoTDB_DIALECT_MODE", config.getIoTDB_DIALECT_MODE() + ""))); config.setIS_DELETE_DATA( Boolean.parseBoolean( properties.getProperty("IS_DELETE_DATA", config.isIS_DELETE_DATA() + ""))); @@ -254,6 +259,8 @@ private void loadProps() { config.setCLIENT_NUMBER( Integer.parseInt( properties.getProperty("CLIENT_NUMBER", config.getCLIENT_NUMBER() + ""))); + config.setIoTDB_TABLE_NAME_PREFIX( + properties.getProperty("IoTDB_TABLE_NAME_PREFIX", config.getIoTDB_TABLE_NAME_PREFIX())); config.setGROUP_NAME_PREFIX( properties.getProperty("GROUP_NAME_PREFIX", config.getGROUP_NAME_PREFIX())); config.setDEVICE_NAME_PREFIX( @@ -290,11 +297,6 @@ private void loadProps() { } else { config.setFIRST_DEVICE_INDEX(0); } - config.setIS_ALL_NODES_VISIBLE( - Boolean.parseBoolean( - properties.getProperty( - "IS_ALL_NODES_VISIBLE", String.valueOf(config.isIS_ALL_NODES_VISIBLE())))); - config.setLINE_RATIO( Double.parseDouble(properties.getProperty("LINE_RATIO", config.getLINE_RATIO() + ""))); config.setSIN_RATIO( @@ -319,6 +321,24 @@ private void loadProps() { config.setGROUP_NUMBER( Integer.parseInt( properties.getProperty("GROUP_NUMBER", config.getGROUP_NUMBER() + ""))); + config.setIoTDB_TABLE_NUMBER( + Integer.parseInt( + properties.getProperty("IoTDB_TABLE_NUMBER", config.getIoTDB_TABLE_NUMBER() + ""))); + + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { + if (config.getGROUP_NUMBER() > config.getIoTDB_TABLE_NUMBER() + || config.getIoTDB_TABLE_NUMBER() > config.getDEVICE_NUMBER()) { + LOGGER.warn( + "Please follow this rule to adjust the parameters: device number >= table number >= database number. Otherwise, device number = table number = database number"); + } + } else { + if (config.getGROUP_NUMBER() > config.getDEVICE_NUMBER()) { + config.setIoTDB_TABLE_NUMBER(config.getGROUP_NUMBER()); + LOGGER.warn( + "Please follow this rule to adjust the parameters: device number >= database number. Otherwise, the total number of databases created is equal to the number of devices"); + } + } + config.setIOTDB_SESSION_POOL_SIZE( Integer.parseInt( properties.getProperty( @@ -606,6 +626,12 @@ private boolean checkConfig() { default: break; } + if ((config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE + && config.getDbConfig().getDB_SWITCH().getInsertMode() != INSERT_USE_SESSION_TABLET)) { + LOGGER.error( + "The iotdb table model only supports INSERT_USE_SESSION_TABLET! Please modify DB_SWITCH in the configuration file."); + result = false; + } result &= checkInsertDataTypeProportion(); result &= checkOperationProportion(); if (config.getCLIENT_NUMBER() == 0) { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java index 96efb9f45..4eebd6403 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java @@ -51,25 +51,19 @@ public class Constants { public static final String HASH_SG_ASSIGN_MODE = "hash"; public static final String DIV_SG_ASSIGN_MODE = "div"; + // TODO iotdb-1.4 + + public static final String IOTDB130 = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB"; public static final String IOTDB130_REST_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBRestAPI"; - public static final String IOTDB130_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB"; - public static final String IOTDB130_SESSION_CLASS = - "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBSession"; - public static final String IOTDB130_ROUNDROBIN_SESSION_CLASS = - "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBClusterSession"; public static final String IOTDB110_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb110.IoTDB"; public static final String IOTDB110_SESSION_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb110.IoTDBSession"; - public static final String IOTDB110_ROUNDROBIN_SESSION_CLASS = - "cn.edu.tsinghua.iot.benchmark.iotdb110.IoTDBClusterSession"; public static final String IOTDB100_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb100.IoTDB"; public static final String IOTDB100_SESSION_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb100.IoTDBSession"; - public static final String IOTDB100_ROUNDROBIN_SESSION_CLASS = - "cn.edu.tsinghua.iot.benchmark.iotdb100.IoTDBClusterSession"; public static final String INFLUXDB_CLASS = "cn.edu.tsinghua.iot.benchmark.influxdb.InfluxDB"; public static final String INFLUXDB2_CLASS = "cn.edu.tsinghua.iot.benchmark.influxdb2.InfluxDB"; diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/Sensor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/Sensor.java index c2172b716..d777fbea2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/Sensor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/Sensor.java @@ -19,6 +19,7 @@ package cn.edu.tsinghua.iot.benchmark.entity; +import cn.edu.tsinghua.iot.benchmark.entity.enums.ColumnCategory; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.utils.ReadWriteIOUtils; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -31,12 +32,20 @@ public class Sensor { private String name; private SensorType sensorType; + private ColumnCategory columnCategory; public Sensor() {} public Sensor(String name, SensorType sensorType) { this.name = name; this.sensorType = sensorType; + this.columnCategory = ColumnCategory.MEASUREMENT; + } + + public Sensor(String name, SensorType sensorType, ColumnCategory columnCategory) { + this.name = name; + this.sensorType = sensorType; + this.columnCategory = columnCategory; } public String getName() { @@ -55,6 +64,14 @@ public void setSensorType(SensorType sensorType) { this.sensorType = sensorType; } + public ColumnCategory getColumnCategory() { + return columnCategory; + } + + public void setColumnCategory(ColumnCategory columnCategory) { + this.columnCategory = columnCategory; + } + /** * serialize to output stream * @@ -63,6 +80,7 @@ public void setSensorType(SensorType sensorType) { public void serialize(ByteArrayOutputStream outputStream) throws IOException { ReadWriteIOUtils.write(name, outputStream); ReadWriteIOUtils.write(sensorType.ordinal(), outputStream); + ReadWriteIOUtils.write(columnCategory.ordinal(), outputStream); } /** @@ -74,6 +92,7 @@ public static Sensor deserialize(ByteArrayInputStream inputStream) throws IOExce Sensor result = new Sensor(); result.name = ReadWriteIOUtils.readString(inputStream); result.sensorType = SensorType.getType(ReadWriteIOUtils.readInt(inputStream)); + result.columnCategory = ColumnCategory.getType(ReadWriteIOUtils.readInt(inputStream)); return result; } @@ -92,12 +111,17 @@ public boolean equals(Object o) { return new EqualsBuilder() .append(name, that.name) .append(sensorType, that.sensorType) + .append(columnCategory, that.columnCategory) .isEquals(); } @Override public int hashCode() { - return new HashCodeBuilder(17, 37).append(name).append(sensorType).toHashCode(); + return new HashCodeBuilder(17, 37) + .append(name) + .append(sensorType) + .append(columnCategory) + .toHashCode(); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/ColumnCategory.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/ColumnCategory.java new file mode 100644 index 000000000..02c178bd0 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/ColumnCategory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cn.edu.tsinghua.iot.benchmark.entity.enums; + +public enum ColumnCategory { + ID("ID"), + ATTRIBUTE("ATTRIBUTE"), + MEASUREMENT("MEASUREMENT"); + + public String name; + + ColumnCategory(String name) { + this.name = name; + } + + public static ColumnCategory getType(int ordinal) { + for (ColumnCategory columnCategory : ColumnCategory.values()) { + if (columnCategory.ordinal() == ordinal) { + return columnCategory; + } + } + // default type + return ColumnCategory.MEASUREMENT; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/SQLDialect.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/SQLDialect.java new file mode 100644 index 000000000..b4673fa07 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/SQLDialect.java @@ -0,0 +1,22 @@ +package cn.edu.tsinghua.iot.benchmark.entity.enums; + +public enum SQLDialect { + TREE("tree"), + TABLE("table"); + + final String name; + + SQLDialect(String name) { + this.name = name; + } + + public static SQLDialect getSQLDialect(String Dialect) { + for (SQLDialect sqlDialect : SQLDialect.values()) { + if (sqlDialect.toString().equalsIgnoreCase(Dialect)) { + return sqlDialect; + } + } + throw new RuntimeException( + java.lang.String.format("Parameter SQLDialect %s is not supported", Dialect)); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/BaseMode.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/BaseMode.java index 734873fd3..921849aae 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/BaseMode.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/BaseMode.java @@ -84,7 +84,9 @@ public void run() { executorService.submit(client); } setTimeLimitScheduler(); - setMiddleMeasureScheduler(); + if (config.getRESULT_PRINT_INTERVAL() != 0) { + setMiddleMeasureScheduler(); + } startTime = System.nanoTime(); executorService.shutdown(); try { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/GenerateDataMode.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/GenerateDataMode.java index 82a0e99d1..26ce5d959 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/GenerateDataMode.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/GenerateDataMode.java @@ -46,7 +46,7 @@ protected void postCheck() { } @Override - void setMiddleMeasureTask() { + void setMiddleMeasureScheduler() { // Do nothing. GenerateDataMode not support measurement. } } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index e3d5efc9a..452f7245d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -13,7 +13,7 @@ public class MetaUtil { - private static Config config = ConfigDescriptor.getInstance().getConfig(); + private static final Config config = ConfigDescriptor.getInstance().getConfig(); private static final String TAG_KEY_PREFIX = config.getTAG_KEY_PREFIX(); private static final String TAG_VALUE_PREFIX = config.getTAG_VALUE_PREFIX(); private static final int TAG_NUMBER = config.getTAG_NUMBER(); @@ -38,18 +38,17 @@ public static int getDeviceId(int deviceId) { return config.getFIRST_DEVICE_INDEX() + deviceId; } - /** Calculate GroupId(integer) from device according to SG_STRATEGY */ - public static int calGroupId(int deviceId) throws WorkloadException { + /** tableId(deviceId) maps to groupId(tableId) according to SG_STRATEGY */ + public static int mappingId(int objectId, int objectNumber, int allocatingObjectNumber) + throws WorkloadException { switch (config.getSG_STRATEGY()) { case Constants.MOD_SG_ASSIGN_MODE: - return deviceId % config.getGROUP_NUMBER(); + return objectId % allocatingObjectNumber; case Constants.HASH_SG_ASSIGN_MODE: - return (deviceId + "").hashCode() % config.getGROUP_NUMBER(); + return String.valueOf(objectId).hashCode() % allocatingObjectNumber; case Constants.DIV_SG_ASSIGN_MODE: - int devicePerGroup = config.getDEVICE_NUMBER() / config.getGROUP_NUMBER(); - return devicePerGroup == 0 - ? deviceId - : (deviceId / devicePerGroup) % config.getGROUP_NUMBER(); + int itemPerObject = objectNumber / allocatingObjectNumber; + return itemPerObject == 0 ? objectId : (objectId / itemPerObject) % allocatingObjectNumber; default: throw new WorkloadException("Unsupported SG_STRATEGY: " + config.getSG_STRATEGY()); } @@ -64,6 +63,12 @@ public static String getGroupIdFromDeviceName(String deviceName) { return String.valueOf(groupId); } + public static String getTableIdFromDeviceName(String deviceName) { + int tableId = Math.abs(deviceName.hashCode()); + tableId = tableId % config.getIoTDB_TABLE_NUMBER(); + return String.valueOf(tableId); + } + /** Get deviceId from str */ public static int getDeviceIdFromStr(String device) { int deviceId = device.hashCode(); @@ -78,6 +83,10 @@ public static String getGroupName(Object groupId) { return config.getGROUP_NAME_PREFIX() + groupId; } + public static String getTableName(Object tableId) { + return config.getIoTDB_TABLE_NAME_PREFIX() + tableId; + } + public static String getDeviceName(Object deviceId) { return config.getDEVICE_NAME_PREFIX() + deviceId; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/DeviceSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/DeviceSchema.java index 45315bcff..547c88262 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/DeviceSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/DeviceSchema.java @@ -19,6 +19,8 @@ package cn.edu.tsinghua.iot.benchmark.schema.schemaImpl; +import cn.edu.tsinghua.iot.benchmark.conf.Config; +import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; @@ -39,10 +41,13 @@ public class DeviceSchema implements Cloneable { private static final Logger LOGGER = LoggerFactory.getLogger(DeviceSchema.class); + private static final Config config = ConfigDescriptor.getInstance().getConfig(); /** prefix of device name */ /** Each device belongs to one group, i.e. database */ private String group; + /** Each device belongs to one table */ + private String table; /** Name of device, e.g. DEVICE_NAME_PREFIX + deviceId */ private String device; /** List of tags */ @@ -66,8 +71,13 @@ public DeviceSchema(int deviceId, List sensors, Map tags this.sensors = sensors; this.tags = tags; try { - int thisDeviceGroupIndex = MetaUtil.calGroupId(deviceId); - this.group = MetaUtil.getGroupName(thisDeviceGroupIndex); + int deviceBelongToWhichTable = + MetaUtil.mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + int tableBelongToWhichGroup = + MetaUtil.mappingId( + deviceBelongToWhichTable, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); + this.table = MetaUtil.getTableName(deviceBelongToWhichTable); + this.group = MetaUtil.getGroupName(tableBelongToWhichGroup); } catch (WorkloadException e) { LOGGER.error("Create device schema failed.", e); } @@ -75,7 +85,31 @@ public DeviceSchema(int deviceId, List sensors, Map tags public DeviceSchema( String groupId, String deviceName, List sensors, Map tags) { + String tableId = MetaUtil.getTableIdFromDeviceName(deviceName); + this.table = MetaUtil.getTableName(tableId); + try { + this.group = + MetaUtil.getGroupName( + MetaUtil.mappingId( + Integer.parseInt(tableId), + config.getIoTDB_TABLE_NUMBER(), + config.getGROUP_NUMBER())); + } catch (WorkloadException e) { + LOGGER.error("Create device schema failed.", e); + } + this.device = deviceName; + this.sensors = sensors; + this.tags = tags; + } + + public DeviceSchema( + String groupId, + String tableName, + String deviceName, + List sensors, + Map tags) { this.group = MetaUtil.getGroupName(groupId); + this.table = MetaUtil.getTableName(tableName); this.device = deviceName; this.sensors = sensors; this.tags = tags; @@ -93,6 +127,14 @@ public int getDeviceId() { return deviceId; } + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + public String getGroup() { return group; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java index 6bdb3b80b..84867e73f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java @@ -32,6 +32,7 @@ import java.io.*; import java.nio.charset.StandardCharsets; +import java.time.LocalDate; import java.util.*; public class CSVDataReader extends DataReader { @@ -100,6 +101,7 @@ public IBatch nextBatch() { recordValues.add(Integer.parseInt(values[i])); break; case INT64: + case TIMESTAMP: recordValues.add(Long.parseLong(values[i])); break; case FLOAT: @@ -109,8 +111,13 @@ public IBatch nextBatch() { recordValues.add(Double.parseDouble(values[i])); break; case TEXT: + case STRING: + case BLOB: recordValues.add(values[i]); break; + case DATE: + recordValues.add(LocalDate.parse(values[i])); + break; default: LOGGER.error("Error Type"); } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java index b36a3011e..a109438dc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java @@ -38,22 +38,19 @@ public IDatabase getDatabase(DBConfig dbConfig) throws SQLException { String dbClass = ""; try { switch (dbConfig.getDB_SWITCH()) { + // TODO iotdb-1.4 + // IoTDB 1.3 case DB_IOT_130_REST: dbClass = Constants.IOTDB130_REST_CLASS; break; case DB_IOT_130_JDBC: - dbClass = Constants.IOTDB130_JDBC_CLASS; - break; case DB_IOT_130_SESSION_BY_TABLET: case DB_IOT_130_SESSION_BY_RECORD: case DB_IOT_130_SESSION_BY_RECORDS: - if (config.isIS_ALL_NODES_VISIBLE()) { - dbClass = Constants.IOTDB130_ROUNDROBIN_SESSION_CLASS; - } else { - dbClass = Constants.IOTDB130_SESSION_CLASS; - } + dbClass = Constants.IOTDB130; break; + // IoTDB 1.1 case DB_IOT_110_JDBC: dbClass = Constants.IOTDB110_JDBC_CLASS; @@ -61,11 +58,7 @@ public IDatabase getDatabase(DBConfig dbConfig) throws SQLException { case DB_IOT_110_SESSION_BY_TABLET: case DB_IOT_110_SESSION_BY_RECORD: case DB_IOT_110_SESSION_BY_RECORDS: - if (config.isIS_ALL_NODES_VISIBLE()) { - dbClass = Constants.IOTDB110_ROUNDROBIN_SESSION_CLASS; - } else { - dbClass = Constants.IOTDB110_SESSION_CLASS; - } + dbClass = Constants.IOTDB110_SESSION_CLASS; break; // IoTDB 1.0 case DB_IOT_100_JDBC: @@ -74,11 +67,7 @@ public IDatabase getDatabase(DBConfig dbConfig) throws SQLException { case DB_IOT_100_SESSION_BY_TABLET: case DB_IOT_100_SESSION_BY_RECORD: case DB_IOT_100_SESSION_BY_RECORDS: - if (config.isIS_ALL_NODES_VISIBLE()) { - dbClass = Constants.IOTDB100_ROUNDROBIN_SESSION_CLASS; - } else { - dbClass = Constants.IOTDB100_SESSION_CLASS; - } + dbClass = Constants.IOTDB100_SESSION_CLASS; break; case DB_INFLUX: dbClass = Constants.INFLUXDB_CLASS; diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBWrapper.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBWrapper.java index 6c6b401d6..4e9911531 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBWrapper.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBWrapper.java @@ -652,6 +652,8 @@ private boolean doComparisonByRecord(Query query, Operation operation, List operationLoops = new ConcurrentHashMap<>(); + private static final Map> tableDeviceMap = initTableDeviceMap(); public GenerateQueryWorkLoad(int id) { super(id); @@ -191,7 +193,9 @@ private long getQueryStartTimestamp(Operation operation) { } /** - * Return the list of deviceSchema + * Return the list of deviceSchema. + * + *

TODO When multi-table query is supported, there is no need to. * * @param typeAllow true: allow generating bool and text type. */ @@ -200,11 +204,26 @@ private List getQueryDeviceSchemaList(boolean typeAllow) throws Wo List queryDevices = new ArrayList<>(); List queryDeviceIds = new ArrayList<>(); List sensors = config.getSENSORS(); - while (queryDevices.size() < Math.min(config.getDEVICE_NUMBER(), config.getQUERY_DEVICE_NUM()) - && queryDeviceIds.size() < config.getDEVICE_NUMBER()) { + int deviceId = + queryDeviceRandom.nextInt(config.getDEVICE_NUMBER()) + config.getFIRST_DEVICE_INDEX(); + int tableId = + MetaUtil.mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + List devices = tableDeviceMap.get(tableId); + + int deviceQueryMaxCount = + (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) + ? config.getDEVICE_NUMBER() + / Math.min(config.getIoTDB_TABLE_NUMBER(), config.getDEVICE_NUMBER()) + : config.getDEVICE_NUMBER(); + while (queryDevices.size() < Math.min(deviceQueryMaxCount, config.getQUERY_DEVICE_NUM()) + && queryDeviceIds.size() < deviceQueryMaxCount) { // get a device belong to [first_device_index, first_device_index + device_number) - int deviceId = - queryDeviceRandom.nextInt(config.getDEVICE_NUMBER()) + config.getFIRST_DEVICE_INDEX(); + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { + deviceId = devices.get(queryDeviceRandom.nextInt(devices.size())); + } else { + deviceId = queryDeviceRandom.nextInt(config.getQUERY_DEVICE_NUM()); + } + deviceId = deviceId + config.getFIRST_DEVICE_INDEX(); // avoid duplicate if (!queryDeviceIds.contains(deviceId)) { queryDeviceIds.add(deviceId); @@ -242,7 +261,7 @@ private List getQueryDeviceSchemaList(boolean typeAllow) throws Wo new DeviceSchema(deviceId, querySensors, MetaUtil.getTags(deviceId)); queryDevices.add(deviceSchema); } - if (queryDevices.size() != config.getQUERY_DEVICE_NUM()) { + if (queryDevices.size() != Math.min(deviceQueryMaxCount, config.getQUERY_DEVICE_NUM())) { LOGGER.warn("There is no suitable sensor for query, please check INSERT_DATATYPE_PROPORTION"); throw new WorkloadException( "There is no suitable sensor for query, please check INSERT_DATATYPE_PROPORTION"); @@ -250,6 +269,20 @@ private List getQueryDeviceSchemaList(boolean typeAllow) throws Wo return queryDevices; } + private static Map> initTableDeviceMap() { + Map> tableDeviceMap = new ConcurrentHashMap<>(); + try { + for (int deviceId = 0; deviceId < config.getDEVICE_NUMBER(); deviceId++) { + int tableId = + MetaUtil.mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + tableDeviceMap.computeIfAbsent(tableId, k -> new ArrayList<>()).add(deviceId); + } + } catch (WorkloadException e) { + LOGGER.error(e.getMessage()); + } + return tableDeviceMap; + } + private void checkQuerySchemaParams() throws WorkloadException { if (!(config.getQUERY_DEVICE_NUM() > 0 && config.getQUERY_DEVICE_NUM() <= config.getDEVICE_NUMBER())) { diff --git a/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java b/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java index 836c92ea9..5f8e03d93 100644 --- a/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java +++ b/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java @@ -152,38 +152,23 @@ public Double registerSchema(List schemaList) throws TsdbException Map> sessionListMap = new HashMap<>(); try { // open meta session - if (!config.isIS_ALL_NODES_VISIBLE()) { + int sessionNumber = dbConfig.getHOST().size(); + List keys = new ArrayList<>(); + for (int i = 0; i < sessionNumber; i++) { Session metaSession = new Session.Builder() - .host(dbConfig.getHOST().get(0)) - .port(Integer.parseInt(dbConfig.getPORT().get(0))) + .host(dbConfig.getHOST().get(i)) + .port(Integer.parseInt(dbConfig.getPORT().get(i))) .username(dbConfig.getUSERNAME()) .password(dbConfig.getPASSWORD()) .version(Version.V_0_13) .build(); metaSession.open(config.isENABLE_THRIFT_COMPRESSION()); - sessionListMap.put(metaSession, createTimeseries(schemaList)); - } else { - int sessionNumber = dbConfig.getHOST().size(); - List keys = new ArrayList<>(); - for (int i = 0; i < sessionNumber; i++) { - Session metaSession = - new Session.Builder() - .host(dbConfig.getHOST().get(i)) - .port(Integer.parseInt(dbConfig.getPORT().get(i))) - .username(dbConfig.getUSERNAME()) - .password(dbConfig.getPASSWORD()) - .version(Version.V_0_13) - .build(); - metaSession.open(config.isENABLE_THRIFT_COMPRESSION()); - keys.add(metaSession); - sessionListMap.put(metaSession, new ArrayList<>()); - } - for (int i = 0; i < schemaList.size(); i++) { - sessionListMap - .get(keys.get(i % sessionNumber)) - .add(createTimeseries(schemaList.get(i))); - } + keys.add(metaSession); + sessionListMap.put(metaSession, new ArrayList<>()); + } + for (int i = 0; i < schemaList.size(); i++) { + sessionListMap.get(keys.get(i % sessionNumber)).add(createTimeseries(schemaList.get(i))); } if (config.isTEMPLATE() && templateInit.compareAndSet(false, true)) { diff --git a/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/SingleNodeJDBCConnection.java b/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/SingleNodeJDBCConnection.java index cde78e313..d8ccae01c 100644 --- a/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/SingleNodeJDBCConnection.java +++ b/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/SingleNodeJDBCConnection.java @@ -29,7 +29,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.List; import java.util.concurrent.atomic.AtomicInteger; public class SingleNodeJDBCConnection { @@ -50,19 +49,8 @@ public SingleNodeJDBCConnection(DBConfig dbConfig) { public void init() throws TsdbException { int nodeSize = 1; String[] urls; - if (config.isIS_ALL_NODES_VISIBLE()) { - nodeSize = dbConfig.getHOST().size(); - urls = new String[nodeSize]; - List clusterHosts = dbConfig.getHOST(); - for (int i = 0; i < nodeSize; i++) { - String jdbcUrl = - String.format(JDBC_URL, dbConfig.getHOST().get(i), dbConfig.getPORT().get(i)); - urls[i] = jdbcUrl; - } - } else { - urls = new String[nodeSize]; - urls[0] = String.format(JDBC_URL, dbConfig.getHOST().get(0), dbConfig.getPORT().get(0)); - } + urls = new String[nodeSize]; + urls[0] = String.format(JDBC_URL, dbConfig.getHOST().get(0), dbConfig.getPORT().get(0)); connections = new Connection[nodeSize]; for (int i = 0; i < connections.length; i++) { diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java index 6e520450f..280bac26c 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java @@ -159,39 +159,16 @@ public Double registerSchema(List schemaList) throws TsdbException Map> sessionListMap = new HashMap<>(); try { // open meta session - if (!config.isIS_ALL_NODES_VISIBLE()) { - Session metaSession = - new Session.Builder() - .host(dbConfig.getHOST().get(0)) - .port(Integer.parseInt(dbConfig.getPORT().get(0))) - .username(dbConfig.getUSERNAME()) - .password(dbConfig.getPASSWORD()) - .version(Version.V_1_0) - .build(); - metaSession.open(config.isENABLE_THRIFT_COMPRESSION()); - sessionListMap.put(metaSession, createTimeseries(schemaList)); - } else { - int sessionNumber = dbConfig.getHOST().size(); - List keys = new ArrayList<>(); - for (int i = 0; i < sessionNumber; i++) { - Session metaSession = - new Session.Builder() - .host(dbConfig.getHOST().get(i)) - .port(Integer.parseInt(dbConfig.getPORT().get(i))) - .username(dbConfig.getUSERNAME()) - .password(dbConfig.getPASSWORD()) - .version(Version.V_1_0) - .build(); - metaSession.open(config.isENABLE_THRIFT_COMPRESSION()); - keys.add(metaSession); - sessionListMap.put(metaSession, new ArrayList<>()); - } - for (int i = 0; i < schemaList.size(); i++) { - sessionListMap - .get(keys.get(i % sessionNumber)) - .add(createTimeseries(schemaList.get(i))); - } - } + Session metaSession = + new Session.Builder() + .host(dbConfig.getHOST().get(0)) + .port(Integer.parseInt(dbConfig.getPORT().get(0))) + .username(dbConfig.getUSERNAME()) + .password(dbConfig.getPASSWORD()) + .version(Version.V_1_0) + .build(); + metaSession.open(config.isENABLE_THRIFT_COMPRESSION()); + sessionListMap.put(metaSession, createTimeseries(schemaList)); if (config.isTEMPLATE() && schemaList.size() > 0 diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/SingleNodeJDBCConnection.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/SingleNodeJDBCConnection.java index ee2c37906..327e92f63 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/SingleNodeJDBCConnection.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/SingleNodeJDBCConnection.java @@ -29,7 +29,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.List; import java.util.concurrent.atomic.AtomicInteger; public class SingleNodeJDBCConnection { @@ -50,19 +49,8 @@ public SingleNodeJDBCConnection(DBConfig dbConfig) { public void init() throws TsdbException { int nodeSize = 1; String[] urls; - if (config.isIS_ALL_NODES_VISIBLE()) { - nodeSize = dbConfig.getHOST().size(); - urls = new String[nodeSize]; - List clusterHosts = dbConfig.getHOST(); - for (int i = 0; i < nodeSize; i++) { - String jdbcUrl = - String.format(JDBC_URL, dbConfig.getHOST().get(i), dbConfig.getPORT().get(i)); - urls[i] = jdbcUrl; - } - } else { - urls = new String[nodeSize]; - urls[0] = String.format(JDBC_URL, dbConfig.getHOST().get(0), dbConfig.getPORT().get(0)); - } + urls = new String[nodeSize]; + urls[0] = String.format(JDBC_URL, dbConfig.getHOST().get(0), dbConfig.getPORT().get(0)); connections = new Connection[nodeSize]; for (int i = 0; i < connections.length; i++) { diff --git a/iotdb-1.3/pom.xml b/iotdb-1.3/pom.xml index 214f010fb..a46344d8b 100644 --- a/iotdb-1.3/pom.xml +++ b/iotdb-1.3/pom.xml @@ -47,7 +47,7 @@ 1.3.14 - 1.3.3-SNAPSHOT + 1.4.0-SNAPSHOT 4.12.0 2.10.1 diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/DMLStrategy.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/DMLStrategy.java new file mode 100644 index 000000000..b74c53196 --- /dev/null +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/DMLStrategy.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cn.edu.tsinghua.iot.benchmark.iotdb130.DMLStrategy; + +import org.apache.iotdb.isession.util.Version; +import org.apache.iotdb.session.Session; + +import cn.edu.tsinghua.iot.benchmark.client.operation.Operation; +import cn.edu.tsinghua.iot.benchmark.conf.Config; +import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; +import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; +import cn.edu.tsinghua.iot.benchmark.entity.DeviceSummary; +import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; +import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; +import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; + +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class DMLStrategy { + protected static final Config config = ConfigDescriptor.getInstance().getConfig(); + protected ExecutorService service; + protected Future task; + protected final DBConfig dbConfig; + + public DMLStrategy(DBConfig dbConfig) { + this.dbConfig = dbConfig; + } + + public abstract Status insertOneBatch(IBatch batch, String devicePath) throws DBConnectException; + + public abstract long executeQueryAndGetStatusImpl( + String executeSQL, Operation operation, AtomicBoolean isOk, List> records) + throws SQLException; + + public abstract List verificationQueryImpl(String sql, Map> recordMap) + throws Exception; + + public abstract List> deviceQueryImpl(String sql) throws Exception; + + public abstract DeviceSummary deviceSummary( + String device, String totalLineNumberSql, String maxTimestampSql, String minTimestampSql) + throws TsdbException, SQLException; + + public abstract void init() throws TsdbException; + + public abstract void cleanup(); + + public abstract void close() throws TsdbException; + + protected Session buildSession(List hostUrls) { + return new Session.Builder() + .nodeUrls(hostUrls) + .username(dbConfig.getUSERNAME()) + .password(dbConfig.getPASSWORD()) + .enableRedirection(true) + .version(Version.V_1_0) + .sqlDialect(config.getIoTDB_DIALECT_MODE().name()) + .build(); + } +} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/JDBCStrategy.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/JDBCStrategy.java new file mode 100644 index 000000000..e08c4f5d1 --- /dev/null +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/JDBCStrategy.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cn.edu.tsinghua.iot.benchmark.iotdb130.DMLStrategy; + +import cn.edu.tsinghua.iot.benchmark.client.operation.Operation; +import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; +import cn.edu.tsinghua.iot.benchmark.entity.DeviceSummary; +import cn.edu.tsinghua.iot.benchmark.entity.Record; +import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; +import cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB; +import cn.edu.tsinghua.iot.benchmark.iotdb130.SingleNodeJDBCConnection; +import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; +import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; +import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; +import cn.edu.tsinghua.iot.benchmark.utils.NamedThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static cn.edu.tsinghua.iot.benchmark.client.operation.Operation.LATEST_POINT_QUERY; + +public class JDBCStrategy extends DMLStrategy { + + private static final Logger LOGGER = LoggerFactory.getLogger(JDBCStrategy.class); + + private SingleNodeJDBCConnection ioTDBConnection; + + public JDBCStrategy(DBConfig dbConfig) { + super(dbConfig); + } + + @Override + public Status insertOneBatch(IBatch batch, String devicePath) throws DBConnectException { + try (Statement statement = ioTDBConnection.getConnection().createStatement()) { + for (Record record : batch.getRecords()) { + String sql = + getInsertOneBatchSql( + batch.getDeviceSchema(), record.getTimestamp(), record.getRecordDataValue()); + statement.addBatch(sql); + } + statement.executeBatch(); + return new Status(true); + } catch (Exception e) { + return new Status(false, 0, e, e.toString()); + } + } + + @Override + public long executeQueryAndGetStatusImpl( + String executeSQL, Operation operation, AtomicBoolean isOk, List> records) + throws SQLException { + Boolean status = true; + AtomicLong queryResultPointNum = new AtomicLong(); + try (Statement statement = ioTDBConnection.getConnection().createStatement()) { + AtomicInteger line = new AtomicInteger(); + task = + service.submit( + () -> { + try { + try (ResultSet resultSet = statement.executeQuery(executeSQL)) { + while (resultSet.next()) { + line.getAndIncrement(); + if (config.isIS_COMPARISON()) { + List record = new ArrayList<>(); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { + switch (operation) { + case LATEST_POINT_QUERY: + if (i == 2 || i >= 4) { + continue; + } + break; + default: + break; + } + record.add(resultSet.getObject(i)); + } + records.add(record); + } + } + } + } catch (SQLException e) { + LOGGER.error("exception occurred when execute query={}", executeSQL, e); + isOk.set(false); + } + long resultPointNum = line.get(); + if (!LATEST_POINT_QUERY.equals(operation)) { + resultPointNum *= config.getQUERY_SENSOR_NUM(); + resultPointNum *= config.getQUERY_DEVICE_NUM(); + } + queryResultPointNum.set(resultPointNum); + }); + } + try { + task.get(config.getREAD_OPERATION_TIMEOUT_MS(), TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + task.cancel(true); + } + return queryResultPointNum.get(); + } + + @Override + public List verificationQueryImpl(String sql, Map> recordMap) + throws Exception { + int point = 0, line = 0; + try (Statement statement = ioTDBConnection.getConnection().createStatement()) { + ResultSet resultSet = statement.executeQuery(sql); + while (resultSet.next()) { + long timeStamp = resultSet.getLong(1); + List values = recordMap.get(timeStamp); + for (int i = 0; i < values.size(); i++) { + String value = resultSet.getString(i + 2); + String target = String.valueOf(values.get(i)); + if (!value.equals(target)) { + LOGGER.error("Using SQL: " + sql + ",Expected:" + value + " but was: " + target); + } else { + point++; + } + } + line++; + } + } + return Arrays.asList(point, line); + } + + @Override + public List> deviceQueryImpl(String sql) throws Exception { + List> result = new ArrayList<>(); + try (Statement statement = ioTDBConnection.getConnection().createStatement()) { + ResultSet resultSet = statement.executeQuery(sql); + int colNumber = resultSet.getMetaData().getColumnCount(); + while (resultSet.next()) { + List line = new ArrayList<>(); + for (int i = 1; i <= colNumber; i++) { + line.add(resultSet.getObject(i)); + } + result.add(line); + } + } + return result; + } + + @Override + public DeviceSummary deviceSummary( + String device, String totalLineNumberSql, String maxTimestampSql, String minTimestampSql) + throws SQLException, TsdbException { + int totalLineNumber = 0; + long minTimeStamp = 0, maxTimeStamp = 0; + try (Statement statement = ioTDBConnection.getConnection().createStatement()) { + ResultSet resultSet = statement.executeQuery(totalLineNumberSql); + resultSet.next(); + totalLineNumber = Integer.parseInt(resultSet.getString(1)); + + resultSet = statement.executeQuery(maxTimestampSql); + resultSet.next(); + maxTimeStamp = Long.parseLong(resultSet.getObject(1).toString()); + + resultSet = statement.executeQuery(minTimestampSql); + resultSet.next(); + minTimeStamp = Long.parseLong(resultSet.getObject(1).toString()); + } + return new DeviceSummary(device, totalLineNumber, minTimeStamp, maxTimeStamp); + } + + @Override + public void init() throws TsdbException { + if (ioTDBConnection == null) { + try { + ioTDBConnection = new SingleNodeJDBCConnection(dbConfig); + ioTDBConnection.init(); + this.service = + Executors.newSingleThreadExecutor(new NamedThreadFactory("DataClientExecuteJob")); + } catch (Exception e) { + throw new TsdbException(e); + } + } + } + + @Override + public void cleanup() { + try (Statement statement = ioTDBConnection.getConnection().createStatement()) { + statement.execute(IoTDB.DELETE_SERIES_SQL); + LOGGER.info("Finish clean data!"); + } catch (Exception e) { + LOGGER.warn("No Data to Clean!"); + } + } + + @Override + public void close() throws TsdbException { + if (ioTDBConnection != null) { + ioTDBConnection.close(); + } + if (service != null) { + service.shutdownNow(); + } + if (task != null) { + task.cancel(true); + } + } + + private String getInsertOneBatchSql( + DeviceSchema deviceSchema, long timestamp, List values) { + StringBuilder builder = new StringBuilder("insert into "); + builder.append(deviceSchema.getDevicePath()).append("(timestamp"); + for (Sensor sensor : deviceSchema.getSensors()) { + builder.append(",").append(sensor.getName()); + } + if (config.isVECTOR()) { + builder.append(") aligned values("); + } else { + builder.append(") values("); + } + builder.append(timestamp); + int sensorIndex = 0; + List sensors = deviceSchema.getSensors(); + for (Object value : values) { + switch (sensors.get(sensorIndex).getSensorType()) { + case BOOLEAN: + case INT32: + case INT64: + case FLOAT: + case DOUBLE: + case TIMESTAMP: + case DATE: + builder.append(",").append(value); + break; + case TEXT: + case STRING: + builder.append(",").append("'").append(value).append("'"); + break; + case BLOB: + builder.append(",").append("X'").append(value).append("'"); + break; + } + sensorIndex++; + } + builder.append(")"); + LOGGER.debug("getInsertOneBatchSql: {}", builder); + return builder.toString(); + } +} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSessionBase.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/SessionStrategy.java similarity index 54% rename from iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSessionBase.java rename to iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/SessionStrategy.java index 138efcae6..ad4fd6048 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSessionBase.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/DMLStrategy/SessionStrategy.java @@ -17,11 +17,12 @@ * under the License. */ -package cn.edu.tsinghua.iot.benchmark.iotdb130; +package cn.edu.tsinghua.iot.benchmark.iotdb130.DMLStrategy; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; import cn.edu.tsinghua.iot.benchmark.client.operation.Operation; import cn.edu.tsinghua.iot.benchmark.conf.Config; @@ -32,13 +33,12 @@ import cn.edu.tsinghua.iot.benchmark.entity.Sensor; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.OperationFailException; +import cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB; +import cn.edu.tsinghua.iot.benchmark.iotdb130.utils.IoTDBUtils; import cn.edu.tsinghua.iot.benchmark.measurement.Status; -import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; import cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode; -import cn.edu.tsinghua.iot.benchmark.workload.query.impl.DeviceQuery; -import cn.edu.tsinghua.iot.benchmark.workload.query.impl.VerificationQuery; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Field; @@ -46,17 +46,21 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.sql.SQLException; import java.time.LocalDate; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -64,30 +68,49 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; -public class IoTDBSessionBase extends IoTDB { - static Logger LOGGER; - +public class SessionStrategy extends DMLStrategy { + private static final Logger LOGGER = LoggerFactory.getLogger(SessionStrategy.class); static final Config config = ConfigDescriptor.getInstance().getConfig(); - IBenchmarkSession sessionWrapper; private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE()); + private final Session session; + private final IoTDB iotdb; - public IoTDBSessionBase(DBConfig dbConfig) { + public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { super(dbConfig); + this.iotdb = iotdb; + List hostUrls = new ArrayList<>(dbConfig.getHOST().size()); + for (int i = 0; i < dbConfig.getHOST().size(); i++) { + hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); + } + session = buildSession(hostUrls); } - public Status insertOneBatchByTablet(IBatch batch) { + @Override + public Status insertOneBatch(IBatch batch, String devicePath) { + DBInsertMode insertMode = dbConfig.getDB_SWITCH().getInsertMode(); + switch (insertMode) { + case INSERT_USE_SESSION_TABLET: + return insertOneBatchByTablet(batch); + case INSERT_USE_SESSION_RECORD: + return insertOneBatchByRecord(batch, devicePath); + case INSERT_USE_SESSION_RECORDS: + return insertOneBatchByRecords(batch, devicePath); + default: + throw new IllegalStateException("Unexpected INSERT_MODE value: " + insertMode); + } + } + + // region private method + + private Status insertOneBatchByTablet(IBatch batch) { Tablet tablet = genTablet(batch); task = service.submit( () -> { try { - if (config.isVECTOR()) { - sessionWrapper.insertAlignedTablet(tablet); - } else { - sessionWrapper.insertTablet(tablet); - } + iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); } @@ -95,8 +118,91 @@ public Status insertOneBatchByTablet(IBatch batch) { return waitWriteTaskToFinishAndGetStatus(); } - public Status insertOneBatchByRecord(IBatch batch) { - String deviceId = getDevicePath(batch.getDeviceSchema()); + private Tablet genTablet(IBatch batch) { + List schemaList = new ArrayList<>(); + List columnTypes = new ArrayList<>(); + List sensors = batch.getDeviceSchema().getSensors(); + iotdb.deleteIDColumnIfNecessary(columnTypes, sensors, batch); + iotdb.addIDColumnIfNecessary(columnTypes, sensors, batch); + int sensorIndex = 0; + for (Sensor sensor : sensors) { + SensorType dataSensorType = sensor.getSensorType(); + schemaList.add( + new MeasurementSchema( + sensor.getName(), + Enum.valueOf(TSDataType.class, dataSensorType.name), + Enum.valueOf( + TSEncoding.class, + Objects.requireNonNull(IoTDB.getEncodingType(dataSensorType))))); + sensorIndex++; + } + String deviceId = iotdb.getInsertTargetName(batch.getDeviceSchema()); + Tablet tablet = + iotdb.createTablet(deviceId, schemaList, columnTypes, batch.getRecords().size()); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + + for (int recordIndex = 0; recordIndex < batch.getRecords().size(); recordIndex++) { + tablet.rowSize++; + Record record = batch.getRecords().get(recordIndex); + sensorIndex = 0; + long currentTime = record.getTimestamp(); + timestamps[recordIndex] = currentTime; + for (int recordValueIndex = 0; + recordValueIndex < record.getRecordDataValue().size(); + recordValueIndex++) { + switch (sensors.get(sensorIndex).getSensorType()) { + case BOOLEAN: + boolean[] sensorsBool = (boolean[]) values[recordValueIndex]; + sensorsBool[recordIndex] = + (boolean) (record.getRecordDataValue().get(recordValueIndex)); + break; + case INT32: + int[] sensorsInt = (int[]) values[recordValueIndex]; + sensorsInt[recordIndex] = (int) (record.getRecordDataValue().get(recordValueIndex)); + break; + case INT64: + long[] sensorsLong = (long[]) values[recordValueIndex]; + sensorsLong[recordIndex] = (long) (record.getRecordDataValue().get(recordValueIndex)); + break; + case FLOAT: + float[] sensorsFloat = (float[]) values[recordValueIndex]; + sensorsFloat[recordIndex] = (float) (record.getRecordDataValue().get(recordValueIndex)); + break; + case DOUBLE: + double[] sensorsDouble = (double[]) values[recordValueIndex]; + sensorsDouble[recordIndex] = + (double) (record.getRecordDataValue().get(recordValueIndex)); + break; + case TEXT: + case STRING: + case BLOB: + Binary[] sensorsText = (Binary[]) values[recordValueIndex]; + sensorsText[recordIndex] = + binaryCache.computeIfAbsent( + (String) record.getRecordDataValue().get(recordValueIndex), + BytesUtils::valueOf); + break; + case TIMESTAMP: + long[] sensorsTimestamp = (long[]) values[recordValueIndex]; + sensorsTimestamp[recordIndex] = + (long) (record.getRecordDataValue().get(recordValueIndex)); + break; + case DATE: + LocalDate[] sensorsDate = (LocalDate[]) values[recordValueIndex]; + sensorsDate[recordIndex] = + (LocalDate) (record.getRecordDataValue().get(recordValueIndex)); + break; + default: + LOGGER.error("Unsupported Type: {}", sensors.get(sensorIndex).getSensorType()); + } + sensorIndex++; + } + } + return tablet; + } + + private Status insertOneBatchByRecord(IBatch batch, String deviceId) { int failRecord = 0; List sensors = batch.getDeviceSchema().getSensors().stream() @@ -105,15 +211,14 @@ public Status insertOneBatchByRecord(IBatch batch) { for (Record record : batch.getRecords()) { long timestamp = record.getTimestamp(); List dataTypes = - constructDataTypes( + IoTDBUtils.constructDataTypes( batch.getDeviceSchema().getSensors(), record.getRecordDataValue().size()); List recordDataValue = convertTypeForBLOB(record, dataTypes); try { if (config.isVECTOR()) { - sessionWrapper.insertAlignedRecord( - deviceId, timestamp, sensors, dataTypes, recordDataValue); + session.insertAlignedRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } else { - sessionWrapper.insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); + session.insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } } catch (IoTDBConnectionException | StatementExecutionException e) { failRecord++; @@ -127,7 +232,7 @@ public Status insertOneBatchByRecord(IBatch batch) { } } - public Status insertOneBatchByRecords(IBatch batch) { + private Status insertOneBatchByRecords(IBatch batch, String deviceId) { List deviceIds = new ArrayList<>(); List times = new ArrayList<>(); List> measurementsList = new ArrayList<>(); @@ -138,13 +243,12 @@ public Status insertOneBatchByRecords(IBatch batch) { .map(Sensor::getName) .collect(Collectors.toList()); while (true) { - String deviceId = getDevicePath(batch.getDeviceSchema()); for (Record record : batch.getRecords()) { deviceIds.add(deviceId); times.add(record.getTimestamp()); measurementsList.add(sensors); List dataTypes = - constructDataTypes( + IoTDBUtils.constructDataTypes( batch.getDeviceSchema().getSensors(), record.getRecordDataValue().size()); valuesList.add(convertTypeForBLOB(record, dataTypes)); typesList.add(dataTypes); @@ -159,11 +263,10 @@ public Status insertOneBatchByRecords(IBatch batch) { () -> { try { if (config.isVECTOR()) { - sessionWrapper.insertAlignedRecords( + session.insertAlignedRecords( deviceIds, times, measurementsList, typesList, valuesList); } else { - sessionWrapper.insertRecords( - deviceIds, times, measurementsList, typesList, valuesList); + session.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); } } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); @@ -172,136 +275,102 @@ public Status insertOneBatchByRecords(IBatch batch) { return waitWriteTaskToFinishAndGetStatus(); } - @Override - protected Status executeQueryAndGetStatus(String sql, Operation operation) { - String executeSQL; - if (config.isIOTDB_USE_DEBUG() && random.nextDouble() < config.getIOTDB_USE_DEBUG_RATIO()) { - executeSQL = "debug " + sql; - } else { - executeSQL = sql; - } - if (!config.isIS_QUIET_MODE()) { - LOGGER.info("{} query SQL: {}", Thread.currentThread().getName(), executeSQL); + private List convertTypeForBLOB(Record record, List dataTypes) { + // String change to Binary + List dataValue = record.getRecordDataValue(); + for (int recordValueIndex = 0; + recordValueIndex < record.getRecordDataValue().size(); + recordValueIndex++) { + if (Objects.requireNonNull(dataTypes.get(recordValueIndex)) == TSDataType.BLOB) { + dataValue.set( + recordValueIndex, + binaryCache.computeIfAbsent( + (String) record.getRecordDataValue().get(recordValueIndex), BytesUtils::valueOf)); + } } - AtomicInteger line = new AtomicInteger(); + return dataValue; + } + + // endregion + + @Override + public long executeQueryAndGetStatusImpl( + String executeSQL, Operation operation, AtomicBoolean isOk, List> records) + throws SQLException { AtomicLong queryResultPointNum = new AtomicLong(); - AtomicBoolean isOk = new AtomicBoolean(true); - try { - List> records = new ArrayList<>(); - task = - service.submit( - () -> { - try { - ISessionDataSet sessionDataSet = sessionWrapper.executeQueryStatement(executeSQL); - if (config.isIS_COMPARISON()) { - while (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - line.getAndIncrement(); - List record = new ArrayList<>(); + AtomicInteger line = new AtomicInteger(); + task = + service.submit( + () -> { + try { + SessionDataSet sessionDataSet = session.executeQueryStatement(executeSQL); + if (config.isIS_COMPARISON()) { + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + line.getAndIncrement(); + List record = new ArrayList<>(); + switch (operation) { + case AGG_RANGE_QUERY: + case AGG_VALUE_QUERY: + case AGG_RANGE_VALUE_QUERY: + break; + default: + record.add(rowRecord.getTimestamp()); + break; + } + List fields = rowRecord.getFields(); + for (int i = 0; i < fields.size(); i++) { switch (operation) { - case AGG_RANGE_QUERY: - case AGG_VALUE_QUERY: - case AGG_RANGE_VALUE_QUERY: - break; + case LATEST_POINT_QUERY: + if (i == 0 || i == 2) { + continue; + } default: - record.add(rowRecord.getTimestamp()); break; } - List fields = rowRecord.getFields(); - for (int i = 0; i < fields.size(); i++) { - switch (operation) { - case LATEST_POINT_QUERY: - if (i == 0 || i == 2) { - continue; - } - default: - break; - } - record.add(fields.get(i).toString()); - } - records.add(record); - } - } else { - SessionDataSet.DataIterator iterator = sessionDataSet.iterator(); - while (iterator.next()) { - line.getAndIncrement(); + record.add(fields.get(i).toString()); } + records.add(record); + } + } else { + SessionDataSet.DataIterator iterator = sessionDataSet.iterator(); + while (iterator.next()) { + line.getAndIncrement(); } - - sessionDataSet.close(); - } catch (StatementExecutionException | IoTDBConnectionException e) { - LOGGER.error("exception occurred when execute query={}", executeSQL, e); - isOk.set(false); - } - long resultPointNum = line.get(); - if (!Operation.LATEST_POINT_QUERY.equals(operation)) { - resultPointNum *= config.getQUERY_SENSOR_NUM(); - resultPointNum *= config.getQUERY_DEVICE_NUM(); } - queryResultPointNum.set(resultPointNum); - }); - try { - task.get(config.getREAD_OPERATION_TIMEOUT_MS(), TimeUnit.MILLISECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - task.cancel(true); - return new Status(false, queryResultPointNum.get(), e, executeSQL); - } - if (isOk.get()) { - if (config.isIS_COMPARISON()) { - return new Status(true, queryResultPointNum.get(), executeSQL, records); - } else { - return new Status(true, queryResultPointNum.get()); - } - } else { - return new Status( - false, queryResultPointNum.get(), new Exception("Failed to execute."), executeSQL); - } - } catch (Exception e) { - return new Status(false, queryResultPointNum.get(), e, executeSQL); - } catch (Throwable t) { - return new Status(false, queryResultPointNum.get(), new Exception(t), executeSQL); + sessionDataSet.close(); + } catch (StatementExecutionException | IoTDBConnectionException e) { + LOGGER.error("exception occurred when execute query={}", executeSQL, e); + isOk.set(false); + } + long resultPointNum = line.get(); + if (!Operation.LATEST_POINT_QUERY.equals(operation)) { + resultPointNum *= config.getQUERY_SENSOR_NUM(); + resultPointNum *= config.getQUERY_DEVICE_NUM(); + } + queryResultPointNum.set(resultPointNum); + }); + try { + task.get(config.getREAD_OPERATION_TIMEOUT_MS(), TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + task.cancel(true); + return queryResultPointNum.get(); } + return queryResultPointNum.get(); } - /** - * Using in verification - * - * @param verificationQuery the query of verification - */ @Override - public Status verificationQuery(VerificationQuery verificationQuery) { - DeviceSchema deviceSchema = verificationQuery.getDeviceSchema(); - List deviceSchemas = new ArrayList<>(); - deviceSchemas.add(deviceSchema); - - List records = verificationQuery.getRecords(); - if (records == null || records.size() == 0) { - return new Status( - false, - new TsdbException("There are no records in verificationQuery."), - "There are no records in verificationQuery."); - } - - StringBuffer sql = new StringBuffer(); - sql.append(getSimpleQuerySqlHead(deviceSchemas)); - Map> recordMap = new HashMap<>(); - sql.append(" WHERE time = ").append(records.get(0).getTimestamp()); - recordMap.put(records.get(0).getTimestamp(), records.get(0).getRecordDataValue()); - for (int i = 1; i < records.size(); i++) { - Record record = records.get(i); - sql.append(" or time = ").append(record.getTimestamp()); - recordMap.put(record.getTimestamp(), record.getRecordDataValue()); - } - int point = 0; - int line = 0; - try { - ISessionDataSet sessionDataSet = sessionWrapper.executeQueryStatement(sql.toString()); + public List verificationQueryImpl(String sql, Map> recordMap) + throws IoTDBConnectionException, StatementExecutionException { + int point = 0, line = 0; + try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql)) { while (sessionDataSet.hasNext()) { RowRecord rowRecord = sessionDataSet.next(); - long timeStamp = rowRecord.getTimestamp(); + // The table model and the tree model obtain time differently + long timeStamp = iotdb.getTimestamp(rowRecord); List values = recordMap.get(timeStamp); for (int i = 0; i < values.size(); i++) { - String value = rowRecord.getFields().get(i).toString(); + String value = iotdb.getValue(rowRecord, i); String target = String.valueOf(values.get(i)); if (!value.equals(target)) { LOGGER.error("Using SQL: " + sql + ",Expected:" + value + " but was: " + target); @@ -311,30 +380,14 @@ public Status verificationQuery(VerificationQuery verificationQuery) { } line++; } - sessionDataSet.close(); - } catch (Exception e) { - LOGGER.error("Query Error: " + sql); - return new Status(false, new TsdbException("Failed to query"), "Failed to query."); - } - if (recordMap.size() != line) { - LOGGER.error( - "Using SQL: " + sql + ",Expected line:" + recordMap.size() + " but was: " + line); } - return new Status(true, point); + return Arrays.asList(point, line); } @Override - public Status deviceQuery(DeviceQuery deviceQuery) { - DeviceSchema deviceSchema = deviceQuery.getDeviceSchema(); - String sql = - getDeviceQuerySql( - deviceSchema, deviceQuery.getStartTimestamp(), deviceQuery.getEndTimestamp()); - if (!config.isIS_QUIET_MODE()) { - LOGGER.info("IoTDB:" + sql); - } + public List> deviceQueryImpl(String sql) throws Exception { List> result = new ArrayList<>(); - try { - ISessionDataSet sessionDataSet = sessionWrapper.executeQueryStatement(sql); + try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql)) { while (sessionDataSet.hasNext()) { List line = new ArrayList<>(); RowRecord rowRecord = sessionDataSet.next(); @@ -345,201 +398,87 @@ public Status deviceQuery(DeviceQuery deviceQuery) { } result.add(line); } - sessionDataSet.close(); - } catch (Exception e) { - LOGGER.error("Query Error: " + sql + " exception:" + e.getMessage()); - return new Status(false, new TsdbException("Failed to query"), "Failed to query."); } - - return new Status(true, 0, sql, result); + return result; } @Override - public DeviceSummary deviceSummary(DeviceQuery deviceQuery) throws TsdbException { - DeviceSchema deviceSchema = deviceQuery.getDeviceSchema(); - int totalLineNumber; + public DeviceSummary deviceSummary( + String device, String totalLineNumberSql, String maxTimestampSql, String minTimestampSql) + throws TsdbException { + int totalLineNumber = 0; long minTimeStamp, maxTimeStamp; try { - ISessionDataSet sessionDataSet = - sessionWrapper.executeQueryStatement(getTotalLineNumberSql(deviceSchema)); - RowRecord rowRecord = sessionDataSet.next(); - totalLineNumber = Integer.parseInt(rowRecord.getFields().get(0).toString()); + SessionDataSet sessionDataSet = session.executeQueryStatement(totalLineNumberSql); + while (sessionDataSet.hasNext()) { + sessionDataSet.next(); + totalLineNumber++; + } sessionDataSet.close(); - sessionDataSet = sessionWrapper.executeQueryStatement(getMaxTimeStampSql(deviceSchema)); - rowRecord = sessionDataSet.next(); - maxTimeStamp = rowRecord.getTimestamp(); + sessionDataSet = session.executeQueryStatement(maxTimestampSql); + RowRecord rowRecord = sessionDataSet.next(); + maxTimeStamp = iotdb.getTimestamp(rowRecord); sessionDataSet.close(); - sessionDataSet = sessionWrapper.executeQueryStatement(getMinTimeStampSql(deviceSchema)); + sessionDataSet = session.executeQueryStatement(minTimestampSql); rowRecord = sessionDataSet.next(); - minTimeStamp = rowRecord.getTimestamp(); + minTimeStamp = iotdb.getTimestamp(rowRecord); sessionDataSet.close(); } catch (IoTDBConnectionException e) { throw new TsdbException("Failed to connect to IoTDB:" + e.getMessage()); } catch (StatementExecutionException e) { throw new TsdbException("Failed to execute statement:" + e.getMessage()); } - return new DeviceSummary(deviceSchema.getDevice(), totalLineNumber, minTimeStamp, maxTimeStamp); - } - - protected Tablet genTablet(IBatch batch) { - config.getWORKLOAD_BUFFER_SIZE(); - List schemaList = new ArrayList<>(); - int sensorIndex = 0; - for (Sensor sensor : batch.getDeviceSchema().getSensors()) { - SensorType dataSensorType = sensor.getSensorType(); - schemaList.add( - new MeasurementSchema( - sensor.getName(), - Enum.valueOf(TSDataType.class, dataSensorType.name), - Enum.valueOf(TSEncoding.class, getEncodingType(dataSensorType)))); - sensorIndex++; - } - String deviceId = getDevicePath(batch.getDeviceSchema()); - Tablet tablet = new Tablet(deviceId, schemaList, batch.getRecords().size()); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - - List sensors = batch.getDeviceSchema().getSensors(); - for (int recordIndex = 0; recordIndex < batch.getRecords().size(); recordIndex++) { - tablet.rowSize++; - Record record = batch.getRecords().get(recordIndex); - sensorIndex = 0; - long currentTime = record.getTimestamp(); - timestamps[recordIndex] = currentTime; - for (int recordValueIndex = 0; - recordValueIndex < record.getRecordDataValue().size(); - recordValueIndex++) { - switch (sensors.get(sensorIndex).getSensorType()) { - case BOOLEAN: - boolean[] sensorsBool = (boolean[]) values[recordValueIndex]; - sensorsBool[recordIndex] = - (boolean) (record.getRecordDataValue().get(recordValueIndex)); - break; - case INT32: - int[] sensorsInt = (int[]) values[recordValueIndex]; - sensorsInt[recordIndex] = (int) (record.getRecordDataValue().get(recordValueIndex)); - break; - case INT64: - long[] sensorsLong = (long[]) values[recordValueIndex]; - sensorsLong[recordIndex] = (long) (record.getRecordDataValue().get(recordValueIndex)); - break; - case FLOAT: - float[] sensorsFloat = (float[]) values[recordValueIndex]; - sensorsFloat[recordIndex] = (float) (record.getRecordDataValue().get(recordValueIndex)); - break; - case DOUBLE: - double[] sensorsDouble = (double[]) values[recordValueIndex]; - sensorsDouble[recordIndex] = - (double) (record.getRecordDataValue().get(recordValueIndex)); - break; - case TEXT: - case STRING: - case BLOB: - Binary[] sensorsText = (Binary[]) values[recordValueIndex]; - sensorsText[recordIndex] = - binaryCache.computeIfAbsent( - (String) record.getRecordDataValue().get(recordValueIndex), - BytesUtils::valueOf); - break; - case TIMESTAMP: - long[] sensorsTimestamp = (long[]) values[recordValueIndex]; - sensorsTimestamp[recordIndex] = - (long) (record.getRecordDataValue().get(recordValueIndex)); - break; - case DATE: - LocalDate[] sensorsDate = (LocalDate[]) values[recordValueIndex]; - sensorsDate[recordIndex] = - (LocalDate) (record.getRecordDataValue().get(recordValueIndex)); - break; - default: - LOGGER.error("Unsupported Type: {}", sensors.get(sensorIndex).getSensorType()); - } - sensorIndex++; - } - } - return tablet; + return new DeviceSummary(device, totalLineNumber, minTimeStamp, maxTimeStamp); } - public List constructDataTypes(List sensors, int recordValueSize) { - List dataTypes = new ArrayList<>(); - for (int sensorIndex = 0; sensorIndex < recordValueSize; sensorIndex++) { - switch (sensors.get(sensorIndex).getSensorType()) { - case BOOLEAN: - dataTypes.add(TSDataType.BOOLEAN); - break; - case INT32: - dataTypes.add(TSDataType.INT32); - break; - case INT64: - dataTypes.add(TSDataType.INT64); - break; - case FLOAT: - dataTypes.add(TSDataType.FLOAT); - break; - case DOUBLE: - dataTypes.add(TSDataType.DOUBLE); - break; - case TEXT: - dataTypes.add(TSDataType.TEXT); - break; - case STRING: - dataTypes.add(TSDataType.STRING); - break; - case BLOB: - dataTypes.add(TSDataType.BLOB); - break; - case TIMESTAMP: - dataTypes.add(TSDataType.TIMESTAMP); - break; - case DATE: - dataTypes.add(TSDataType.DATE); - break; - } + Status waitWriteTaskToFinishAndGetStatus() { + try { + task.get(config.getWRITE_OPERATION_TIMEOUT_MS(), TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + task.cancel(true); + LOGGER.error("insertion failed", e); + return new Status(false, 0, e, e.toString()); } - return dataTypes; + return new Status(true); } - public List convertTypeForBLOB(Record record, List dataTypes) { - // String change to Binary - List dataValue = record.getRecordDataValue(); - for (int recordValueIndex = 0; - recordValueIndex < record.getRecordDataValue().size(); - recordValueIndex++) { - if (Objects.requireNonNull(dataTypes.get(recordValueIndex)) == TSDataType.BLOB) { - dataValue.set( - recordValueIndex, - binaryCache.computeIfAbsent( - (String) record.getRecordDataValue().get(recordValueIndex), BytesUtils::valueOf)); + @Override + public void init() { + try { + if (config.isENABLE_THRIFT_COMPRESSION()) { + session.open(true); + } else { + session.open(); } + this.service = Executors.newSingleThreadExecutor(); + } catch (IoTDBConnectionException e) { + LOGGER.error("Failed to add session", e); } - return dataValue; } @Override - public Status insertOneBatch(IBatch batch) { - DBInsertMode insertMode = dbConfig.getDB_SWITCH().getInsertMode(); - switch (insertMode) { - case INSERT_USE_SESSION_TABLET: - return insertOneBatchByTablet(batch); - case INSERT_USE_SESSION_RECORD: - return insertOneBatchByRecord(batch); - case INSERT_USE_SESSION_RECORDS: - return insertOneBatchByRecords(batch); - default: - throw new IllegalStateException("Unexpected INSERT_MODE value: " + insertMode); + public void cleanup() { + try { + iotdb.sessionCleanupImpl(session); + } catch (IoTDBConnectionException e) { + LOGGER.warn("Failed to connect to IoTDB:" + e.getMessage()); + } catch (StatementExecutionException e) { + LOGGER.warn("Failed to execute statement:" + e.getMessage()); } } - Status waitWriteTaskToFinishAndGetStatus() { + @Override + public void close() throws TsdbException { try { - task.get(config.getWRITE_OPERATION_TIMEOUT_MS(), TimeUnit.MILLISECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - task.cancel(true); - LOGGER.error("insertion failed", e); - return new Status(false, 0, e, e.toString()); + if (session != null) { + session.close(); + } + service.shutdown(); + } catch (IoTDBConnectionException ioTDBConnectionException) { + LOGGER.error("Failed to close session."); + throw new TsdbException(ioTDBConnectionException); } - return new Status(true); } } diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IBenchmarkSession.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IBenchmarkSession.java deleted file mode 100644 index 955f31302..000000000 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IBenchmarkSession.java +++ /dev/null @@ -1,60 +0,0 @@ -package cn.edu.tsinghua.iot.benchmark.iotdb130; - -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.record.Tablet; - -import java.util.List; - -public interface IBenchmarkSession { - void open() throws IoTDBConnectionException; - - void open(boolean enableRPCCompression) throws IoTDBConnectionException; - - void insertRecord( - String deviceId, - long time, - List measurements, - List types, - List values) - throws IoTDBConnectionException, StatementExecutionException; - - void insertAlignedRecord( - String multiSeriesId, - long time, - List multiMeasurementComponents, - List types, - List values) - throws IoTDBConnectionException, StatementExecutionException; - - void insertRecords( - List deviceIds, - List times, - List> measurementsList, - List> typesList, - List> valuesList) - throws IoTDBConnectionException, StatementExecutionException; - - void insertAlignedRecords( - List multiSeriesIds, - List times, - List> multiMeasurementComponentsList, - List> typesList, - List> valuesList) - throws IoTDBConnectionException, StatementExecutionException; - - void insertTablet(Tablet tablet) throws IoTDBConnectionException, StatementExecutionException; - - void insertAlignedTablet(Tablet tablet) - throws IoTDBConnectionException, StatementExecutionException; - - ISessionDataSet executeQueryStatement(String sql) - throws IoTDBConnectionException, StatementExecutionException; - - void executeNonQueryStatement(String deleteSeriesSql) - throws IoTDBConnectionException, StatementExecutionException; - - void close() throws IoTDBConnectionException; -} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ISessionDataSet.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ISessionDataSet.java deleted file mode 100644 index ba097dd9b..000000000 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ISessionDataSet.java +++ /dev/null @@ -1,17 +0,0 @@ -package cn.edu.tsinghua.iot.benchmark.iotdb130; - -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; - -import org.apache.tsfile.read.common.RowRecord; - -public interface ISessionDataSet { - RowRecord next() throws IoTDBConnectionException, StatementExecutionException; - - boolean hasNext() throws IoTDBConnectionException, StatementExecutionException; - - void close() throws IoTDBConnectionException, StatementExecutionException; - - SessionDataSet.DataIterator iterator(); -} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDB.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDB.java index 9af12dfa6..8954835cc 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDB.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDB.java @@ -19,12 +19,10 @@ package cn.edu.tsinghua.iot.benchmark.iotdb130; -import org.apache.iotdb.isession.template.Template; import org.apache.iotdb.isession.util.Version; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; -import org.apache.iotdb.session.template.MeasurementNode; import cn.edu.tsinghua.iot.benchmark.client.operation.Operation; import cn.edu.tsinghua.iot.benchmark.conf.Config; @@ -33,14 +31,22 @@ import cn.edu.tsinghua.iot.benchmark.entity.DeviceSummary; import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; +import cn.edu.tsinghua.iot.benchmark.iotdb130.DMLStrategy.DMLStrategy; +import cn.edu.tsinghua.iot.benchmark.iotdb130.DMLStrategy.JDBCStrategy; +import cn.edu.tsinghua.iot.benchmark.iotdb130.DMLStrategy.SessionStrategy; +import cn.edu.tsinghua.iot.benchmark.iotdb130.ModelStrategy.IoTDBModelStrategy; +import cn.edu.tsinghua.iot.benchmark.iotdb130.ModelStrategy.TableStrategy; +import cn.edu.tsinghua.iot.benchmark.iotdb130.ModelStrategy.TreeStrategy; +import cn.edu.tsinghua.iot.benchmark.iotdb130.utils.IoTDBUtils; import cn.edu.tsinghua.iot.benchmark.measurement.Status; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; -import cn.edu.tsinghua.iot.benchmark.utils.NamedThreadFactory; +import cn.edu.tsinghua.iot.benchmark.utils.BlobUtils; import cn.edu.tsinghua.iot.benchmark.utils.TimeUtils; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.AggRangeQuery; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.AggRangeValueQuery; @@ -55,176 +61,106 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.time.LocalDate; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; import java.util.Set; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; /** this class will create more than one connection. */ public class IoTDB implements IDatabase { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDB.class); - private static final String ALREADY_KEYWORD = "already"; - private static final AtomicBoolean templateInit = new AtomicBoolean(false); - protected final String DELETE_SERIES_SQL; - private final String ORDER_BY_TIME_DESC = " order by time desc "; - protected SingleNodeJDBCConnection ioTDBConnection; - - protected static final Config config = ConfigDescriptor.getInstance().getConfig(); - protected static final CyclicBarrier templateBarrier = - new CyclicBarrier(config.getCLIENT_NUMBER()); - protected static final CyclicBarrier schemaBarrier = new CyclicBarrier(config.getCLIENT_NUMBER()); - protected static final CyclicBarrier activateTemplateBarrier = - new CyclicBarrier(config.getCLIENT_NUMBER()); - protected static Set storageGroups = Collections.synchronizedSet(new HashSet<>()); - protected final String ROOT_SERIES_NAME; - protected ExecutorService service; - protected Future task; - protected DBConfig dbConfig; - protected Random random = new Random(config.getDATA_SEED()); + + public static String DELETE_SERIES_SQL; + public static String ROOT_SERIES_NAME; + private final DBConfig dbConfig; + private final Random random = new Random(config.getDATA_SEED()); + private final DMLStrategy dmlStrategy; + private final IoTDBModelStrategy modelStrategy; + + public static final String ALREADY_KEYWORD = "already"; + private static final Config config = ConfigDescriptor.getInstance().getConfig(); + private static final String ORDER_BY_TIME_DESC = " order by time desc "; public IoTDB(DBConfig dbConfig) { this.dbConfig = dbConfig; ROOT_SERIES_NAME = "root." + dbConfig.getDB_NAME(); DELETE_SERIES_SQL = "delete storage group root." + dbConfig.getDB_NAME() + ".*"; + // init IoTDBModelStrategy and DMLStrategy + modelStrategy = + config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE + ? new TableStrategy(dbConfig) + : new TreeStrategy(dbConfig); + switch (dbConfig.getDB_SWITCH()) { + case DB_IOT_130_REST: + case DB_IOT_130_SESSION_BY_TABLET: + case DB_IOT_130_SESSION_BY_RECORD: + case DB_IOT_130_SESSION_BY_RECORDS: + dmlStrategy = new SessionStrategy(this, dbConfig); + break; + case DB_IOT_130_JDBC: + dmlStrategy = new JDBCStrategy(dbConfig); + break; + default: + throw new IllegalArgumentException("Unsupported DB SWITCH: " + dbConfig.getDB_SWITCH()); + } } @Override public void init() throws TsdbException { - if (ioTDBConnection == null) { - try { - ioTDBConnection = new SingleNodeJDBCConnection(dbConfig); - ioTDBConnection.init(); - this.service = - Executors.newSingleThreadExecutor(new NamedThreadFactory("DataClientExecuteJob")); - } catch (Exception e) { - throw new TsdbException(e); - } - } + dmlStrategy.init(); } @Override - public void cleanup() { - try (Statement statement = ioTDBConnection.getConnection().createStatement()) { - statement.execute(DELETE_SERIES_SQL); - LOGGER.info("Finish clean data!"); - } catch (Exception e) { - LOGGER.warn("No Data to Clean!"); - } + public void cleanup() throws TsdbException { + dmlStrategy.cleanup(); } @Override public void close() throws TsdbException { - if (ioTDBConnection != null) { - ioTDBConnection.close(); - } - if (service != null) { - service.shutdownNow(); - } - if (task != null) { - task.cancel(true); - } + dmlStrategy.close(); } + /** + * create timeseries one by one is too slow in current cluster server. therefore, we use session + * to create time series in batch. + * + * @param schemaList schema of devices to register + */ @Override public Double registerSchema(List schemaList) throws TsdbException { - // create timeseries one by one is too slow in current cluster server. - // therefore, we use session to create time series in batch. long start = System.nanoTime(); - long end; if (config.hasWrite()) { Map> sessionListMap = new HashMap<>(); try { // open meta session - if (!config.isIS_ALL_NODES_VISIBLE()) { - Session metaSession = - new Session.Builder() - .host(dbConfig.getHOST().get(0)) - .port(Integer.parseInt(dbConfig.getPORT().get(0))) - .username(dbConfig.getUSERNAME()) - .password(dbConfig.getPASSWORD()) - .version(Version.V_1_0) - .build(); - metaSession.open(config.isENABLE_THRIFT_COMPRESSION()); - sessionListMap.put(metaSession, createTimeseries(schemaList)); - } else { - int sessionNumber = dbConfig.getHOST().size(); - List keys = new ArrayList<>(); - for (int i = 0; i < sessionNumber; i++) { - Session metaSession = - new Session.Builder() - .host(dbConfig.getHOST().get(i)) - .port(Integer.parseInt(dbConfig.getPORT().get(i))) - .username(dbConfig.getUSERNAME()) - .password(dbConfig.getPASSWORD()) - .version(Version.V_1_0) - .build(); - metaSession.open(config.isENABLE_THRIFT_COMPRESSION()); - keys.add(metaSession); - sessionListMap.put(metaSession, new ArrayList<>()); - } - for (int i = 0; i < schemaList.size(); i++) { - sessionListMap - .get(keys.get(i % sessionNumber)) - .add(createTimeseries(schemaList.get(i))); - } - } - - if (config.isTEMPLATE() && templateInit.compareAndSet(false, true)) { - Template template = null; - if (config.isTEMPLATE() && schemaList.size() > 0) { - template = createTemplate(schemaList.get(0)); - } - start = System.nanoTime(); - int sessionIndex = random.nextInt(sessionListMap.size()); - Session templateSession = new ArrayList<>(sessionListMap.keySet()).get(sessionIndex); - registerTemplate(templateSession, template); - } else { - start = System.nanoTime(); - } - templateBarrier.await(); - for (Map.Entry> pair : sessionListMap.entrySet()) { - registerStorageGroups(pair.getKey(), pair.getValue()); - } - schemaBarrier.await(); - if (config.isTEMPLATE()) { - for (Map.Entry> pair : sessionListMap.entrySet()) { - activateTemplate(pair.getKey(), pair.getValue()); - } - activateTemplateBarrier.await(); - } - if (!config.isTEMPLATE()) { - for (Map.Entry> pair : sessionListMap.entrySet()) { - registerTimeseries(pair.getKey(), pair.getValue()); - } - } + Session metaSession = + new Session.Builder() + .host(dbConfig.getHOST().get(0)) + .port(Integer.parseInt(dbConfig.getPORT().get(0))) + .username(dbConfig.getUSERNAME()) + .password(dbConfig.getPASSWORD()) + .version(Version.V_1_0) + .sqlDialect(config.getIoTDB_DIALECT_MODE().name()) + .build(); + metaSession.open(config.isENABLE_THRIFT_COMPRESSION()); + sessionListMap.put(metaSession, createTimeseries(schemaList)); + modelStrategy.registerSchema(sessionListMap, schemaList); } catch (Exception e) { throw new TsdbException(e); } finally { - if (sessionListMap.size() != 0) { + if (!sessionListMap.isEmpty()) { Set sessions = sessionListMap.keySet(); for (Session session : sessions) { try { @@ -236,84 +172,17 @@ public Double registerSchema(List schemaList) throws TsdbException } } } - end = System.nanoTime(); + long end = System.nanoTime(); return TimeUtils.convertToSeconds(end - start, "ns"); } - /** create template */ - private Template createTemplate(DeviceSchema deviceSchema) { - Template template = null; - if (config.isTEMPLATE()) { - if (config.isVECTOR()) { - template = new Template(config.getTEMPLATE_NAME(), true); - } else { - template = new Template(config.getTEMPLATE_NAME(), false); - } - try { - for (Sensor sensor : deviceSchema.getSensors()) { - MeasurementNode measurementNode = - new MeasurementNode( - sensor.getName(), - Enum.valueOf(TSDataType.class, sensor.getSensorType().name), - Enum.valueOf(TSEncoding.class, getEncodingType(sensor.getSensorType())), - Enum.valueOf(CompressionType.class, config.getCOMPRESSOR())); - template.addToTemplate(measurementNode); - } - } catch (StatementExecutionException e) { - LOGGER.error(e.getMessage()); - return null; - } - } - return template; - } - - /** register template */ - private void registerTemplate(Session metaSession, Template template) - throws IoTDBConnectionException, IOException { - try { - metaSession.createSchemaTemplate(template); - } catch (StatementExecutionException e) { - // do nothing - e.printStackTrace(); - } - } - - private void registerStorageGroups(Session metaSession, List schemaList) - throws TsdbException { - // get all storage groups - Set groups = new HashSet<>(); - for (TimeseriesSchema timeseriesSchema : schemaList) { - DeviceSchema schema = timeseriesSchema.getDeviceSchema(); - synchronized (IoTDB.class) { - if (!storageGroups.contains(schema.getGroup())) { - groups.add(schema.getGroup()); - storageGroups.add(schema.getGroup()); - } - } - } - // register storage groups - for (String group : groups) { - try { - metaSession.setStorageGroup(ROOT_SERIES_NAME + "." + group); - if (config.isTEMPLATE()) { - metaSession.setSchemaTemplate(config.getTEMPLATE_NAME(), ROOT_SERIES_NAME + "." + group); - } - } catch (Exception e) { - handleRegisterException(e); - } - } - } - - private void activateTemplate(Session metaSession, List schemaList) { - try { - List devicePaths = - schemaList.stream() - .map(schema -> ROOT_SERIES_NAME + "." + schema.getDeviceSchema().getDevicePath()) - .collect(Collectors.toList()); - metaSession.createTimeseriesUsingSchemaTemplate(devicePaths); - } catch (Throwable t) { - t.printStackTrace(); + private List createTimeseries(List schemaList) { + List timeseriesSchemas = new ArrayList<>(); + for (DeviceSchema deviceSchema : schemaList) { + TimeseriesSchema timeseriesSchema = createTimeseries(deviceSchema); + timeseriesSchemas.add(timeseriesSchema); } + return timeseriesSchemas; } private TimeseriesSchema createTimeseries(DeviceSchema deviceSchema) { @@ -329,90 +198,34 @@ private TimeseriesSchema createTimeseries(DeviceSchema deviceSchema) { } SensorType datatype = sensor.getSensorType(); tsDataTypes.add(Enum.valueOf(TSDataType.class, datatype.name)); - tsEncodings.add(Enum.valueOf(TSEncoding.class, getEncodingType(datatype))); + tsEncodings.add( + Enum.valueOf(TSEncoding.class, Objects.requireNonNull(IoTDB.getEncodingType(datatype)))); compressionTypes.add(Enum.valueOf(CompressionType.class, config.getCOMPRESSOR())); } TimeseriesSchema timeseriesSchema = new TimeseriesSchema(deviceSchema, paths, tsDataTypes, tsEncodings, compressionTypes); if (config.isVECTOR()) { - timeseriesSchema.setDeviceId(getDevicePath(deviceSchema)); + timeseriesSchema.setDeviceId(IoTDBUtils.getDevicePath(deviceSchema, ROOT_SERIES_NAME)); } return timeseriesSchema; } - private List createTimeseries(List schemaList) { - List timeseriesSchemas = new ArrayList<>(); - for (DeviceSchema deviceSchema : schemaList) { - TimeseriesSchema timeseriesSchema = createTimeseries(deviceSchema); - timeseriesSchemas.add(timeseriesSchema); - } - return timeseriesSchemas; - } - - private void registerTimeseries(Session metaSession, List timeseriesSchemas) - throws TsdbException { - // create time series - for (TimeseriesSchema timeseriesSchema : timeseriesSchemas) { - try { - if (config.isVECTOR()) { - metaSession.createAlignedTimeseries( - timeseriesSchema.getDeviceId(), - timeseriesSchema.getPaths(), - timeseriesSchema.getTsDataTypes(), - timeseriesSchema.getTsEncodings(), - timeseriesSchema.getCompressionTypes(), - null); - } else { - metaSession.createMultiTimeseries( - timeseriesSchema.getPaths(), - timeseriesSchema.getTsDataTypes(), - timeseriesSchema.getTsEncodings(), - timeseriesSchema.getCompressionTypes(), - null, - null, - null, - null); - } - } catch (Exception e) { - handleRegisterException(e); - } - } - } - - private void handleRegisterException(Exception e) throws TsdbException { - // ignore if already has the time series - if (!e.getMessage().contains(ALREADY_KEYWORD) && !e.getMessage().contains("300")) { - LOGGER.error("Register IoTDB schema failed because ", e); - throw new TsdbException(e); - } - } - @Override public Status insertOneBatch(IBatch batch) throws DBConnectException { - try (Statement statement = ioTDBConnection.getConnection().createStatement()) { - for (Record record : batch.getRecords()) { - String sql = - getInsertOneBatchSql( - batch.getDeviceSchema(), record.getTimestamp(), record.getRecordDataValue()); - statement.addBatch(sql); - } - statement.executeBatch(); - return new Status(true); - } catch (Exception e) { - return new Status(false, 0, e, e.toString()); - } + String deviceId = IoTDBUtils.getDevicePath(batch.getDeviceSchema(), ROOT_SERIES_NAME); + return dmlStrategy.insertOneBatch(batch, deviceId); } /** * Q1: PreciseQuery SQL: select {sensors} from {devices} where time = {time} * * @param preciseQuery universal precise query condition parameters - * @return */ @Override public Status preciseQuery(PreciseQuery preciseQuery) { String strTime = preciseQuery.getTimestamp() + ""; String sql = getSimpleQuerySqlHead(preciseQuery.getDeviceSchema()) + " WHERE time = " + strTime; + sql = modelStrategy.addDeviceIDColumnIfNecessary(preciseQuery.getDeviceSchema(), sql); return executeQueryAndGetStatus(sql, Operation.PRECISE_QUERY); } @@ -421,7 +234,6 @@ public Status preciseQuery(PreciseQuery preciseQuery) { * {endTime} * * @param rangeQuery universal range query condition parameters - * @return */ @Override public Status rangeQuery(RangeQuery rangeQuery) { @@ -438,7 +250,6 @@ public Status rangeQuery(RangeQuery rangeQuery) { * {endTime} and {sensors} > {value} * * @param valueRangeQuery contains universal range query with value filter parameters - * @return */ @Override public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { @@ -451,7 +262,6 @@ public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { * time <= {endTime} * * @param aggRangeQuery contains universal aggregation query with time filter parameters - * @return */ @Override public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { @@ -467,7 +277,6 @@ public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { * Q5: AggValueQuery SQL: select {AggFun}({sensors}) from {devices} where {sensors} > {value} * * @param aggValueQuery contains universal aggregation query with value filter parameters - * @return */ @Override public Status aggValueQuery(AggValueQuery aggValueQuery) { @@ -488,7 +297,6 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { * * @param aggRangeValueQuery contains universal aggregation query with time and value filters * parameters - * @return */ @Override public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { @@ -510,7 +318,6 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { * {Granularity}ms) * * @param groupByQuery contains universal group by query condition parameters - * @return */ @Override public Status groupByQuery(GroupByQuery groupByQuery) { @@ -529,7 +336,6 @@ public Status groupByQuery(GroupByQuery groupByQuery) { * Q8: LatestPointQuery SQL: select last {sensors} from {devices} * * @param latestPointQuery contains universal latest point query condition parameters - * @return */ @Override public Status latestPointQuery(LatestPointQuery latestPointQuery) { @@ -542,7 +348,6 @@ public Status latestPointQuery(LatestPointQuery latestPointQuery) { * {endTime} order by time desc * * @param rangeQuery universal range query condition parameters - * @return */ @Override public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { @@ -560,7 +365,6 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { * {endTime} and {sensors} > {value} order by time desc * * @param valueRangeQuery contains universal range query with value filter parameters - * @return */ @Override public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { @@ -591,6 +395,7 @@ public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { protected String getSimpleQuerySqlHead(List devices) { StringBuilder builder = new StringBuilder(); builder.append("SELECT "); + builder.append(modelStrategy.selectTimeColumnIfNecessary()); List querySensors = devices.get(0).getSensors(); builder.append(querySensors.get(0).getName()); for (int i = 1; i < querySensors.size(); i++) { @@ -618,16 +423,10 @@ private String getAggQuerySqlHead(List devices, String aggFun) { /** * Add from Clause * - * @param devices - * @param builder * @return From clause, e.g. FROM devices */ private String addFromClause(List devices, StringBuilder builder) { - builder.append(" FROM ").append(getDevicePath(devices.get(0))); - for (int i = 1; i < devices.size(); i++) { - builder.append(", ").append(getDevicePath(devices.get(i))); - } - return builder.toString(); + return modelStrategy.addFromClause(devices, builder); } private String getValueRangeQuerySql(ValueRangeQuery valueRangeQuery) { @@ -644,21 +443,7 @@ private String getValueRangeQuerySql(ValueRangeQuery valueRangeQuery) { private String getValueFilterClause(List deviceSchemas, int valueThreshold) { StringBuilder builder = new StringBuilder(); - for (DeviceSchema deviceSchema : deviceSchemas) { - for (Sensor sensor : deviceSchema.getSensors()) { - builder - .append(" AND ") - .append(getDevicePath(deviceSchema)) - .append(".") - .append(sensor.getName()) - .append(" > "); - if (sensor.getSensorType() == SensorType.DATE) { - builder.append("'").append(LocalDate.ofEpochDay(Math.abs(valueThreshold))).append("'"); - } else { - builder.append(valueThreshold); - } - } - } + modelStrategy.getValueFilterClause(deviceSchemas, valueThreshold, builder); return builder.toString(); } @@ -674,7 +459,8 @@ private String getLatestPointQuerySql(List devices) { } private String getRangeQuerySql(List deviceSchemas, long start, long end) { - return addWhereTimeClause(getSimpleQuerySqlHead(deviceSchemas), start, end); + return modelStrategy.addDeviceIDColumnIfNecessary( + deviceSchemas, addWhereTimeClause(getSimpleQuerySqlHead(deviceSchemas), start, end)); } private String addWhereTimeClause(String prefix, long start, long end) { @@ -687,22 +473,6 @@ private String addGroupByClause(String prefix, long start, long end, long granul return prefix + " group by ([" + start + "," + end + ")," + granularity + "ms) "; } - /** - * convert deviceSchema to the format - * - * @param deviceSchema - * @return format, e.g. root.group_1.d_1 - */ - protected String getDevicePath(DeviceSchema deviceSchema) { - StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); - name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : deviceSchema.getTags().entrySet()) { - name.append(".").append(pair.getValue()); - } - name.append(".").append(deviceSchema.getDevice()); - return name.toString(); - } - protected Status executeQueryAndGetStatus(String sql, Operation operation) { String executeSQL; if (config.isIOTDB_USE_DEBUG() && random.nextDouble() < config.getIOTDB_USE_DEBUG_RATIO()) { @@ -713,124 +483,43 @@ protected Status executeQueryAndGetStatus(String sql, Operation operation) { if (!config.isIS_QUIET_MODE()) { LOGGER.info("{} query SQL: {}", Thread.currentThread().getName(), executeSQL); } - AtomicInteger line = new AtomicInteger(); - AtomicLong queryResultPointNum = new AtomicLong(); + + long queryResultPointNum = 0; AtomicBoolean isOk = new AtomicBoolean(true); - try (Statement statement = ioTDBConnection.getConnection().createStatement()) { - List> records = new ArrayList<>(); - task = - service.submit( - () -> { - try { - try (ResultSet resultSet = statement.executeQuery(executeSQL)) { - while (resultSet.next()) { - line.getAndIncrement(); - if (config.isIS_COMPARISON()) { - List record = new ArrayList<>(); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - switch (operation) { - case LATEST_POINT_QUERY: - if (i == 2 || i >= 4) { - continue; - } - break; - default: - break; - } - record.add(resultSet.getObject(i)); - } - records.add(record); - } - } - } - } catch (SQLException e) { - LOGGER.error("exception occurred when execute query={}", executeSQL, e); - isOk.set(false); - } - long resultPointNum = line.get(); - if (!Operation.LATEST_POINT_QUERY.equals(operation)) { - resultPointNum *= config.getQUERY_SENSOR_NUM(); - resultPointNum *= config.getQUERY_DEVICE_NUM(); - } - queryResultPointNum.set(resultPointNum); - }); - try { - task.get(config.getREAD_OPERATION_TIMEOUT_MS(), TimeUnit.MILLISECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - task.cancel(true); - return new Status(false, queryResultPointNum.get(), e, executeSQL); - } - if (isOk.get() == true) { - if (config.isIS_COMPARISON()) { - return new Status(true, queryResultPointNum.get(), executeSQL, records); - } else { - return new Status(true, queryResultPointNum.get()); - } - } else { - return new Status( - false, queryResultPointNum.get(), new Exception("Failed to execute."), executeSQL); - } - } catch (Exception e) { - return new Status(false, queryResultPointNum.get(), e, executeSQL); + List> records = new ArrayList<>(); + try { + queryResultPointNum = + dmlStrategy.executeQueryAndGetStatusImpl(executeSQL, operation, isOk, records); } catch (Throwable t) { - return new Status(false, queryResultPointNum.get(), new Exception(t), executeSQL); + return new Status(false, queryResultPointNum, new Exception(t), executeSQL); } - } - - public String getInsertOneBatchSql( - DeviceSchema deviceSchema, long timestamp, List values) { - StringBuilder builder = new StringBuilder("insert into "); - builder.append(getDevicePath(deviceSchema)).append("(timestamp"); - for (Sensor sensor : deviceSchema.getSensors()) { - builder.append(",").append(sensor.getName()); - } - if (config.isVECTOR() == true) { - builder.append(") aligned values("); - } else { - builder.append(") values("); - } - builder.append(timestamp); - int sensorIndex = 0; - List sensors = deviceSchema.getSensors(); - for (Object value : values) { - switch (sensors.get(sensorIndex).getSensorType()) { - case BOOLEAN: - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case TIMESTAMP: - case DATE: - builder.append(",").append(value); - break; - case TEXT: - case STRING: - builder.append(",").append("'").append(value).append("'"); - break; - case BLOB: - builder.append(",").append("X'").append(value).append("'"); - break; + if (isOk.get()) { + if (config.isIS_COMPARISON()) { + return new Status(true, queryResultPointNum, executeSQL, records); + } else { + return new Status(true, queryResultPointNum); } - sensorIndex++; + } else { + return new Status( + false, queryResultPointNum, new Exception("Failed to execute."), executeSQL); } - builder.append(")"); - LOGGER.debug("getInsertOneBatchSql: {}", builder); - return builder.toString(); } - /** - * Using in verification - * - * @param verificationQuery - */ + /** Using in verification */ @Override public Status verificationQuery(VerificationQuery verificationQuery) { DeviceSchema deviceSchema = verificationQuery.getDeviceSchema(); List deviceSchemas = new ArrayList<>(); deviceSchemas.add(deviceSchema); - List records = verificationQuery.getRecords(); - if (records == null || records.size() == 0) { + List records = new ArrayList<>(); + List tsDataTypes = + IoTDBUtils.constructDataTypes(deviceSchema.getSensors(), deviceSchema.getSensors().size()); + for (Record record : verificationQuery.getRecords()) { + records.add( + new Record(record.getTimestamp(), convertTypeForBlobAndDate(record, tsDataTypes))); + } + if (records.isEmpty()) { return new Status( false, new TsdbException("There are no records in verficationQuery."), @@ -840,42 +529,48 @@ public Status verificationQuery(VerificationQuery verificationQuery) { StringBuffer sql = new StringBuffer(); sql.append(getSimpleQuerySqlHead(deviceSchemas)); Map> recordMap = new HashMap<>(); - sql.append(" WHERE time = ").append(records.get(0).getTimestamp()); - recordMap.put(records.get(0).getTimestamp(), records.get(0).getRecordDataValue()); - for (int i = 1; i < records.size(); i++) { - Record record = records.get(i); - sql.append(" or time = ").append(record.getTimestamp()); - recordMap.put(record.getTimestamp(), record.getRecordDataValue()); - } - int point = 0; - int line = 0; - try (Statement statement = ioTDBConnection.getConnection().createStatement()) { - ResultSet resultSet = statement.executeQuery(sql.toString()); - while (resultSet.next()) { - long timeStamp = resultSet.getLong(1); - List values = recordMap.get(timeStamp); - for (int i = 0; i < values.size(); i++) { - String value = resultSet.getString(i + 2); - String target = String.valueOf(values.get(i)); - if (!value.equals(target)) { - LOGGER.error("Using SQL: " + sql + ",Expected:" + value + " but was: " + target); - } else { - point++; - } - } - line++; - } + modelStrategy.addVerificationQueryWhereClause(sql, records, recordMap, deviceSchema); + int point, line; + try { + List resultList = dmlStrategy.verificationQueryImpl(sql.toString(), recordMap); + point = resultList.get(0); + line = resultList.get(1); } catch (Exception e) { - LOGGER.error("Query Error: " + sql); + LOGGER.error("Query Error: {}", sql, e); return new Status(false, new TsdbException("Failed to query"), "Failed to query."); } if (recordMap.size() != line) { - LOGGER.error( - "Using SQL: " + sql + ",Expected line:" + recordMap.size() + " but was: " + line); + LOGGER.error("Using SQL: {},Expected line:{} but was: {}", sql, recordMap.size(), line); } return new Status(true, point); } + private List convertTypeForBlobAndDate(Record record, List dataTypes) { + List dataValue = record.getRecordDataValue(); + for (int recordValueIndex = 0; + recordValueIndex < record.getRecordDataValue().size(); + recordValueIndex++) { + switch (dataTypes.get(recordValueIndex)) { + case BLOB: + // "7I" to "0x3749" + dataValue.set( + recordValueIndex, + "0x" + + BlobUtils.stringToHex( + (String) record.getRecordDataValue().get(recordValueIndex)) + .toLowerCase()); + break; + case DATE: + // "2024-04-07" to "20240407" + String value = record.getRecordDataValue().get(recordValueIndex).toString(); + value = value.substring(0, 4) + value.substring(5, 7) + value.substring(8); + dataValue.set(recordValueIndex, value); + break; + } + } + return dataValue; + } + @Override public Status deviceQuery(DeviceQuery deviceQuery) throws SQLException, TsdbException { DeviceSchema deviceSchema = deviceQuery.getDeviceSchema(); @@ -883,25 +578,16 @@ public Status deviceQuery(DeviceQuery deviceQuery) throws SQLException, TsdbExce getDeviceQuerySql( deviceSchema, deviceQuery.getStartTimestamp(), deviceQuery.getEndTimestamp()); if (!config.isIS_QUIET_MODE()) { - LOGGER.info("IoTDB:" + sql); + LOGGER.info("Query: {}", sql); } - List> result = new ArrayList<>(); - try (Statement statement = ioTDBConnection.getConnection().createStatement()) { - ResultSet resultSet = statement.executeQuery(sql); - int colNumber = resultSet.getMetaData().getColumnCount(); - while (resultSet.next()) { - List line = new ArrayList<>(); - for (int i = 1; i <= colNumber; i++) { - line.add(resultSet.getObject(i)); - } - result.add(line); - } + List> result; + try { + result = dmlStrategy.deviceQueryImpl(sql); } catch (Exception e) { - LOGGER.error("Query Error: " + sql + " exception:" + e.getMessage()); + LOGGER.error("Query Error: {}", sql, e); return new Status(false, new TsdbException("Failed to query"), "Failed to query."); } - - return new Status(true, 0, sql.toString(), result); + return new Status(true, 0, sql, result); } protected String getDeviceQuerySql( @@ -918,38 +604,20 @@ protected String getDeviceQuerySql( @Override public DeviceSummary deviceSummary(DeviceQuery deviceQuery) throws SQLException, TsdbException { - DeviceSchema deviceSchema = deviceQuery.getDeviceSchema(); - int totalLineNumber = 0; - long minTimeStamp = 0, maxTimeStamp = 0; - try (Statement statement = ioTDBConnection.getConnection().createStatement()) { - ResultSet resultSet = statement.executeQuery(getTotalLineNumberSql(deviceSchema)); - resultSet.next(); - totalLineNumber = Integer.parseInt(resultSet.getString(1)); - - resultSet = statement.executeQuery(getMaxTimeStampSql(deviceSchema)); - resultSet.next(); - maxTimeStamp = Long.parseLong(resultSet.getObject(1).toString()); - - resultSet = statement.executeQuery(getMinTimeStampSql(deviceSchema)); - resultSet.next(); - minTimeStamp = Long.parseLong(resultSet.getObject(1).toString()); - } - return new DeviceSummary(deviceSchema.getDevice(), totalLineNumber, minTimeStamp, maxTimeStamp); - } - - protected String getTotalLineNumberSql(DeviceSchema deviceSchema) { - return "select count(*) from " + getDevicePath(deviceSchema); - } - - protected String getMinTimeStampSql(DeviceSchema deviceSchema) { - return "select * from " + getDevicePath(deviceSchema) + " order by time limit 1"; + DeviceSchema schema = deviceQuery.getDeviceSchema(); + return dmlStrategy.deviceSummary( + schema.getDevice(), + modelStrategy.getTotalLineNumberSql(schema), + modelStrategy.getMaxTimeStampSql(schema), + modelStrategy.getMinTimeStampSql(schema)); } - protected String getMaxTimeStampSql(DeviceSchema deviceSchema) { - return "select * from " + getDevicePath(deviceSchema) + " order by time desc limit 1"; + @Override + public String typeMap(SensorType iotdbSensorType) { + return IDatabase.super.typeMap(iotdbSensorType); } - String getEncodingType(SensorType dataSensorType) { + public static String getEncodingType(SensorType dataSensorType) { switch (dataSensorType) { case BOOLEAN: return config.getENCODING_BOOLEAN(); @@ -977,14 +645,48 @@ String getEncodingType(SensorType dataSensorType) { } } - /** - * convert deviceSchema and sensor to the format: root.group_1.d_1.s_1 - * - * @param deviceSchema - * @param sensor - * @return - */ - private String getSensorPath(DeviceSchema deviceSchema, String sensor) { - return getDevicePath(deviceSchema) + "." + sensor; + /** convert deviceSchema and sensor to the format: root.group_1.d_1.s_1 */ + public static String getSensorPath(DeviceSchema deviceSchema, String sensor) { + return IoTDBUtils.getDevicePath(deviceSchema, ROOT_SERIES_NAME) + "." + sensor; + } + + public String getInsertTargetName(DeviceSchema schema) { + return modelStrategy.getInsertTargetName(schema); + } + + public Tablet createTablet( + String insertTargetName, + List schemas, + List columnTypes, + int maxRowNumber) { + return modelStrategy.createTablet(insertTargetName, schemas, columnTypes, maxRowNumber); + } + + public void sessionCleanupImpl(Session session) + throws IoTDBConnectionException, StatementExecutionException { + modelStrategy.sessionCleanupImpl(session); + } + + public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) + throws IoTDBConnectionException, StatementExecutionException { + modelStrategy.sessionInsertImpl(session, tablet, deviceSchema); + } + + public void addIDColumnIfNecessary( + List columnTypes, List sensors, IBatch batch) { + modelStrategy.addIDColumnIfNecessary(columnTypes, sensors, batch); + } + + public void deleteIDColumnIfNecessary( + List columnTypes, List sensors, IBatch batch) { + modelStrategy.deleteIDColumnIfNecessary(columnTypes, sensors, batch); + } + + public long getTimestamp(RowRecord rowRecord) { + return modelStrategy.getTimestamp(rowRecord); + } + + public String getValue(RowRecord rowRecord, int i) { + return rowRecord.getFields().get(i + modelStrategy.getQueryOffset()).toString(); } } diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBClusterSession.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBClusterSession.java deleted file mode 100644 index 84c084366..000000000 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBClusterSession.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package cn.edu.tsinghua.iot.benchmark.iotdb130; - -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.isession.pool.SessionDataSetWrapper; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.pool.SessionPool; - -import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; -import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.RowRecord; -import org.apache.tsfile.write.record.Tablet; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Executors; - -public class IoTDBClusterSession extends IoTDBSessionBase { - private class BenchmarkSessionPool implements IBenchmarkSession { - private final SessionPool sessionPool; - - public BenchmarkSessionPool( - List hostUrls, - String user, - String password, - int maxSize, - boolean enableCompression, - boolean enableRedirection) { - this.sessionPool = - new SessionPool( - hostUrls, - dbConfig.getUSERNAME(), - dbConfig.getPASSWORD(), - MAX_SESSION_CONNECTION_PER_CLIENT, - config.isENABLE_THRIFT_COMPRESSION(), - true); - } - - @Override - public void open() { - // Do nothing - } - - @Override - public void open(boolean enableRPCCompression) { - // Do nothing - } - - @Override - public void insertRecord( - String deviceId, - long time, - List measurements, - List types, - List values) - throws IoTDBConnectionException, StatementExecutionException { - sessionPool.insertRecord(deviceId, time, measurements, types, values); - } - - @Override - public void insertAlignedRecord( - String multiSeriesId, - long time, - List multiMeasurementComponents, - List types, - List values) - throws IoTDBConnectionException, StatementExecutionException { - sessionPool.insertAlignedRecord( - multiSeriesId, time, multiMeasurementComponents, types, values); - } - - @Override - public void insertRecords( - List deviceIds, - List times, - List> measurementsList, - List> typesList, - List> valuesList) - throws IoTDBConnectionException, StatementExecutionException { - sessionPool.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); - } - - @Override - public void insertAlignedRecords( - List multiSeriesIds, - List times, - List> multiMeasurementComponentsList, - List> typesList, - List> valuesList) - throws IoTDBConnectionException, StatementExecutionException { - sessionPool.insertAlignedRecords( - multiSeriesIds, times, multiMeasurementComponentsList, typesList, valuesList); - } - - @Override - public void insertTablet(Tablet tablet) - throws IoTDBConnectionException, StatementExecutionException { - sessionPool.insertTablet(tablet); - } - - @Override - public void insertAlignedTablet(Tablet tablet) - throws IoTDBConnectionException, StatementExecutionException { - sessionPool.insertAlignedTablet(tablet); - } - - @Override - public ISessionDataSet executeQueryStatement(String sql) - throws IoTDBConnectionException, StatementExecutionException { - return new SessionDataSet2(sessionPool.executeQueryStatement(sql)); - } - - @Override - public void close() { - sessionPool.close(); - } - - @Override - public void executeNonQueryStatement(String deleteSeriesSql) - throws IoTDBConnectionException, StatementExecutionException { - sessionPool.executeNonQueryStatement(deleteSeriesSql); - } - - private class SessionDataSet2 implements ISessionDataSet { - SessionDataSetWrapper sessionDataSet; - - public SessionDataSet2(SessionDataSetWrapper sessionDataSetWrapper) { - this.sessionDataSet = sessionDataSetWrapper; - } - - @Override - public RowRecord next() throws IoTDBConnectionException, StatementExecutionException { - return sessionDataSet.next(); - } - - @Override - public boolean hasNext() throws IoTDBConnectionException, StatementExecutionException { - return sessionDataSet.hasNext(); - } - - @Override - public void close() throws IoTDBConnectionException, StatementExecutionException { - sessionDataSet.close(); - } - - @Override - public SessionDataSet.DataIterator iterator() { - return sessionDataSet.iterator(); - } - } - } - - private static final int MAX_SESSION_CONNECTION_PER_CLIENT = 3; - - public IoTDBClusterSession(DBConfig dbConfig) { - super(dbConfig); - LOGGER = LoggerFactory.getLogger(IoTDBClusterSession.class); - List hostUrls = new ArrayList<>(dbConfig.getHOST().size()); - for (int i = 0; i < dbConfig.getHOST().size(); i++) { - hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); - } - sessionWrapper = - new BenchmarkSessionPool( - hostUrls, - dbConfig.getUSERNAME(), - dbConfig.getPASSWORD(), - MAX_SESSION_CONNECTION_PER_CLIENT, - config.isENABLE_THRIFT_COMPRESSION(), - true); - } - - @Override - public void init() throws TsdbException { - // do nothing - this.service = Executors.newSingleThreadExecutor(); - } - - @Override - public void close() throws TsdbException { - if (sessionWrapper != null) { - try { - sessionWrapper.close(); - } catch (IoTDBConnectionException ignored) { - // should never happen - } - } - if (ioTDBConnection != null) { - ioTDBConnection.close(); - } - this.service.shutdown(); - } - - @Override - public void cleanup() { - try { - sessionWrapper.executeNonQueryStatement( - "drop database root." + config.getDbConfig().getDB_NAME() + ".**"); - } catch (IoTDBConnectionException e) { - LOGGER.error("Failed to connect to IoTDB:" + e.getMessage()); - } catch (StatementExecutionException e) { - LOGGER.error("Failed to execute statement:" + e.getMessage()); - } - - try { - sessionWrapper.executeNonQueryStatement("drop schema template " + config.getTEMPLATE_NAME()); - } catch (IoTDBConnectionException e) { - LOGGER.error("Failed to connect to IoTDB:" + e.getMessage()); - } catch (StatementExecutionException e) { - LOGGER.error("Failed to execute statement:" + e.getMessage()); - } - } -} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestAPI.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestAPI.java index 234f7f215..4758c2720 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestAPI.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestAPI.java @@ -137,4 +137,20 @@ private Status executeQueryAndGetStatus(String sql) { return new Status(false); } } + + private static class IoTDBRestPayload { + public String device; + public boolean is_aligned; + public List> values; + public List data_types; + public List measurements; + public List timestamps; + } + + private static class IoTDBRestQueryResult { + public List expressions; + public List column_names; + public List timestamps; + public List> values; + } } diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestPayload.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestPayload.java deleted file mode 100644 index 04a17bb72..000000000 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestPayload.java +++ /dev/null @@ -1,12 +0,0 @@ -package cn.edu.tsinghua.iot.benchmark.iotdb130; - -import java.util.List; - -public class IoTDBRestPayload { - public String device; - public boolean is_aligned; - public List> values; - public List data_types; - public List measurements; - public List timestamps; -} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestQueryResult.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestQueryResult.java deleted file mode 100644 index 11d6034a4..000000000 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBRestQueryResult.java +++ /dev/null @@ -1,10 +0,0 @@ -package cn.edu.tsinghua.iot.benchmark.iotdb130; - -import java.util.List; - -public class IoTDBRestQueryResult { - public List expressions; - public List column_names; - public List timestamps; - public List> values; -} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSession.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSession.java deleted file mode 100644 index 8faa5270e..000000000 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSession.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package cn.edu.tsinghua.iot.benchmark.iotdb130; - -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.isession.util.Version; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; - -import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; -import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.RowRecord; -import org.apache.tsfile.write.record.Tablet; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Executors; - -public class IoTDBSession extends IoTDBSessionBase { - private class BenchmarkSession implements IBenchmarkSession { - private final Session session; - - public BenchmarkSession(Session session) { - this.session = session; - } - - @Override - public void open() throws IoTDBConnectionException { - session.open(); - } - - @Override - public void open(boolean enableRPCCompression) throws IoTDBConnectionException { - session.open(enableRPCCompression); - } - - @Override - public void insertRecord( - String deviceId, - long time, - List measurements, - List types, - List values) - throws IoTDBConnectionException, StatementExecutionException { - session.insertRecord(deviceId, time, measurements, types, values); - } - - @Override - public void insertAlignedRecord( - String multiSeriesId, - long time, - List multiMeasurementComponents, - List types, - List values) - throws IoTDBConnectionException, StatementExecutionException { - session.insertAlignedRecord(multiSeriesId, time, multiMeasurementComponents, types, values); - } - - @Override - public void insertRecords( - List deviceIds, - List times, - List> measurementsList, - List> typesList, - List> valuesList) - throws IoTDBConnectionException, StatementExecutionException { - session.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); - } - - @Override - public void insertAlignedRecords( - List multiSeriesIds, - List times, - List> multiMeasurementComponentsList, - List> typesList, - List> valuesList) - throws IoTDBConnectionException, StatementExecutionException { - session.insertAlignedRecords( - multiSeriesIds, times, multiMeasurementComponentsList, typesList, valuesList); - } - - @Override - public void insertTablet(Tablet tablet) - throws IoTDBConnectionException, StatementExecutionException { - session.insertTablet(tablet); - } - - @Override - public void insertAlignedTablet(Tablet tablet) - throws IoTDBConnectionException, StatementExecutionException { - session.insertAlignedTablet(tablet); - } - - @Override - public ISessionDataSet executeQueryStatement(String sql) - throws IoTDBConnectionException, StatementExecutionException { - return new SessionDataSet1(session.executeQueryStatement(sql)); - } - - @Override - public void close() throws IoTDBConnectionException { - session.close(); - } - - @Override - public void executeNonQueryStatement(String deleteSeriesSql) - throws IoTDBConnectionException, StatementExecutionException { - session.executeNonQueryStatement(deleteSeriesSql); - } - - private class SessionDataSet1 implements ISessionDataSet { - SessionDataSet sessionDataSet; - - public SessionDataSet1(SessionDataSet sessionDataSet) { - this.sessionDataSet = sessionDataSet; - } - - @Override - public RowRecord next() throws IoTDBConnectionException, StatementExecutionException { - return sessionDataSet.next(); - } - - @Override - public boolean hasNext() throws IoTDBConnectionException, StatementExecutionException { - return sessionDataSet.hasNext(); - } - - @Override - public void close() throws IoTDBConnectionException, StatementExecutionException { - sessionDataSet.close(); - } - - @Override - public SessionDataSet.DataIterator iterator() { - return sessionDataSet.iterator(); - } - } - } - - public IoTDBSession(DBConfig dbConfig) { - super(dbConfig); - LOGGER = LoggerFactory.getLogger(IoTDBSession.class); - List hostUrls = new ArrayList<>(dbConfig.getHOST().size()); - for (int i = 0; i < dbConfig.getHOST().size(); i++) { - hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); - } - sessionWrapper = - new BenchmarkSession( - new Session.Builder() - .nodeUrls(hostUrls) - .username(dbConfig.getUSERNAME()) - .password(dbConfig.getPASSWORD()) - .enableRedirection(true) - .version(Version.V_1_0) - .build()); - } - - @Override - public void init() { - try { - if (config.isENABLE_THRIFT_COMPRESSION()) { - sessionWrapper.open(true); - } else { - sessionWrapper.open(); - } - this.service = Executors.newSingleThreadExecutor(); - } catch (IoTDBConnectionException e) { - LOGGER.error("Failed to add session", e); - } - } - - @Override - public void cleanup() { - try { - sessionWrapper.executeNonQueryStatement( - "drop database root." + config.getDbConfig().getDB_NAME() + ".**"); - } catch (IoTDBConnectionException e) { - LOGGER.error("Failed to connect to IoTDB:" + e.getMessage()); - } catch (StatementExecutionException e) { - LOGGER.error("Failed to execute statement:" + e.getMessage()); - } - - try { - sessionWrapper.executeNonQueryStatement("drop schema template " + config.getTEMPLATE_NAME()); - } catch (IoTDBConnectionException e) { - LOGGER.error("Failed to connect to IoTDB:" + e.getMessage()); - } catch (StatementExecutionException e) { - LOGGER.error("Failed to execute statement:" + e.getMessage()); - } - } - - @Override - public void close() throws TsdbException { - try { - if (sessionWrapper != null) { - sessionWrapper.close(); - } - if (ioTDBConnection != null) { - ioTDBConnection.close(); - } - this.service.shutdown(); - } catch (IoTDBConnectionException ioTDBConnectionException) { - LOGGER.error("Failed to close session."); - throw new TsdbException(ioTDBConnectionException); - } - } -} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/IoTDBModelStrategy.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/IoTDBModelStrategy.java new file mode 100644 index 000000000..bab6ff812 --- /dev/null +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/IoTDBModelStrategy.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cn.edu.tsinghua.iot.benchmark.iotdb130.ModelStrategy; + +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import cn.edu.tsinghua.iot.benchmark.conf.Config; +import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; +import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; +import cn.edu.tsinghua.iot.benchmark.entity.Record; +import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB; +import cn.edu.tsinghua.iot.benchmark.iotdb130.TimeseriesSchema; +import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; +import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; +import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class IoTDBModelStrategy { + protected static final Config config = ConfigDescriptor.getInstance().getConfig(); + protected final DBConfig dbConfig; + protected static String ROOT_SERIES_NAME; + protected static int queryBaseOffset; + protected static final Set databases = new HashSet<>(); + + public IoTDBModelStrategy(DBConfig dbConfig) { + this.dbConfig = dbConfig; + } + + public abstract void registerSchema( + Map> sessionListMap, List schemaList) + throws TsdbException; + + // region select + + public abstract String selectTimeColumnIfNecessary(); + + public abstract String addFromClause(List devices, StringBuilder builder); + + public abstract String addDeviceIDColumnIfNecessary(List deviceSchemas, String sql); + + public abstract void deleteIDColumnIfNecessary( + List columnTypes, List sensors, IBatch batch); + + public abstract void addVerificationQueryWhereClause( + StringBuffer sql, + List records, + Map> recordMap, + DeviceSchema deviceSchema); + + public abstract void getValueFilterClause( + List deviceSchemas, int valueThreshold, StringBuilder builder); + + public abstract long getTimestamp(RowRecord rowRecord); + + public abstract int getQueryOffset(); + + public abstract String getTotalLineNumberSql(DeviceSchema deviceSchema); + + public abstract String getMaxTimeStampSql(DeviceSchema deviceSchema); + + public abstract String getMinTimeStampSql(DeviceSchema deviceSchema); + + // endregion + + // region insert + + public Set getAllDataBase(List schemaList) { + Set databaseNames = new HashSet<>(); + for (TimeseriesSchema timeseriesSchema : schemaList) { + DeviceSchema schema = timeseriesSchema.getDeviceSchema(); + synchronized (IoTDB.class) { + if (!databases.contains(schema.getGroup())) { + databaseNames.add(schema.getGroup()); + databases.add(schema.getGroup()); + } + } + } + return databaseNames; + } + + public abstract void registerDatabases(Session metaSession, List schemaList) + throws TsdbException; + + public abstract Tablet createTablet( + String insertTargetName, + List schemas, + List columnTypes, + int maxRowNumber); + + public abstract String getInsertTargetName(DeviceSchema schema); + + public abstract void addIDColumnIfNecessary( + List columnTypes, List sensors, IBatch batch); + + public abstract void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) + throws IoTDBConnectionException, StatementExecutionException; + + public abstract void sessionCleanupImpl(Session session) + throws IoTDBConnectionException, StatementExecutionException; + + // endregion + + public abstract Logger getLogger(); + + public void handleRegisterException(Exception e) throws TsdbException { + // ignore if already has the time series + if (!e.getMessage().contains(IoTDB.ALREADY_KEYWORD) && !e.getMessage().contains("300")) { + Logger LOGGER = getLogger(); + LOGGER.error("Register IoTDB schema failed because ", e); + throw new TsdbException(e); + } + } +} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/TableStrategy.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/TableStrategy.java new file mode 100644 index 000000000..457f69a97 --- /dev/null +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/TableStrategy.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cn.edu.tsinghua.iot.benchmark.iotdb130.ModelStrategy; + +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; +import cn.edu.tsinghua.iot.benchmark.entity.Record; +import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; +import cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB; +import cn.edu.tsinghua.iot.benchmark.iotdb130.TimeseriesSchema; +import cn.edu.tsinghua.iot.benchmark.mode.enums.BenchmarkMode; +import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; +import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; +import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.LocalDate; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CyclicBarrier; + +public class TableStrategy extends IoTDBModelStrategy { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableStrategy.class); + private static final CyclicBarrier schemaBarrier = new CyclicBarrier(config.getCLIENT_NUMBER()); + + public TableStrategy(DBConfig dbConfig) { + super(dbConfig); + ROOT_SERIES_NAME = IoTDB.ROOT_SERIES_NAME; + queryBaseOffset = 1; + } + + @Override + public void registerSchema( + Map> sessionListMap, List schemaList) + throws TsdbException { + try { + for (Map.Entry> pair : sessionListMap.entrySet()) { + registerDatabases(pair.getKey(), pair.getValue()); + } + schemaBarrier.await(); + for (Map.Entry> pair : sessionListMap.entrySet()) { + registerTable(pair.getKey(), pair.getValue()); + } + } catch (Exception e) { + throw new TsdbException(e); + } + } + + /** root.test.g_0.d_0 test is the database name.Ensure that only one client creates the table. */ + @Override + public void registerDatabases(Session metaSession, List schemaList) + throws TsdbException { + Set databaseNames = getAllDataBase(schemaList); + // register storage groups + for (String databaseName : databaseNames) { + try { + metaSession.executeNonQueryStatement( + "CREATE DATABASE " + dbConfig.getDB_NAME() + "_" + databaseName); + } catch (Exception e) { + handleRegisterException(e); + } + } + } + + private void registerTable(Session metaSession, List timeseriesSchemas) + throws TsdbException { + try { + HashMap tables = new HashMap<>(); + for (TimeseriesSchema timeseriesSchema : timeseriesSchemas) { + DeviceSchema deviceSchema = timeseriesSchema.getDeviceSchema(); + StringBuilder builder = new StringBuilder(); + builder.append("create table if not exists ").append(deviceSchema.getTable()).append("("); + for (int i = 0; i < deviceSchema.getSensors().size(); i++) { + if (i != 0) builder.append(", "); + builder + .append(deviceSchema.getSensors().get(i).getName()) + .append(" ") + .append(deviceSchema.getSensors().get(i).getSensorType()) + .append(" ") + .append(deviceSchema.getSensors().get(i).getColumnCategory()); + } + builder + .append(", device_id") + .append(" ") + .append(SensorType.STRING) + .append(" ") + .append(Tablet.ColumnType.ID); + for (String key : deviceSchema.getTags().keySet()) { + builder + .append(", ") + .append(key) + .append(" ") + .append(SensorType.STRING) + .append(" ") + .append(Tablet.ColumnType.ID); + } + builder.append(")"); + tables.put(dbConfig.getDB_NAME() + "_" + deviceSchema.getGroup(), builder.toString()); + } + + for (Map.Entry database : tables.entrySet()) { + metaSession.executeNonQueryStatement("use " + database.getKey()); + metaSession.executeNonQueryStatement(database.getValue()); + } + } catch (Exception e) { + handleRegisterException(e); + } + } + + // region select + + @Override + public String selectTimeColumnIfNecessary() { + if (config.getBENCHMARK_WORK_MODE() == BenchmarkMode.VERIFICATION_QUERY) { + return "time, "; + } else { + return ""; + } + } + + @Override + public String addFromClause(List devices, StringBuilder builder) { + // "root.test.g_0.d_0.s_0" + // tree mode: select ... from root.test.g_0.d_0 + // table mode： select ... from test_g_0.table_0 + builder + .append(" FROM ") + .append(dbConfig.getDB_NAME()) + .append("_") + .append(devices.get(0).getGroup()) + .append(".") + .append(devices.get(0).getTable()); + return builder.toString(); + } + + @Override + public String addDeviceIDColumnIfNecessary(List deviceSchemas, String sql) { + Set deviceIds = new HashSet<>(); + StringBuilder builder = new StringBuilder(); + builder + .append(sql) + .append(" AND (") + .append("device_id") + .append(" = '") + .append(deviceSchemas.get(0).getDevice()) + .append("'"); + for (int i = 1; i < deviceSchemas.size(); i++) { + if (!deviceIds.contains(deviceSchemas.get(i).getDevice())) { + deviceIds.add(deviceSchemas.get(i).getDevice()); + builder + .append(" OR ") + .append("device_id") + .append(" = '") + .append(deviceSchemas.get(i).getDevice()) + .append("'"); + } + } + builder.append(")"); + return builder.toString(); + } + + @Override + public void addVerificationQueryWhereClause( + StringBuffer sql, + List records, + Map> recordMap, + DeviceSchema deviceSchema) { + sql.append(" WHERE (time = ").append(records.get(0).getTimestamp()); + recordMap.put(records.get(0).getTimestamp(), records.get(0).getRecordDataValue()); + for (int i = 1; i < records.size(); i++) { + Record record = records.get(i); + sql.append(" or time = ").append(record.getTimestamp()); + recordMap.put(record.getTimestamp(), record.getRecordDataValue()); + } + sql.append(" ) AND device_id = '").append(deviceSchema.getDevice()).append("'"); + Map tags = deviceSchema.getTags(); + if (tags != null) { + for (Map.Entry entry : tags.entrySet()) { + sql.append(" AND ") + .append(entry.getKey()) + .append(" = '") + .append(entry.getValue()) + .append("'"); + } + } + } + + @Override + public void getValueFilterClause( + List deviceSchemas, int valueThreshold, StringBuilder builder) { + DeviceSchema deviceSchema = deviceSchemas.get(0); + for (Sensor sensor : deviceSchema.getSensors()) { + builder.append(" AND ").append(sensor.getName()).append(" > "); + if (sensor.getSensorType() == SensorType.DATE) { + builder + .append("CAST(") + .append("'") + .append(LocalDate.ofEpochDay(Math.abs(valueThreshold))) + .append("'") + .append(" AS DATE)"); + } else { + builder.append(valueThreshold); + } + } + } + + @Override + public long getTimestamp(RowRecord rowRecord) { + return rowRecord.getFields().get(0).getLongV(); + } + + @Override + public int getQueryOffset() { + return queryBaseOffset; + } + + // TODO 用 count + @Override + public String getTotalLineNumberSql(DeviceSchema deviceSchema) { + return "select * from " + + dbConfig.getDB_NAME() + + "_" + + deviceSchema.getGroup() + + "." + + deviceSchema.getTable() + + " where device_id = '" + + deviceSchema.getDevice() + + "'"; + } + + @Override + public String getMaxTimeStampSql(DeviceSchema deviceSchema) { + return "select * from " + + dbConfig.getDB_NAME() + + "_" + + deviceSchema.getGroup() + + "." + + deviceSchema.getTable() + + " order by time desc limit 1"; + } + + @Override + public String getMinTimeStampSql(DeviceSchema deviceSchema) { + return "select * from " + + dbConfig.getDB_NAME() + + "_" + + deviceSchema.getGroup() + + "." + + deviceSchema.getTable() + + " order by time limit 1"; + } + + // endregion + + // region insert + + @Override + public Tablet createTablet( + String insertTargetName, + List schemas, + List columnTypes, + int maxRowNumber) { + return new Tablet(insertTargetName, schemas, columnTypes, maxRowNumber); + } + + @Override + public String getInsertTargetName(DeviceSchema schema) { + return schema.getTable(); + } + + @Override + public void addIDColumnIfNecessary( + List columnTypes, List sensors, IBatch batch) { + // All sensors are of type measurement + for (int i = 0; i < sensors.size(); i++) { + columnTypes.add(Tablet.ColumnType.MEASUREMENT); + } + // tag and device as ID column + // Add Identity Column Information to Schema + sensors.add(new Sensor("device_id", SensorType.STRING)); + columnTypes.add(Tablet.ColumnType.ID); + for (String key : batch.getDeviceSchema().getTags().keySet()) { + // Currently, the identity column can only be String + sensors.add(new Sensor(key, SensorType.STRING)); + columnTypes.add(Tablet.ColumnType.ID); + } + // Add the value of the identity column to the value of each record + for (int i = 0; i < batch.getRecords().size(); i++) { + List dataValue = batch.getRecords().get(i).getRecordDataValue(); + dataValue.add(batch.getDeviceSchema().getDevice()); + for (String key : batch.getDeviceSchema().getTags().keySet()) { + dataValue.add(batch.getDeviceSchema().getTags().get(key)); + } + } + } + + @Override + public void deleteIDColumnIfNecessary( + List columnTypes, List sensors, IBatch batch) { + // do nothing + } + + @Override + public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement( + "use " + dbConfig.getDB_NAME() + "_" + deviceSchema.getGroup()); + session.insertRelationalTablet(tablet); + } + + @Override + public void sessionCleanupImpl(Session session) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = session.executeQueryStatement("show databases"); + while (dataSet.hasNext()) { + String databaseName = dataSet.next().getFields().get(0).toString(); + if (databaseName.contains(".")) { + continue; + } + session.executeNonQueryStatement("drop database " + databaseName); + } + } + + // endregion + + @Override + public Logger getLogger() { + return LOGGER; + } +} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/TreeStrategy.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/TreeStrategy.java new file mode 100644 index 000000000..e8ee40d9e --- /dev/null +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ModelStrategy/TreeStrategy.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cn.edu.tsinghua.iot.benchmark.iotdb130.ModelStrategy; + +import org.apache.iotdb.isession.template.Template; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.template.MeasurementNode; + +import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; +import cn.edu.tsinghua.iot.benchmark.entity.Record; +import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; +import cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB; +import cn.edu.tsinghua.iot.benchmark.iotdb130.TimeseriesSchema; +import cn.edu.tsinghua.iot.benchmark.iotdb130.utils.IoTDBUtils; +import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; +import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; +import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +public class TreeStrategy extends IoTDBModelStrategy { + + private static final Logger LOGGER = LoggerFactory.getLogger(TreeStrategy.class); + + private final Random random = new Random(config.getDATA_SEED()); + private static final CyclicBarrier templateBarrier = new CyclicBarrier(config.getCLIENT_NUMBER()); + private static final CyclicBarrier schemaBarrier = new CyclicBarrier(config.getCLIENT_NUMBER()); + private static final CyclicBarrier activateTemplateBarrier = + new CyclicBarrier(config.getCLIENT_NUMBER()); + private static final AtomicBoolean templateInit = new AtomicBoolean(false); + + public TreeStrategy(DBConfig dbConfig) { + super(dbConfig); + ROOT_SERIES_NAME = IoTDB.ROOT_SERIES_NAME; + queryBaseOffset = 0; + } + + @Override + public void registerSchema( + Map> sessionListMap, List schemaList) + throws TsdbException { + try { + if (config.isTEMPLATE() && templateInit.compareAndSet(false, true)) { + Template template = null; + if (config.isTEMPLATE() && !schemaList.isEmpty()) { + template = createTemplate(schemaList.get(0)); + } + int sessionIndex = random.nextInt(sessionListMap.size()); + Session templateSession = new ArrayList<>(sessionListMap.keySet()).get(sessionIndex); + registerTemplate(templateSession, template); + } + templateBarrier.await(); + for (Map.Entry> pair : sessionListMap.entrySet()) { + registerDatabases(pair.getKey(), pair.getValue()); + } + schemaBarrier.await(); + if (config.isTEMPLATE()) { + for (Map.Entry> pair : sessionListMap.entrySet()) { + activateTemplate(pair.getKey(), pair.getValue()); + } + activateTemplateBarrier.await(); + } + if (!config.isTEMPLATE()) { + for (Map.Entry> pair : sessionListMap.entrySet()) { + registerTimeSeries(pair.getKey(), pair.getValue()); + } + } + } catch (Exception e) { + throw new TsdbException(e); + } + } + + /** create template */ + private Template createTemplate(DeviceSchema deviceSchema) { + Template template = null; + if (config.isTEMPLATE()) { + if (config.isVECTOR()) { + template = new Template(config.getTEMPLATE_NAME(), true); + } else { + template = new Template(config.getTEMPLATE_NAME(), false); + } + try { + for (Sensor sensor : deviceSchema.getSensors()) { + MeasurementNode measurementNode = + new MeasurementNode( + sensor.getName(), + Enum.valueOf(TSDataType.class, sensor.getSensorType().name), + Enum.valueOf( + TSEncoding.class, + Objects.requireNonNull(IoTDB.getEncodingType(sensor.getSensorType()))), + Enum.valueOf(CompressionType.class, config.getCOMPRESSOR())); + template.addToTemplate(measurementNode); + } + } catch (StatementExecutionException e) { + LOGGER.error(e.getMessage()); + return null; + } + } + return template; + } + + /** register template */ + private void registerTemplate(Session metaSession, Template template) + throws IoTDBConnectionException, IOException, TsdbException { + try { + metaSession.createSchemaTemplate(template); + } catch (StatementExecutionException e) { + // do nothing + handleRegisterException(e); + } + } + + @Override + public void registerDatabases(Session metaSession, List schemaList) + throws TsdbException { + // get all database + Set databaseNames = getAllDataBase(schemaList); + // register database + for (String databaseName : databaseNames) { + try { + metaSession.setStorageGroup(ROOT_SERIES_NAME + "." + databaseName); + if (config.isTEMPLATE()) { + metaSession.setSchemaTemplate( + config.getTEMPLATE_NAME(), ROOT_SERIES_NAME + "." + databaseName); + } + } catch (Exception e) { + handleRegisterException(e); + } + } + } + + private void activateTemplate(Session metaSession, List schemaList) { + try { + List devicePaths = + schemaList.stream() + .map(schema -> ROOT_SERIES_NAME + "." + schema.getDeviceSchema().getDevicePath()) + .collect(Collectors.toList()); + metaSession.createTimeseriesUsingSchemaTemplate(devicePaths); + } catch (Throwable t) { + t.printStackTrace(); + } + } + + private void registerTimeSeries(Session metaSession, List timeseriesSchemas) + throws TsdbException { + // create time series + for (TimeseriesSchema timeseriesSchema : timeseriesSchemas) { + try { + if (config.isVECTOR()) { + metaSession.createAlignedTimeseries( + timeseriesSchema.getDeviceId(), + timeseriesSchema.getPaths(), + timeseriesSchema.getTsDataTypes(), + timeseriesSchema.getTsEncodings(), + timeseriesSchema.getCompressionTypes(), + null); + } else { + metaSession.createMultiTimeseries( + timeseriesSchema.getPaths(), + timeseriesSchema.getTsDataTypes(), + timeseriesSchema.getTsEncodings(), + timeseriesSchema.getCompressionTypes(), + null, + null, + null, + null); + } + } catch (Exception e) { + handleRegisterException(e); + } + } + } + + // region select + + @Override + public String selectTimeColumnIfNecessary() { + return ""; + } + + @Override + public String addFromClause(List devices, StringBuilder builder) { + builder.append(" FROM ").append(IoTDBUtils.getDevicePath(devices.get(0), ROOT_SERIES_NAME)); + for (int i = 1; i < devices.size(); i++) { + builder.append(", ").append(IoTDBUtils.getDevicePath(devices.get(i), ROOT_SERIES_NAME)); + } + return builder.toString(); + } + + @Override + public String addDeviceIDColumnIfNecessary(List deviceSchemas, String sql) { + // do nothing + return sql; + } + + @Override + public void addVerificationQueryWhereClause( + StringBuffer sql, + List records, + Map> recordMap, + DeviceSchema deviceSchema) { + sql.append(" WHERE time = ").append(records.get(0).getTimestamp()); + recordMap.put(records.get(0).getTimestamp(), records.get(0).getRecordDataValue()); + for (int i = 1; i < records.size(); i++) { + Record record = records.get(i); + sql.append(" or time = ").append(record.getTimestamp()); + recordMap.put(record.getTimestamp(), record.getRecordDataValue()); + } + } + + @Override + public void getValueFilterClause( + List deviceSchemas, int valueThreshold, StringBuilder builder) { + for (DeviceSchema deviceSchema : deviceSchemas) { + for (Sensor sensor : deviceSchema.getSensors()) { + builder + .append(" AND ") + .append(IoTDBUtils.getDevicePath(deviceSchema, ROOT_SERIES_NAME)) + .append(".") + .append(sensor.getName()) + .append(" > "); + if (sensor.getSensorType() == SensorType.DATE) { + builder.append("'").append(LocalDate.ofEpochDay(Math.abs(valueThreshold))).append("'"); + } else { + builder.append(valueThreshold); + } + } + } + } + + @Override + public long getTimestamp(RowRecord rowRecord) { + return rowRecord.getTimestamp(); + } + + @Override + public int getQueryOffset() { + return queryBaseOffset; + } + + // TODO 用 count + @Override + public String getTotalLineNumberSql(DeviceSchema deviceSchema) { + return "select * from " + IoTDBUtils.getDevicePath(deviceSchema, ROOT_SERIES_NAME); + } + + @Override + public String getMaxTimeStampSql(DeviceSchema deviceSchema) { + return "select * from " + + IoTDBUtils.getDevicePath(deviceSchema, ROOT_SERIES_NAME) + + " order by time desc limit 1"; + } + + @Override + public String getMinTimeStampSql(DeviceSchema deviceSchema) { + return "select * from " + + IoTDBUtils.getDevicePath(deviceSchema, ROOT_SERIES_NAME) + + " order by time limit 1"; + } + + // endregion + + // region insert + @Override + public Tablet createTablet( + String insertTargetName, + List schemas, + List columnTypes, + int maxRowNumber) { + return new Tablet(insertTargetName, schemas, maxRowNumber); + } + + @Override + public String getInsertTargetName(DeviceSchema schema) { + return IoTDBUtils.getDevicePath(schema, ROOT_SERIES_NAME); + } + + @Override + public void addIDColumnIfNecessary( + List columnTypes, List sensors, IBatch batch) { + // do nothing + } + + @Override + public void deleteIDColumnIfNecessary( + List columnTypes, List sensors, IBatch batch) { + // delete the value of the identity column to the value of each record + for (int i = 0; i < batch.getRecords().size(); i++) { + List dataValue = batch.getRecords().get(i).getRecordDataValue(); + dataValue.remove(batch.getDeviceSchema().getDevice()); + for (String key : batch.getDeviceSchema().getTags().keySet()) { + dataValue.remove(batch.getDeviceSchema().getTags().get(key)); + } + } + } + + @Override + public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) + throws IoTDBConnectionException, StatementExecutionException { + if (config.isVECTOR()) { + session.insertAlignedTablet(tablet); + } else { + session.insertTablet(tablet); + } + } + + @Override + public void sessionCleanupImpl(Session session) { + try { + session.executeNonQueryStatement( + "drop database root." + config.getDbConfig().getDB_NAME() + ".**"); + session.executeNonQueryStatement("drop schema template " + config.getTEMPLATE_NAME()); + } catch (IoTDBConnectionException e) { + LOGGER.warn("Failed to connect to IoTDB:{}", e.getMessage()); + } catch (StatementExecutionException e) { + LOGGER.warn("Failed to execute statement:{}", e.getMessage()); + } + } + + // endregion + + @Override + public Logger getLogger() { + return LOGGER; + } +} diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/SingleNodeJDBCConnection.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/SingleNodeJDBCConnection.java index 697d1af4b..41838acfe 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/SingleNodeJDBCConnection.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/SingleNodeJDBCConnection.java @@ -29,7 +29,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.List; import java.util.concurrent.atomic.AtomicInteger; public class SingleNodeJDBCConnection { @@ -50,19 +49,8 @@ public SingleNodeJDBCConnection(DBConfig dbConfig) { public void init() throws TsdbException { int nodeSize = 1; String[] urls; - if (config.isIS_ALL_NODES_VISIBLE()) { - nodeSize = dbConfig.getHOST().size(); - urls = new String[nodeSize]; - List clusterHosts = dbConfig.getHOST(); - for (int i = 0; i < nodeSize; i++) { - String jdbcUrl = - String.format(JDBC_URL, dbConfig.getHOST().get(i), dbConfig.getPORT().get(i)); - urls[i] = jdbcUrl; - } - } else { - urls = new String[nodeSize]; - urls[0] = String.format(JDBC_URL, dbConfig.getHOST().get(0), dbConfig.getPORT().get(0)); - } + urls = new String[nodeSize]; + urls[0] = String.format(JDBC_URL, dbConfig.getHOST().get(0), dbConfig.getPORT().get(0)); connections = new Connection[nodeSize]; for (int i = 0; i < connections.length; i++) { diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/utils/IoTDBUtils.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/utils/IoTDBUtils.java new file mode 100644 index 000000000..2b8d9c12d --- /dev/null +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/utils/IoTDBUtils.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cn.edu.tsinghua.iot.benchmark.iotdb130.utils; + +import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; +import org.apache.tsfile.enums.TSDataType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class IoTDBUtils { + + public IoTDBUtils() {} + + public static List constructDataTypes(List sensors, int recordValueSize) { + List dataTypes = new ArrayList<>(); + for (int sensorIndex = 0; sensorIndex < recordValueSize; sensorIndex++) { + switch (sensors.get(sensorIndex).getSensorType()) { + case BOOLEAN: + dataTypes.add(TSDataType.BOOLEAN); + break; + case INT32: + dataTypes.add(TSDataType.INT32); + break; + case INT64: + dataTypes.add(TSDataType.INT64); + break; + case FLOAT: + dataTypes.add(TSDataType.FLOAT); + break; + case DOUBLE: + dataTypes.add(TSDataType.DOUBLE); + break; + case TEXT: + dataTypes.add(TSDataType.TEXT); + break; + case STRING: + dataTypes.add(TSDataType.STRING); + break; + case BLOB: + dataTypes.add(TSDataType.BLOB); + break; + case TIMESTAMP: + dataTypes.add(TSDataType.TIMESTAMP); + break; + case DATE: + dataTypes.add(TSDataType.DATE); + break; + } + } + return dataTypes; + } + + /** + * convert deviceSchema to the format + * + * @return format, e.g. root.group_1.d_1 + */ + public static String getDevicePath(DeviceSchema deviceSchema, String rootSeriesName) { + StringBuilder name = new StringBuilder(rootSeriesName); + name.append(".").append(deviceSchema.getGroup()); + for (Map.Entry pair : deviceSchema.getTags().entrySet()) { + name.append(".").append(pair.getValue()); + } + name.append(".").append(deviceSchema.getDevice()); + return name.toString(); + } +}