Support table model (#438)

* support table model

* change default value for ENABLE_TABLE

* Solve table model query errors

* Solve the CSVdataReader error under the table model

* Change the iotdb version number to 1.4.0.SNAPSHOT

* Simplify the logic of generating tablets in table models. Change ENABLE_TABLE to IoTDB_ENABLE_TABLE. Make the function of parameters clearer.

* Extract the iotdb 1.4 module. The iotdb 1.3 no longer supports the table model. The iotdb 1.4 module supports the table model.

* iotdb1.3 rollback

* Solve the CSVDataReader data reading exception.

* Refactoring iotdb inheritance structure

* drop iotdb-1.4

* Delete useless comments and parameters

* Delete useless comments and parameters(2)

* init

* Refactoring to Strategy Pattern

* Refactoring to Strategy Pattern(2)

* take a look～

* take a look～

* Refactoring to Strategy Pattern(3)

* The table model can generate multiple databases and tables. Devices are grouped into tables, and tables are grouped into databases.

* Fix the verificationWriteMode bug

* take a look～

* Fix the bug for Date value query

* fix something

* Refactoring to Strategy Pattern(4)

* Fixed the issue in 'verificationQuery()' where the expected number of records was not equal to the actual number of records.

* Restore configuration parameter default values

* Remove useless comments

* Add a log: device number >= table number >= group number

* Add the device filter condition to the table model query statement; Add a mandatory conversion of DATE; Dual write mode supports starting a tree model and a table model; Fix table model registration failure bug.

* Modify the random device selection strategy in GenerateQueryWorkLoad.

* Print a log comparing two query result sets

* delete a log comparing two query result sets

* Modify the query statement to concatenate duplicate sensors

* Code style adjustment

* Modify the log level

* Fix type conversion errors

* Simplified configuration parameters

* Improve variable names

* Remove the parameters ANOTHER_IoTDB_ENABLE_TABLE and ALL_NODES_VISIBLE

* Modify the deviceQueryMaxCount parameter

* Remove Blank Lines

* Add IoTDBUtils.java

---------

Co-authored-by: liyuheng <[email protected]>
Co-authored-by: OneSizeFitsQuorum <[email protected]>

configuration/conf/config.properties

@@ -6,6 +6,11 @@
 # 被测试的数据库，目前的格式为{name}{-version}{-insert mode}(注意-号)其全部参考值参见README文件，注意版本需要匹配
 # DB_SWITCH=IoTDB-130-SESSION_BY_TABLET
 
+# IoTDB需指定sql_dialect, 并且一个IoTDB只能指定一种。支持tree、table,默认值为tree。
+# sql_dialect等于table时，要满足：device数量 >= table数量 >= database数量
+# sql_dialect等于tree时，要满足：device数量 >= database数量
+# IoTDB_DIALECT_MODE=tree
+
 # 主机列表，如果有多个主机则使用英文逗号进行分割
 # HOST=127.0.0.1
 
@@ -70,15 +75,12 @@
 # 如果 BENCHMARK_CLUSTER=true, 则为当前Benchmark的编号
 # BENCHMARK_INDEX=0
 
-# 在集群模式下：是否所有数据库都可见，如果可见则可以向其他数据库节点发送数据
-# IS_ALL_NODES_VISIBLE=false
-
 ################ Benchmark：双写模式 ####################
-# 双写模式仅支持不同数据库之间进行比较，不支持同一个数据库不同版本进行双写
+# 双写模式仅支持不同数据库之间进行比较，不支持同一个数据库不同版本进行双写，不支持iotdb树模型与表模型的比较。
 # IS_DOUBLE_WRITE=false
 
 # 另一个写入的数据库，目前的格式为{name}{-version}{-insert mode}(注意-号)其全部参考值参见README文件
-# ANOTHER_DB_SWITCH=IoTDB-100-SESSION_BY_TABLET
+# ANOTHER_DB_SWITCH=IoTDB-130-SESSION_BY_TABLET
 
 # 另一个写入的数据库的主机
 # ANOTHER_HOST=127.0.0.1
@@ -130,6 +132,9 @@
 # 存储组名称前缀
 # GROUP_NAME_PREFIX=g_
 
+# IoTDB 表模型下(IoTDB_DIALECT_MODE=table)，表名称前缀
+# IoTDB_TABLE_NAME_PREFIX=table_
+
 # 设备名称前缀
 # DEVICE_NAME_PREFIX=d_
 
@@ -167,6 +172,9 @@
 # 存储组的数量，对IoTDB而言为database的数量
 # GROUP_NUMBER=1
 
+# IoTDB 表模型下(IoTDB_DIALECT_MODE=table)，表的数量
+# IoTDB_TABLE_NUMBER=1
+
 # IoTDB核心线程池数量
 # IOTDB_SESSION_POOL_SIZE=50
 
@@ -298,7 +306,7 @@
 ########################################################
 
 ################### 操作信息整体配置 #####################
-# 各操作的比例,按照顺序为 写入:Q1:Q2:Q3:Q4:Q5:Q6:Q7:Q8:Q9:Q10, 请注意使用英文冒号。比例中的每一项是整数。
+# 各操作的比例,按照顺序为 写入:Q1:Q2:Q3:Q4:Q5:Q6:Q7:Q8:Q9:Q10:Q11, 请注意使用英文冒号。比例中的每一项是整数。
 # Qi表示的查询如下:
 # Q1  精确点查询 select v1... from data where time = ? and device in ?
 # Q2  范围查询（只限制起止时间）select v1... from data where time > ? and time < ? and device in ?
@@ -311,6 +319,7 @@
 # Q9  倒序范围查询（只限制起止时间）select v1... from data where time > ? and time < ? and device in ? order by time desc
 # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc
 # Q11 分组聚合查询，倒序；目前仅支持iotdb、tdengine-3.0、influxdb v1
+# IoTDB V1表模型支持 Q1 Q2 Q3 Q9 Q10
 # OPERATION_PROPORTION=1:0:0:0:0:0:0:0:0:0:0:0
 
 # 最长等待写时间，单位毫秒，即如果整个写操作在指定时间内没有返回，则终止此操作
@@ -463,4 +472,5 @@
 # 测试过程中测试进度日志的输出间隔，单位为秒
 # LOG_PRINT_INTERVAL=5
 # 测试过程中当前测试结果日志的输出间隔，单位为秒
+# 设为0时，benchmark 仅在结束前输出一次测试结果日志
 # RESULT_PRINT_INTERVAL=3600

core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java

@@ -20,6 +20,7 @@
 package cn.edu.tsinghua.iot.benchmark.conf;
 
 import cn.edu.tsinghua.iot.benchmark.entity.Sensor;
+import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect;
 import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType;
 import cn.edu.tsinghua.iot.benchmark.function.FunctionParam;
 import cn.edu.tsinghua.iot.benchmark.function.FunctionXml;
@@ -60,6 +61,9 @@ public class Config {
   /** Total number of operations that each client process */
   private long LOOP = 100;
 
+  /** iotdb supports table model and tree model */
+  private SQLDialect IoTDB_DIALECT_MODE = SQLDialect.TREE;
+
   /**
    * The running mode of benchmark 1. testWithDefaultPath: Conventional test mode, supporting mixed
    * loads of multiple read and write operations 2. writeWithRealDataSet: Write the real data set
@@ -84,8 +88,6 @@ public class Config {
   private int BENCHMARK_INDEX = 0;
   /** Calculated in this way: FIRST_DEVICE_INDEX = BENCHMARK_INDEX * DEVICE_NUMBER */
   private int FIRST_DEVICE_INDEX = 0;
-  /** 是否都可见，如果可见就可以向其他node发送 Whether access all nodes, rather than just one coordinator */
-  private boolean IS_ALL_NODES_VISIBLE = false;
 
   // 初始化：被测数据库配置
   private DBConfig dbConfig = new DBConfig();
@@ -188,6 +190,8 @@ public class Config {
    */
   private int CLIENT_NUMBER = 20;
 
+  /** name prefix of table */
+  private String IoTDB_TABLE_NAME_PREFIX = "table_";
   /** name prefix of group */
   private String GROUP_NAME_PREFIX = "g_";
   /** name prefix of device */
@@ -222,6 +226,8 @@ public class Config {
   private String SG_STRATEGY = "mod";
   /** The number of storage group, must less than or equal to number of devices */
   private int GROUP_NUMBER = 1;
+  /** The number of table, In the tree model, it is equal to group_number */
+  private int IoTDB_TABLE_NUMBER = 1;
   /** The size of IoTDB core session pool */
   private int IOTDB_SESSION_POOL_SIZE = 50;
   /** Whether to use templates */
@@ -627,6 +633,14 @@ public void setLOOP(long LOOP) {
     this.LOOP = LOOP;
   }
 
+  public SQLDialect getIoTDB_DIALECT_MODE() {
+    return IoTDB_DIALECT_MODE;
+  }
+
+  public void setIoTDB_DIALECT_MODE(SQLDialect ioTDB_DIALECT_MODE) {
+    IoTDB_DIALECT_MODE = ioTDB_DIALECT_MODE;
+  }
+
   public BenchmarkMode getBENCHMARK_WORK_MODE() {
     return BENCHMARK_WORK_MODE;
   }
@@ -683,14 +697,6 @@ public void setFIRST_DEVICE_INDEX(int FIRST_DEVICE_INDEX) {
     this.FIRST_DEVICE_INDEX = FIRST_DEVICE_INDEX;
   }
 
-  public boolean isIS_ALL_NODES_VISIBLE() {
-    return IS_ALL_NODES_VISIBLE;
-  }
-
-  public void setIS_ALL_NODES_VISIBLE(boolean IS_ALL_NODES_VISIBLE) {
-    this.IS_ALL_NODES_VISIBLE = IS_ALL_NODES_VISIBLE;
-  }
-
   public String getKAFKA_LOCATION() {
     return KAFKA_LOCATION;
   }
@@ -939,6 +945,14 @@ public void setTAG_VALUE_CARDINALITY(List<Integer> TAG_VALUE_CARDINALITY) {
     this.TAG_VALUE_CARDINALITY = TAG_VALUE_CARDINALITY;
   }
 
+  public String getIoTDB_TABLE_NAME_PREFIX() {
+    return IoTDB_TABLE_NAME_PREFIX;
+  }
+
+  public void setIoTDB_TABLE_NAME_PREFIX(String IoTDB_TABLE_NAME_PREFIX) {
+    this.IoTDB_TABLE_NAME_PREFIX = IoTDB_TABLE_NAME_PREFIX;
+  }
+
   public String getGROUP_NAME_PREFIX() {
     return GROUP_NAME_PREFIX;
   }
@@ -1035,6 +1049,14 @@ public void setGROUP_NUMBER(int GROUP_NUMBER) {
     this.GROUP_NUMBER = GROUP_NUMBER;
   }
 
+  public int getIoTDB_TABLE_NUMBER() {
+    return IoTDB_TABLE_NUMBER;
+  }
+
+  public void setIoTDB_TABLE_NUMBER(int ioTDB_TABLE_NUMBER) {
+    IoTDB_TABLE_NUMBER = ioTDB_TABLE_NUMBER;
+  }
+
   public int getIOTDB_SESSION_POOL_SIZE() {
     return IOTDB_SESSION_POOL_SIZE;
   }
@@ -1766,6 +1788,7 @@ public ConfigProperties getShowConfigProperties() {
     ConfigProperties configProperties = new ConfigProperties();
 
     configProperties.addProperty("Test Mode", "BENCHMARK_WORK_MODE", this.BENCHMARK_WORK_MODE);
+    configProperties.addProperty("Test Mode", "IoTDB_DIALECT_MODE", this.getIoTDB_DIALECT_MODE());
 
     configProperties.addProperty(
         "Database Connection Information", "DOUBLE_WRITE", this.IS_DOUBLE_WRITE);
@@ -1778,6 +1801,7 @@ public ConfigProperties getShowConfigProperties() {
           this.ANOTHER_DBConfig.getMainConfig());
     }
     configProperties.addProperty("Data Mode", "GROUP_NUMBER", this.GROUP_NUMBER);
+    configProperties.addProperty("Data Mode", "IoTDB_TABLE_NUMBER", this.IoTDB_TABLE_NUMBER);
     configProperties.addProperty("Data Mode", "DEVICE_NUMBER", this.DEVICE_NUMBER);
     configProperties.addProperty("Data Mode", "REAL_INSERT_RATE", this.REAL_INSERT_RATE);
     configProperties.addProperty("Data Mode", "SENSOR_NUMBER", this.SENSOR_NUMBER);
@@ -1813,7 +1837,15 @@ public ConfigProperties getShowConfigProperties() {
             + "/"
             + this.ENCODING_DOUBLE
             + "/"
-            + this.ENCODING_TEXT);
+            + this.ENCODING_TEXT
+            + "/"
+            + this.ENCODING_STRING
+            + "/"
+            + this.ENCODING_BLOB
+            + "/"
+            + this.ENCODING_TIMESTAMP
+            + "/"
+            + this.ENCODING_DATE);
     configProperties.addProperty("Data Amount", "COMPRESSOR", this.COMPRESSOR);
     if (hasQuery()) {
       configProperties.addProperty("Query Param", "QUERY_DEVICE_NUM", this.QUERY_DEVICE_NUM);
@@ -1836,8 +1868,6 @@ public ConfigProperties getShowConfigProperties() {
     if (this.BENCHMARK_CLUSTER) {
       configProperties.addProperty("Other Param", "BENCHMARK_INDEX", this.BENCHMARK_INDEX);
       configProperties.addProperty("Other Param", "FIRST_DEVICE_INDEX", this.FIRST_DEVICE_INDEX);
-      configProperties.addProperty(
-          "Other Param", "IS_ALL_NODES_VISIBLE", this.IS_ALL_NODES_VISIBLE);
     }
     if (this.TEMPLATE) {
       configProperties.addProperty("Other Param", "TEMPLATE", this.TEMPLATE);

core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java

@@ -19,6 +19,7 @@
 
 package cn.edu.tsinghua.iot.benchmark.conf;
 
+import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect;
 import cn.edu.tsinghua.iot.benchmark.mode.enums.BenchmarkMode;
 import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig;
 import cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBSwitch;
@@ -40,6 +41,7 @@
 import java.util.stream.Collectors;
 
 import static cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode.INSERT_USE_SESSION_RECORDS;
+import static cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode.INSERT_USE_SESSION_TABLET;
 
 public class ConfigDescriptor {
   private static final Logger LOGGER = LoggerFactory.getLogger(ConfigDescriptor.class);
@@ -86,6 +88,9 @@ private void loadProps() {
       Properties properties = new Properties();
       try {
         properties.load(inputStream);
+        config.setIoTDB_DIALECT_MODE(
+            SQLDialect.getSQLDialect(
+                properties.getProperty("IoTDB_DIALECT_MODE", config.getIoTDB_DIALECT_MODE() + "")));
         config.setIS_DELETE_DATA(
             Boolean.parseBoolean(
                 properties.getProperty("IS_DELETE_DATA", config.isIS_DELETE_DATA() + "")));
@@ -254,6 +259,8 @@ private void loadProps() {
         config.setCLIENT_NUMBER(
             Integer.parseInt(
                 properties.getProperty("CLIENT_NUMBER", config.getCLIENT_NUMBER() + "")));
+        config.setIoTDB_TABLE_NAME_PREFIX(
+            properties.getProperty("IoTDB_TABLE_NAME_PREFIX", config.getIoTDB_TABLE_NAME_PREFIX()));
         config.setGROUP_NAME_PREFIX(
             properties.getProperty("GROUP_NAME_PREFIX", config.getGROUP_NAME_PREFIX()));
         config.setDEVICE_NAME_PREFIX(
@@ -290,11 +297,6 @@ private void loadProps() {
         } else {
           config.setFIRST_DEVICE_INDEX(0);
         }
-        config.setIS_ALL_NODES_VISIBLE(
-            Boolean.parseBoolean(
-                properties.getProperty(
-                    "IS_ALL_NODES_VISIBLE", String.valueOf(config.isIS_ALL_NODES_VISIBLE()))));
-
         config.setLINE_RATIO(
             Double.parseDouble(properties.getProperty("LINE_RATIO", config.getLINE_RATIO() + "")));
         config.setSIN_RATIO(
@@ -319,6 +321,24 @@ private void loadProps() {
         config.setGROUP_NUMBER(
             Integer.parseInt(
                 properties.getProperty("GROUP_NUMBER", config.getGROUP_NUMBER() + "")));
+        config.setIoTDB_TABLE_NUMBER(
+            Integer.parseInt(
+                properties.getProperty("IoTDB_TABLE_NUMBER", config.getIoTDB_TABLE_NUMBER() + "")));
+
+        if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) {
+          if (config.getGROUP_NUMBER() > config.getIoTDB_TABLE_NUMBER()
+              || config.getIoTDB_TABLE_NUMBER() > config.getDEVICE_NUMBER()) {
+            LOGGER.warn(
+                "Please follow this rule to adjust the parameters: device number >= table number >= database number. Otherwise, device number = table number = database number");
+          }
+        } else {
+          if (config.getGROUP_NUMBER() > config.getDEVICE_NUMBER()) {
+            config.setIoTDB_TABLE_NUMBER(config.getGROUP_NUMBER());
+            LOGGER.warn(
+                "Please follow this rule to adjust the parameters: device number >= database number. Otherwise, the total number of databases created is equal to the number of devices");
+          }
+        }
+
         config.setIOTDB_SESSION_POOL_SIZE(
             Integer.parseInt(
                 properties.getProperty(
@@ -606,6 +626,12 @@ private boolean checkConfig() {
       default:
         break;
     }
+    if ((config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE
+        && config.getDbConfig().getDB_SWITCH().getInsertMode() != INSERT_USE_SESSION_TABLET)) {
+      LOGGER.error(
+          "The iotdb table model only supports INSERT_USE_SESSION_TABLET! Please modify DB_SWITCH in the configuration file.");
+      result = false;
+    }
     result &= checkInsertDataTypeProportion();
     result &= checkOperationProportion();
     if (config.getCLIENT_NUMBER() == 0) {

core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java

@@ -51,25 +51,19 @@ public class Constants {
   public static final String HASH_SG_ASSIGN_MODE = "hash";
   public static final String DIV_SG_ASSIGN_MODE = "div";
 
+  // TODO iotdb-1.4
+
+  public static final String IOTDB130 = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB";
   public static final String IOTDB130_REST_CLASS =
       "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBRestAPI";
-  public static final String IOTDB130_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB";
-  public static final String IOTDB130_SESSION_CLASS =
-      "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBSession";
-  public static final String IOTDB130_ROUNDROBIN_SESSION_CLASS =
-      "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBClusterSession";
 
   public static final String IOTDB110_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb110.IoTDB";
   public static final String IOTDB110_SESSION_CLASS =
       "cn.edu.tsinghua.iot.benchmark.iotdb110.IoTDBSession";
-  public static final String IOTDB110_ROUNDROBIN_SESSION_CLASS =
-      "cn.edu.tsinghua.iot.benchmark.iotdb110.IoTDBClusterSession";
 
   public static final String IOTDB100_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb100.IoTDB";
   public static final String IOTDB100_SESSION_CLASS =
       "cn.edu.tsinghua.iot.benchmark.iotdb100.IoTDBSession";
-  public static final String IOTDB100_ROUNDROBIN_SESSION_CLASS =
-      "cn.edu.tsinghua.iot.benchmark.iotdb100.IoTDBClusterSession";
 
   public static final String INFLUXDB_CLASS = "cn.edu.tsinghua.iot.benchmark.influxdb.InfluxDB";
   public static final String INFLUXDB2_CLASS = "cn.edu.tsinghua.iot.benchmark.influxdb2.InfluxDB";