From 88120bd99d27d15d1d52e09f2effbc2bbe05f90c Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 26 Dec 2023 11:16:14 +0800 Subject: [PATCH 01/19] terrible idea --- configuration/conf/config.properties | 31 ++++++++++++------- .../tsinghua/iot/benchmark/conf/Config.java | 28 +++++++++++++++++ .../iot/benchmark/tsdb/IDatabase.java | 4 +++ .../iot/benchmark/influxdb2/InfluxDB.java | 2 +- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index a5f5dcf9c..5f2e5fae2 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -4,25 +4,25 @@ ################### 被测数据库配置 ####################### # 被测试的数据库,目前的格式为{name}{-version}{-insert mode}(注意-号)其全部参考值参见README文件,注意版本需要匹配 -# DB_SWITCH=IoTDB-110-SESSION_BY_TABLET + DB_SWITCH=InfluxDB-2.x # 主机列表,如果有多个主机则使用英文逗号进行分割 -# HOST=127.0.0.1 +# HOST=192.168.130.37 # 端口列表,需要和HOST数量一致,保持一一对应。如果有多个端口则使用英文逗号进行分割。 -# PORT=6667 + PORT=8086 # 所有被测数据库的用户名,如果为多个数据库,则要求保持一致 -# USERNAME=root + USERNAME=lyh # 所有被测数据库的密码,如果为多个数据库,则要求保持一致 -# PASSWORD=root + PASSWORD=1234qwer # 即将被测试写入的数据库的名称 -# DB_NAME=test +# DB_NAME=gg # 数据库连接认证Token,InfluxDB 2.0使用 -# TOKEN=token + TOKEN=tJY-JVZwJt9zJDUtmP_ZZaFVRA8Y_Tht_FHFrG5_IrtUingRxqiWiT2twUeu6IvOo4YFtdNeSkpTPEfcU9AyYQ== ############### 数据库初始化:清理数据 #################### # 测试启动前是否清理数据 @@ -40,7 +40,7 @@ ################## Benchmark配置 ####################### # 总操作次数:具体每种类型操作会按OPERATION_PROPORTION定义的比例划分,例如OPERATION_PROPORTION=1:1:1:1:1:1:1:1:1:1:1 -# LOOP=100 + LOOP=10 # 目前Benchmark支持多种运行模式,支持模式如下 # testWithDefaultPath 常规测试模式,支持多种读和写操作的混合负载 @@ -111,7 +111,7 @@ ############## 数据:设备、传感器、客户端 ################## # 设备总数 -# DEVICE_NUMBER=6000 + DEVICE_NUMBER=60 # 实际写入设备比例,(0, 1] # REAL_INSERT_RATE=1.0 @@ -203,7 +203,7 @@ ############## 被测系统为Influxdb 2.x时扩展参数 ######## # influxdb ORG名 -# INFLUXDB_ORG=company1 + INFLUXDB_ORG=timecho ################### 操作时间配置 ######################### # 时间戳间隔,即生成的数据两个时间戳之间的固定长度(如果定长生成),非正常速率 @@ -297,7 +297,7 @@ # Q8 最近点查询 select time, v1... where device = ? and time = max(time) # Q9 倒序范围查询(只限制起止时间)select v1... from data where time > ? and time < ? and device in ? order by time desc # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc -# OPERATION_PROPORTION=1:0:0:0:0:0:0:0:0:0:0 +# OPERATION_PROPORTION=0:1:0:0:0:0:0:0:0:0:0 # 最长等待写时间,单位毫秒,即如果整个写操作在指定时间内没有返回,则终止此操作 # WRITE_OPERATION_TIMEOUT_MS=120000 @@ -386,6 +386,15 @@ # 查询语句的随机数种子 # QUERY_SEED=151658 +# 是否order by time desc +# ORDER_BY_TIME_DESC=false + +# 行数限制, -1表示不使用limit语句 +# ROW_LIMIT=-1 + +# 是否align by device +# ALIGN_BY_DEVICE=false + ################ Workload:相关参数 ###################### # workload的缓冲区的大小 # WORKLOAD_BUFFER_SIZE=100 diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index c9b4832b4..af60fd0e1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -338,6 +338,10 @@ public class Config { /** Query random seed */ private long QUERY_SEED = 151658L; + private boolean ORDER_BY_TIME_DESC = false; + private long ROW_LIMIT = -1; + private boolean ALIGN_BY_DEVICE = false; + // workload 相关部分 /** The size of workload buffer size */ private int WORKLOAD_BUFFER_SIZE = 100; @@ -1254,6 +1258,30 @@ public void setQUERY_SEED(long QUERY_SEED) { this.QUERY_SEED = QUERY_SEED; } + public boolean isORDER_BY_TIME_DESC() { + return ORDER_BY_TIME_DESC; + } + + public void setORDER_BY_TIME_DESC(boolean ORDER_BY_TIME_DESC) { + this.ORDER_BY_TIME_DESC = ORDER_BY_TIME_DESC; + } + + public long getROW_LIMIT() { + return ROW_LIMIT; + } + + public void setROW_LIMIT(long ROW_LIMIT) { + this.ROW_LIMIT = ROW_LIMIT; + } + + public boolean isALIGN_BY_DEVICE() { + return ALIGN_BY_DEVICE; + } + + public void setALIGN_BY_DEVICE(boolean ALIGN_BY_DEVICE) { + this.ALIGN_BY_DEVICE = ALIGN_BY_DEVICE; + } + public int getWORKLOAD_BUFFER_SIZE() { return WORKLOAD_BUFFER_SIZE; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java index c525163d6..ddfe4aae6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java @@ -189,4 +189,8 @@ default DeviceSummary deviceSummary(DeviceQuery deviceQuery) throws SQLException default String typeMap(SensorType iotdbSensorType) { return iotdbSensorType.name; } + // + // Status executeQueryAndGetStatus(String sql); + // + // Status executeQueryAndGetStatus(String sql, Operation operation); } diff --git a/influxdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb2/InfluxDB.java b/influxdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb2/InfluxDB.java index 1ddd2551c..420ddd39d 100644 --- a/influxdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb2/InfluxDB.java +++ b/influxdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb2/InfluxDB.java @@ -527,7 +527,7 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { return new Status(true, result); } - private Status executeQueryAndGetStatus(String sql) { + public Status executeQueryAndGetStatus(String sql) { if (!config.isIS_QUIET_MODE()) { LOGGER.debug("{} query SQL: {}", Thread.currentThread().getName(), sql); } From b572a596cbefa7c82e04fd7fa83983164ad34d3b Mon Sep 17 00:00:00 2001 From: liyuheng Date: Thu, 28 Dec 2023 18:15:12 +0800 Subject: [PATCH 02/19] config --- .../tsinghua/iot/benchmark/conf/Config.java | 19 +++++----------- .../iot/benchmark/conf/ConfigDescriptor.java | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index af60fd0e1..bd34d60b7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -338,8 +338,7 @@ public class Config { /** Query random seed */ private long QUERY_SEED = 151658L; - private boolean ORDER_BY_TIME_DESC = false; - private long ROW_LIMIT = -1; + private long RESULT_ROW_LIMIT = -1; private boolean ALIGN_BY_DEVICE = false; // workload 相关部分 @@ -1258,20 +1257,12 @@ public void setQUERY_SEED(long QUERY_SEED) { this.QUERY_SEED = QUERY_SEED; } - public boolean isORDER_BY_TIME_DESC() { - return ORDER_BY_TIME_DESC; + public long getRESULT_ROW_LIMIT() { + return RESULT_ROW_LIMIT; } - public void setORDER_BY_TIME_DESC(boolean ORDER_BY_TIME_DESC) { - this.ORDER_BY_TIME_DESC = ORDER_BY_TIME_DESC; - } - - public long getROW_LIMIT() { - return ROW_LIMIT; - } - - public void setROW_LIMIT(long ROW_LIMIT) { - this.ROW_LIMIT = ROW_LIMIT; + public void setRESULT_ROW_LIMIT(long RESULT_ROW_LIMIT) { + this.RESULT_ROW_LIMIT = RESULT_ROW_LIMIT; } public boolean isALIGN_BY_DEVICE() { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 5ac46a79d..f1669ab7d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -436,6 +436,12 @@ private void loadProps() { properties.getProperty("GROUP_BY_TIME_UNIT", config.getGROUP_BY_TIME_UNIT() + ""))); config.setQUERY_SEED( Long.parseLong(properties.getProperty("QUERY_SEED", config.getQUERY_SEED() + ""))); + config.setRESULT_ROW_LIMIT( + Long.parseLong( + properties.getProperty("RESULT_ROW_LIMIT", config.getRESULT_ROW_LIMIT() + ""))); + config.setALIGN_BY_DEVICE( + Boolean.parseBoolean( + properties.getProperty("ALIGN_BY_DEVICE", config.isALIGN_BY_DEVICE() + ""))); config.setWORKLOAD_BUFFER_SIZE( Integer.parseInt( @@ -588,9 +594,25 @@ private boolean checkConfig() { } result &= checkDeviceNumPerWrite(); result &= checkTag(); + if (!isGoodDB()) { + if (config.isALIGN_BY_DEVICE()) { + LOGGER.error("ALIGN_BY_DEVICE is not supported for this database"); + } + if (config.getRESULT_ROW_LIMIT() >= 0) { + LOGGER.error("RESULT_ROW_LIMIT is not supported for this database"); + } + } return result; } + private boolean isGoodDB() { + DBType dbType = config.getDbConfig().getDB_SWITCH().getType(); + DBVersion dbVersion = config.getDbConfig().getDB_SWITCH().getVersion(); + return dbType.equals(DBType.IoTDB) + || dbVersion.equals(DBVersion.TDengine_3) + || dbVersion.equals(DBVersion.InfluxDB_2); + } + private boolean checkOperationProportion() { String[] op = config.getOPERATION_PROPORTION().split(":"); int minOps = 0; From 2f7ea16ca165fae60926ba52505903e8cb5539b7 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 11:49:27 +0800 Subject: [PATCH 03/19] iotdb done? --- .../iot/benchmark/iotdb110/IoTDB.java | 29 ++++++++++++------- .../benchmark/iotdb110/IoTDBSessionBase.java | 8 ++++- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java index da770de36..9af461c38 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java @@ -424,7 +424,7 @@ public Status insertOneBatch(IBatch batch) throws DBConnectException { public Status preciseQuery(PreciseQuery preciseQuery) { String strTime = preciseQuery.getTimestamp() + ""; String sql = getSimpleQuerySqlHead(preciseQuery.getDeviceSchema()) + " WHERE time = " + strTime; - return executeQueryAndGetStatus(sql, Operation.PRECISE_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.PRECISE_QUERY); } /** @@ -441,7 +441,7 @@ public Status rangeQuery(RangeQuery rangeQuery) { rangeQuery.getDeviceSchema(), rangeQuery.getStartTimestamp(), rangeQuery.getEndTimestamp()); - return executeQueryAndGetStatus(sql, Operation.RANGE_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.RANGE_QUERY); } /** @@ -454,7 +454,7 @@ public Status rangeQuery(RangeQuery rangeQuery) { @Override public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { String sql = getValueRangeQuerySql(valueRangeQuery); - return executeQueryAndGetStatus(sql, Operation.VALUE_RANGE_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.VALUE_RANGE_QUERY); } /** @@ -471,7 +471,7 @@ public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { String sql = addWhereTimeClause( aggQuerySqlHead, aggRangeQuery.getStartTimestamp(), aggRangeQuery.getEndTimestamp()); - return executeQueryAndGetStatus(sql, Operation.AGG_RANGE_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.AGG_RANGE_QUERY); } /** @@ -490,7 +490,7 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { + getValueFilterClause( aggValueQuery.getDeviceSchema(), (int) aggValueQuery.getValueThreshold()) .substring(4); - return executeQueryAndGetStatus(sql, Operation.AGG_VALUE_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.AGG_VALUE_QUERY); } /** @@ -513,7 +513,7 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { sql += getValueFilterClause( aggRangeValueQuery.getDeviceSchema(), (int) aggRangeValueQuery.getValueThreshold()); - return executeQueryAndGetStatus(sql, Operation.AGG_RANGE_VALUE_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.AGG_RANGE_VALUE_QUERY); } /** @@ -533,7 +533,7 @@ public Status groupByQuery(GroupByQuery groupByQuery) { groupByQuery.getStartTimestamp(), groupByQuery.getEndTimestamp(), groupByQuery.getGranularity()); - return executeQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY); } /** @@ -545,7 +545,7 @@ public Status groupByQuery(GroupByQuery groupByQuery) { @Override public Status latestPointQuery(LatestPointQuery latestPointQuery) { String aggQuerySqlHead = getLatestPointQuerySql(latestPointQuery.getDeviceSchema()); - return executeQueryAndGetStatus(aggQuerySqlHead, Operation.LATEST_POINT_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus(aggQuerySqlHead, Operation.LATEST_POINT_QUERY); } /** @@ -563,7 +563,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { rangeQuery.getStartTimestamp(), rangeQuery.getEndTimestamp()) + " order by time desc"; - return executeQueryAndGetStatus(sql, Operation.RANGE_QUERY_ORDER_BY_TIME_DESC); + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.RANGE_QUERY_ORDER_BY_TIME_DESC); } /** @@ -576,7 +576,8 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { @Override public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { String sql = getValueRangeQuerySql(valueRangeQuery) + " order by time desc"; - return executeQueryAndGetStatus(sql, Operation.VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC); + return addSomeClauseAndExecuteQueryAndGetStatus( + sql, Operation.VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC); } /** @@ -696,7 +697,13 @@ protected String getDevicePath(DeviceSchema deviceSchema) { return name.toString(); } - protected Status executeQueryAndGetStatus(String sql, Operation operation) { + protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation operation) { + if (config.getRESULT_ROW_LIMIT() >= 0) { + sql += " limit " + config.getRESULT_ROW_LIMIT(); + } + if (config.isALIGN_BY_DEVICE()) { + sql += " align by device"; + } String executeSQL; if (config.isIOTDB_USE_DEBUG() && random.nextDouble() < config.getIOTDB_USE_DEBUG_RATIO()) { executeSQL = "debug " + sql; diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java index b085c2fdd..edc9a45ff 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java @@ -169,7 +169,13 @@ public Status insertOneBatchByRecords(IBatch batch) { } @Override - protected Status executeQueryAndGetStatus(String sql, Operation operation) { + protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation operation) { + if (config.getRESULT_ROW_LIMIT() >= 0) { + sql += " limit " + config.getRESULT_ROW_LIMIT(); + } + if (config.isALIGN_BY_DEVICE()) { + sql += " align by device"; + } String executeSQL; if (config.isIOTDB_USE_DEBUG() && random.nextDouble() < config.getIOTDB_USE_DEBUG_RATIO()) { executeSQL = "debug " + sql; From 1fb9d5b91632477171be7c9a963890ab47363e44 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 11:50:03 +0800 Subject: [PATCH 04/19] tdengine align by device done? --- .../benchmark/workload/ValueRangeFilter.java | 60 ++++++ .../iot/benchmark/tdengine3/TDengine.java | 178 ++++++++++-------- 2 files changed, 163 insertions(+), 75 deletions(-) create mode 100644 core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/ValueRangeFilter.java diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/ValueRangeFilter.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/ValueRangeFilter.java new file mode 100644 index 000000000..967f663ae --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/ValueRangeFilter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cn.edu.tsinghua.iot.benchmark.workload; + +import cn.edu.tsinghua.iot.benchmark.entity.Sensor; + +import java.util.List; + +public class ValueRangeFilter { + private final Double minValue; + private final List sensors; + private final Double maxValue; + + public ValueRangeFilter(double minValue, List sensors, double maxValue) { + this.sensors = sensors; + this.minValue = minValue; + this.maxValue = maxValue; + } + + public ValueRangeFilter(double minValue, List sensors) { + this.minValue = minValue; + this.sensors = sensors; + this.maxValue = null; + } + + public ValueRangeFilter(List sensors, double maxValue) { + this.minValue = null; + this.sensors = sensors; + this.maxValue = maxValue; + } + + public List getSensors() { + return sensors; + } + + public Double getMinValue() { + return minValue; + } + + public Double getMaxValue() { + return maxValue; + } +} diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index c79fe8669..ee94f44a1 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -32,6 +32,7 @@ import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; import cn.edu.tsinghua.iot.benchmark.utils.TimeUtils; +import cn.edu.tsinghua.iot.benchmark.workload.ValueRangeFilter; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.AggRangeQuery; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.AggRangeValueQuery; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.AggValueQuery; @@ -275,7 +276,9 @@ public Status preciseQuery(PreciseQuery preciseQuery) { @Override public Status rangeQuery(RangeQuery rangeQuery) { String rangeQueryHead = getSimpleQuerySqlHead(rangeQuery.getDeviceSchema()); - String sql = addWhereTimeClause(rangeQueryHead, rangeQuery); + String sql = + addWhereClause( + rangeQueryHead, rangeQuery, null, getTableNameFilterForAlignByDevice(rangeQuery)); return executeQueryAndGetStatus(sql); } @@ -286,13 +289,15 @@ public Status rangeQuery(RangeQuery rangeQuery) { @Override public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { String rangeQueryHead = getSimpleQuerySqlHead(valueRangeQuery.getDeviceSchema()); - String sqlWithTimeFilter = addWhereTimeClause(rangeQueryHead, valueRangeQuery); - String sqlWithValueFilter = - addWhereValueClause( - valueRangeQuery.getDeviceSchema(), - sqlWithTimeFilter, - valueRangeQuery.getValueThreshold()); - return executeQueryAndGetStatus(sqlWithValueFilter); + String sql = + addWhereClause( + rangeQueryHead, + valueRangeQuery, + new ValueRangeFilter( + valueRangeQuery.getValueThreshold(), + valueRangeQuery.getDeviceSchema().get(0).getSensors()), + getTableNameFilterForAlignByDevice(valueRangeQuery)); + return executeQueryAndGetStatus(sql); } /** @@ -303,7 +308,12 @@ public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { String aggQuerySqlHead = getAggQuerySqlHead(aggRangeQuery.getDeviceSchema(), aggRangeQuery.getAggFun()); - String sql = addWhereTimeClause(aggQuerySqlHead, aggRangeQuery); + String sql = + addWhereClause( + aggQuerySqlHead, + aggRangeQuery, + null, + getTableNameFilterForAlignByDevice(aggRangeQuery)); return executeQueryAndGetStatus(sql); } @@ -313,8 +323,13 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { String aggQuerySqlHead = getAggQuerySqlHead(aggValueQuery.getDeviceSchema(), aggValueQuery.getAggFun()); String sql = - addWhereValueWithoutTimeClause( - aggValueQuery.getDeviceSchema(), aggQuerySqlHead, aggValueQuery.getValueThreshold()); + addWhereClause( + aggQuerySqlHead, + null, + new ValueRangeFilter( + aggValueQuery.getValueThreshold(), + aggValueQuery.getDeviceSchema().get(0).getSensors()), + getTableNameFilterForAlignByDevice(aggValueQuery)); return executeQueryAndGetStatus(sql); } @@ -326,13 +341,15 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { String rangeQueryHead = getAggQuerySqlHead(aggRangeValueQuery.getDeviceSchema(), aggRangeValueQuery.getAggFun()); - String sqlWithTimeFilter = addWhereTimeClause(rangeQueryHead, aggRangeValueQuery); - String sqlWithValueFilter = - addWhereValueClause( - aggRangeValueQuery.getDeviceSchema(), - sqlWithTimeFilter, - aggRangeValueQuery.getValueThreshold()); - return executeQueryAndGetStatus(sqlWithValueFilter); + String sql = + addWhereClause( + rangeQueryHead, + aggRangeValueQuery, + new ValueRangeFilter( + aggRangeValueQuery.getValueThreshold(), + aggRangeValueQuery.getDeviceSchema().get(0).getSensors()), + getTableNameFilterForAlignByDevice(aggRangeValueQuery)); + return executeQueryAndGetStatus(sql); } /** @@ -342,7 +359,9 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { @Override public Status groupByQuery(GroupByQuery groupByQuery) { String sqlHeader = getAggQuerySqlHead(groupByQuery.getDeviceSchema(), groupByQuery.getAggFun()); - String sqlWithTimeFilter = addWhereTimeClause(sqlHeader, groupByQuery); + String sqlWithTimeFilter = + addWhereClause( + sqlHeader, groupByQuery, null, getTableNameFilterForAlignByDevice(groupByQuery)); String sqlWithGroupBy = addGroupByClause(sqlWithTimeFilter, groupByQuery.getGranularity()); return executeQueryAndGetStatus(sqlWithGroupBy); } @@ -357,21 +376,26 @@ public Status latestPointQuery(LatestPointQuery latestPointQuery) { @Override public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { String rangeQueryHead = getSimpleQuerySqlHead(rangeQuery.getDeviceSchema()); - String sql = addWhereTimeClause(rangeQueryHead, rangeQuery) + " order by time desc"; + String sql = + addWhereClause( + rangeQueryHead, rangeQuery, null, getTableNameFilterForAlignByDevice(rangeQuery)); + sql += " order by time desc"; return executeQueryAndGetStatus(sql); } @Override public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { String rangeQueryHead = getSimpleQuerySqlHead(valueRangeQuery.getDeviceSchema()); - String sqlWithTimeFilter = addWhereTimeClause(rangeQueryHead, valueRangeQuery); - String sqlWithValueFilter = - addWhereValueClause( - valueRangeQuery.getDeviceSchema(), - sqlWithTimeFilter, - valueRangeQuery.getValueThreshold()) - + " order by time desc"; - return executeQueryAndGetStatus(sqlWithValueFilter); + String sql = + addWhereClause( + rangeQueryHead, + valueRangeQuery, + new ValueRangeFilter( + valueRangeQuery.getValueThreshold(), + valueRangeQuery.getDeviceSchema().get(0).getSensors()), + getTableNameFilterForAlignByDevice(valueRangeQuery)); + sql += " order by time desc"; + return executeQueryAndGetStatus(sql); } private String getPreciseQuerySql(PreciseQuery preciseQuery) { @@ -408,14 +432,11 @@ private static String getSimpleQuerySqlHead(List devices) { */ private static String generateConstrainForDevices(List devices) { StringBuilder builder = new StringBuilder(); - builder.append(" FROM ").append(devices.get(0).getDevice()); - // builder.append(" WHERE "); - /*for (DeviceSchema d : devices) { - builder.append(" device = '").append(d.getDevice()).append("' OR"); + if (config.isALIGN_BY_DEVICE()) { + builder.append(" FROM ").append(SUPER_TABLE_NAME); + } else { + builder.append(" FROM ").append(devices.get(0).getDevice()); } - builder.delete(builder.lastIndexOf("OR"), builder.length()); - builder.append(")"); - */ return builder.toString(); } @@ -443,52 +464,59 @@ private Status executeQueryAndGetStatus(String sql) { } /** - * add time filter for query statements. + * Add WHERE clause. Include time filter, value filter, table name filter * - * @param sql sql header - * @param rangeQuery range query - * @return sql with time filter + * @param sql The original sql + * @param timeRangeQuery + * @param valueRangeFilter + * @param alignByDeviceTableNameFilter + * @return New sql, with WHERE clause */ - private static String addWhereTimeClause(String sql, RangeQuery rangeQuery) { - String startTime = "" + rangeQuery.getStartTimestamp(); - String endTime = "" + rangeQuery.getEndTimestamp(); - return sql + " Where time >= " + startTime + " AND time <= " + endTime; - } - - /** - * add value filter for query statements. - * - * @param devices query device schema - * @param sqlHeader sql header - * @param valueThreshold lower bound of query value filter - * @return sql with value filter - */ - private static String addWhereValueClause( - List devices, String sqlHeader, double valueThreshold) { - StringBuilder builder = new StringBuilder(sqlHeader); - for (Sensor sensor : devices.get(0).getSensors()) { - builder.append(" AND ").append(sensor.getName()).append(" > ").append(valueThreshold); + private static String addWhereClause( + String sql, + RangeQuery timeRangeQuery, + ValueRangeFilter valueRangeFilter, + List alignByDeviceTableNameFilter) { + if (timeRangeQuery == null + && valueRangeFilter == null + && alignByDeviceTableNameFilter == null) { + return sql; } - return builder.toString(); + StringBuilder sqlBuilder = new StringBuilder(sql); + sqlBuilder.append(" WHERE"); + if (timeRangeQuery != null) { + String startTime = "" + timeRangeQuery.getStartTimestamp(); + String endTime = "" + timeRangeQuery.getEndTimestamp(); + sqlBuilder.append(" time >= ").append(startTime).append(" AND time <= ").append(endTime); + } + if (valueRangeFilter != null) { + if (!sqlBuilder.toString().endsWith("WHERE")) { + sqlBuilder.append(" AND "); + } + double valueThreshold = valueRangeFilter.getMinValue(); + for (Sensor sensor : valueRangeFilter.getSensors()) { + sqlBuilder.append(sensor.getName()).append(" > ").append(valueThreshold).append(" AND "); + } + } + if (alignByDeviceTableNameFilter != null) { + if (!sqlBuilder.toString().endsWith("WHERE")) { + sqlBuilder.append(" AND "); + } + sqlBuilder.append(" tbname in ("); + for (DeviceSchema deviceSchema : alignByDeviceTableNameFilter) { + sqlBuilder.append('"').append(deviceSchema.getDevice()).append('"').append(','); + } + sqlBuilder.deleteCharAt(sqlBuilder.length() - 1); + sqlBuilder.append(')'); + } + return sqlBuilder.toString(); } - /** - * add value filter without time filter for query statements. - * - * @param devices query device schema - * @param sqlHeader sql header - * @param valueThreshold lower bound of query value filter - * @return sql with value filter - */ - private static String addWhereValueWithoutTimeClause( - List devices, String sqlHeader, double valueThreshold) { - StringBuilder builder = new StringBuilder(sqlHeader); - builder.append(" Where "); - for (Sensor sensor : devices.get(0).getSensors()) { - builder.append(sensor.getName()).append(" > ").append(valueThreshold).append(" AND "); + private static List getTableNameFilterForAlignByDevice(RangeQuery rangeQuery) { + if (config.isALIGN_BY_DEVICE()) { + return rangeQuery.getDeviceSchema(); } - builder.delete(builder.lastIndexOf("AND"), builder.length()); - return builder.toString(); + return null; } /** From 24c9f19fe1f2a01359d1f95e01897c52598c12e8 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 12:13:58 +0800 Subject: [PATCH 05/19] tdengine limit done? --- .../iot/benchmark/tdengine3/TDengine.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index ee94f44a1..37e6af3f9 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -279,6 +279,7 @@ public Status rangeQuery(RangeQuery rangeQuery) { String sql = addWhereClause( rangeQueryHead, rangeQuery, null, getTableNameFilterForAlignByDevice(rangeQuery)); + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } @@ -297,6 +298,7 @@ public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { valueRangeQuery.getValueThreshold(), valueRangeQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(valueRangeQuery)); + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } @@ -314,6 +316,7 @@ public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { aggRangeQuery, null, getTableNameFilterForAlignByDevice(aggRangeQuery)); + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } @@ -330,6 +333,7 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { aggValueQuery.getValueThreshold(), aggValueQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(aggValueQuery)); + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } @@ -349,6 +353,7 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { aggRangeValueQuery.getValueThreshold(), aggRangeValueQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(aggRangeValueQuery)); + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } @@ -359,11 +364,12 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { @Override public Status groupByQuery(GroupByQuery groupByQuery) { String sqlHeader = getAggQuerySqlHead(groupByQuery.getDeviceSchema(), groupByQuery.getAggFun()); - String sqlWithTimeFilter = + String sql = addWhereClause( sqlHeader, groupByQuery, null, getTableNameFilterForAlignByDevice(groupByQuery)); - String sqlWithGroupBy = addGroupByClause(sqlWithTimeFilter, groupByQuery.getGranularity()); - return executeQueryAndGetStatus(sqlWithGroupBy); + sql = addGroupByClause(sql, groupByQuery.getGranularity()); + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); + return executeQueryAndGetStatus(sql); } /** eg. SELECT last(s_2) FROM group_2 WHERE ( device = 'd_8' ). */ @@ -380,6 +386,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { addWhereClause( rangeQueryHead, rangeQuery, null, getTableNameFilterForAlignByDevice(rangeQuery)); sql += " order by time desc"; + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } @@ -395,6 +402,7 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { valueRangeQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(valueRangeQuery)); sql += " order by time desc"; + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } @@ -519,6 +527,13 @@ private static List getTableNameFilterForAlignByDevice(RangeQuery return null; } + private static String addLimitClause(String sql, long limitation) { + if (config.getRESULT_ROW_LIMIT() >= 0) { + sql += " limit " + limitation; + } + return sql; + } + /** * generate aggregation query header. * From 783223904581903e470341e1577b1a6dfbc677cb Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 13:42:50 +0800 Subject: [PATCH 06/19] Q11 done? --- configuration/conf/config.properties | 3 ++- .../tsinghua/iot/benchmark/conf/Config.java | 2 +- .../iot/benchmark/tsdb/IDatabase.java | 4 ++++ .../iot/benchmark/iotdb110/IoTDB.java | 24 +++++++++++++++---- .../iot/benchmark/tdengine3/TDengine.java | 17 +++++++++++-- 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 5f2e5fae2..1dc47aa30 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -297,7 +297,8 @@ # Q8 最近点查询 select time, v1... where device = ? and time = max(time) # Q9 倒序范围查询(只限制起止时间)select v1... from data where time > ? and time < ? and device in ? order by time desc # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc -# OPERATION_PROPORTION=0:1:0:0:0:0:0:0:0:0:0 +# Q11 分组聚合查询,倒序 +# OPERATION_PROPORTION=0:1:0:0:0:0:0:0:0:0:0:0 # 最长等待写时间,单位毫秒,即如果整个写操作在指定时间内没有返回,则终止此操作 # WRITE_OPERATION_TIMEOUT_MS=120000 diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index bd34d60b7..5014816f3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -319,7 +319,7 @@ public class Config { * value filtering in reverse order, Eg. select v1... from data where time > ? and time < ? and v1 * > ? and device in ? order by time desc */ - private String OPERATION_PROPORTION = "1:0:0:0:0:0:0:0:0:0:0"; + private String OPERATION_PROPORTION = "1:0:0:0:0:0:0:0:0:0:0:0"; /** The number of sensors involved in each query */ private int QUERY_SENSOR_NUM = 1; /** The number of devices involved in each query */ diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java index ddfe4aae6..65ad613f3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java @@ -164,6 +164,10 @@ default Status insertOneBatchWithCheck(IBatch batch) throws Exception { /** similar to rangeQuery, but order by time desc. */ Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery); + default Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { + throw new UnsupportedOperationException("This operation is not supported for this database"); + } + /** Using in verification */ default Status verificationQuery(VerificationQuery verificationQuery) { WorkloadException workloadException = new WorkloadException("Not Supported Verification Query"); diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java index 9af461c38..7634c6881 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java @@ -89,6 +89,7 @@ public class IoTDB implements IDatabase { private static final AtomicBoolean templateInit = new AtomicBoolean(false); private static final int ACTIVATE_TEMPLATE_THRESHOLD = 1000; protected final String DELETE_SERIES_SQL; + private final String ORDER_BY_TIME_DESC = " order by time desc "; protected SingleNodeJDBCConnection ioTDBConnection; protected static final Config config = ConfigDescriptor.getInstance().getConfig(); @@ -562,7 +563,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { rangeQuery.getDeviceSchema(), rangeQuery.getStartTimestamp(), rangeQuery.getEndTimestamp()) - + " order by time desc"; + + ORDER_BY_TIME_DESC; return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.RANGE_QUERY_ORDER_BY_TIME_DESC); } @@ -575,11 +576,26 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { */ @Override public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { - String sql = getValueRangeQuerySql(valueRangeQuery) + " order by time desc"; + String sql = getValueRangeQuerySql(valueRangeQuery) + ORDER_BY_TIME_DESC; return addSomeClauseAndExecuteQueryAndGetStatus( sql, Operation.VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC); } + /** Q11: Q7 order by time desc */ + @Override + public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { + String aggQuerySqlHead = + getAggQuerySqlHead(groupByQuery.getDeviceSchema(), groupByQuery.getAggFun()); + String sql = + addGroupByClause( + aggQuerySqlHead, + groupByQuery.getStartTimestamp(), + groupByQuery.getEndTimestamp(), + groupByQuery.getGranularity()); + sql += ORDER_BY_TIME_DESC; + return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY); + } + /** * Generate simple query header. * @@ -906,7 +922,7 @@ protected String getDeviceQuerySql( sql.append(getSimpleQuerySqlHead(deviceSchemas)); sql.append(" where time >= ").append(startTimeStamp); sql.append(" and time <").append(endTimeStamp); - sql.append(" order by time desc"); + sql.append(ORDER_BY_TIME_DESC); return sql.toString(); } @@ -940,7 +956,7 @@ protected String getMinTimeStampSql(DeviceSchema deviceSchema) { } protected String getMaxTimeStampSql(DeviceSchema deviceSchema) { - return "select * from " + getDevicePath(deviceSchema) + " order by time desc limit 1"; + return "select * from " + getDevicePath(deviceSchema) + ORDER_BY_TIME_DESC + " limit 1"; } String getEncodingType(SensorType dataSensorType) { diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index 37e6af3f9..29431e432 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -65,6 +65,7 @@ public class TDengine implements IDatabase { new CyclicBarrier(config.getCLIENT_NUMBER()); private static final String USE_DB = "use %s"; private static final String SUPER_TABLE_NAME = "device"; + private static final String ORDER_BY_TIME_DESC = " order by time desc "; private static final AtomicBoolean isInit = new AtomicBoolean(false); private final String CREATE_STABLE; @@ -385,7 +386,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { String sql = addWhereClause( rangeQueryHead, rangeQuery, null, getTableNameFilterForAlignByDevice(rangeQuery)); - sql += " order by time desc"; + sql += ORDER_BY_TIME_DESC; sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } @@ -401,7 +402,19 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { valueRangeQuery.getValueThreshold(), valueRangeQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(valueRangeQuery)); - sql += " order by time desc"; + sql += ORDER_BY_TIME_DESC; + sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); + return executeQueryAndGetStatus(sql); + } + + @Override + public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { + String sqlHeader = getAggQuerySqlHead(groupByQuery.getDeviceSchema(), groupByQuery.getAggFun()); + String sql = + addWhereClause( + sqlHeader, groupByQuery, null, getTableNameFilterForAlignByDevice(groupByQuery)); + sql = addGroupByClause(sql, groupByQuery.getGranularity()); + sql += ORDER_BY_TIME_DESC; sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); return executeQueryAndGetStatus(sql); } From 24cc39eb38efe60090c7459090d77176aecb6136 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 13:47:53 +0800 Subject: [PATCH 07/19] revert sth --- configuration/conf/config.properties | 20 +++++++++---------- .../iot/benchmark/tsdb/IDatabase.java | 4 ---- .../iot/benchmark/influxdb2/InfluxDB.java | 2 +- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 1dc47aa30..17e83e02f 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -4,25 +4,25 @@ ################### 被测数据库配置 ####################### # 被测试的数据库,目前的格式为{name}{-version}{-insert mode}(注意-号)其全部参考值参见README文件,注意版本需要匹配 - DB_SWITCH=InfluxDB-2.x +# DB_SWITCH=IoTDB-110-SESSION_BY_TABLET # 主机列表,如果有多个主机则使用英文逗号进行分割 -# HOST=192.168.130.37 +# HOST=127.0.0.1 # 端口列表,需要和HOST数量一致,保持一一对应。如果有多个端口则使用英文逗号进行分割。 - PORT=8086 +# PORT=6667 # 所有被测数据库的用户名,如果为多个数据库,则要求保持一致 - USERNAME=lyh +# USERNAME=root # 所有被测数据库的密码,如果为多个数据库,则要求保持一致 - PASSWORD=1234qwer +# PASSWORD=root # 即将被测试写入的数据库的名称 -# DB_NAME=gg +# DB_NAME=test # 数据库连接认证Token,InfluxDB 2.0使用 - TOKEN=tJY-JVZwJt9zJDUtmP_ZZaFVRA8Y_Tht_FHFrG5_IrtUingRxqiWiT2twUeu6IvOo4YFtdNeSkpTPEfcU9AyYQ== +# TOKEN=token ############### 数据库初始化:清理数据 #################### # 测试启动前是否清理数据 @@ -40,7 +40,7 @@ ################## Benchmark配置 ####################### # 总操作次数:具体每种类型操作会按OPERATION_PROPORTION定义的比例划分,例如OPERATION_PROPORTION=1:1:1:1:1:1:1:1:1:1:1 - LOOP=10 +# LOOP=100 # 目前Benchmark支持多种运行模式,支持模式如下 # testWithDefaultPath 常规测试模式,支持多种读和写操作的混合负载 @@ -111,7 +111,7 @@ ############## 数据:设备、传感器、客户端 ################## # 设备总数 - DEVICE_NUMBER=60 +# DEVICE_NUMBER=6000 # 实际写入设备比例,(0, 1] # REAL_INSERT_RATE=1.0 @@ -203,7 +203,7 @@ ############## 被测系统为Influxdb 2.x时扩展参数 ######## # influxdb ORG名 - INFLUXDB_ORG=timecho +# INFLUXDB_ORG=company1 ################### 操作时间配置 ######################### # 时间戳间隔,即生成的数据两个时间戳之间的固定长度(如果定长生成),非正常速率 diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java index 65ad613f3..028ca8a37 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/IDatabase.java @@ -193,8 +193,4 @@ default DeviceSummary deviceSummary(DeviceQuery deviceQuery) throws SQLException default String typeMap(SensorType iotdbSensorType) { return iotdbSensorType.name; } - // - // Status executeQueryAndGetStatus(String sql); - // - // Status executeQueryAndGetStatus(String sql, Operation operation); } diff --git a/influxdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb2/InfluxDB.java b/influxdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb2/InfluxDB.java index 420ddd39d..1ddd2551c 100644 --- a/influxdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb2/InfluxDB.java +++ b/influxdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb2/InfluxDB.java @@ -527,7 +527,7 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { return new Status(true, result); } - public Status executeQueryAndGetStatus(String sql) { + private Status executeQueryAndGetStatus(String sql) { if (!config.isIS_QUIET_MODE()) { LOGGER.debug("{} query SQL: {}", Thread.currentThread().getName(), sql); } From 99f3a94601e1387300da126c4f211532798310bf Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 13:52:02 +0800 Subject: [PATCH 08/19] revert sth --- configuration/conf/config.properties | 3 --- 1 file changed, 3 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 17e83e02f..15f22c6e6 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -387,9 +387,6 @@ # 查询语句的随机数种子 # QUERY_SEED=151658 -# 是否order by time desc -# ORDER_BY_TIME_DESC=false - # 行数限制, -1表示不使用limit语句 # ROW_LIMIT=-1 From a154d4811b8e78657fc9c2d05b47d9a5005b6b0c Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 14:59:20 +0800 Subject: [PATCH 09/19] test iotdb --- configuration/conf/config.properties | 13 ++++++---- .../generate/GenerateDataMixClient.java | 3 +++ .../benchmark/client/operation/Operation.java | 1 + .../client/operation/OperationController.java | 5 +++- .../iot/benchmark/tsdb/DBWrapper.java | 25 +++++++++++++++++++ .../iot/benchmark/iotdb110/IoTDB.java | 6 +++-- 6 files changed, 45 insertions(+), 8 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 15f22c6e6..75fd7a185 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -111,7 +111,7 @@ ############## 数据:设备、传感器、客户端 ################## # 设备总数 -# DEVICE_NUMBER=6000 + DEVICE_NUMBER=60 # 实际写入设备比例,(0, 1] # REAL_INSERT_RATE=1.0 @@ -298,7 +298,8 @@ # Q9 倒序范围查询(只限制起止时间)select v1... from data where time > ? and time < ? and device in ? order by time desc # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc # Q11 分组聚合查询,倒序 -# OPERATION_PROPORTION=0:1:0:0:0:0:0:0:0:0:0:0 + OPERATION_PROPORTION=1:1:1:0:0:0:0:0:0:1:0:0 +#OPERATION_PROPORTION=1:1:1:1:1:1:1:1:1:1:1:1 # 最长等待写时间,单位毫秒,即如果整个写操作在指定时间内没有返回,则终止此操作 # WRITE_OPERATION_TIMEOUT_MS=120000 @@ -370,7 +371,7 @@ # QUERY_SENSOR_NUM=1 # 每条查询语句中查询涉及到的设备数量 -# QUERY_DEVICE_NUM=1 + QUERY_DEVICE_NUM=5 # 在聚集查询中使用的聚集函数,比如count # QUERY_AGGREGATE_FUN=count @@ -388,10 +389,12 @@ # QUERY_SEED=151658 # 行数限制, -1表示不使用limit语句 -# ROW_LIMIT=-1 + RESULT_ROW_LIMIT=10 # 是否align by device -# ALIGN_BY_DEVICE=false +# 实测发现,iotdb对于align by device的使用有诸多限制 +# 推荐设置OPERATION_PROPORTION不超过范围1:1:1:0:0:0:0:0:0:1:0:0 + ALIGN_BY_DEVICE=true ################ Workload:相关参数 ###################### # workload的缓冲区的大小 diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/generate/GenerateDataMixClient.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/generate/GenerateDataMixClient.java index a8dffcfe7..edf91bdd9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/generate/GenerateDataMixClient.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/generate/GenerateDataMixClient.java @@ -95,6 +95,9 @@ protected void doTest() { case VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC: dbWrapper.valueRangeQueryOrderByDesc(queryWorkLoad.getValueRangeQuery()); break; + case GROUP_BY_QUERY_ORDER_BY_TIME_DESC: + dbWrapper.groupByQueryOrderByDesc(queryWorkLoad.getGroupByQuery()); + break; default: LOGGER.error("Unsupported operation sensorType {}", operation); } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/operation/Operation.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/operation/Operation.java index b0090ab7d..10134ac4c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/operation/Operation.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/operation/Operation.java @@ -34,6 +34,7 @@ public enum Operation { LATEST_POINT_QUERY("LATEST_POINT"), RANGE_QUERY_ORDER_BY_TIME_DESC("RANGE_QUERY_DESC"), VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC("VALUE_RANGE_QUERY_DESC"), + GROUP_BY_QUERY_ORDER_BY_TIME_DESC("GROUP_BY_DESC"), VERIFICATION_QUERY("VERIFICATION_QUERY"), DEVICE_QUERY("DEVICE_QUERY"); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/operation/OperationController.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/operation/OperationController.java index 4e761b337..dd6696311 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/operation/OperationController.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/client/operation/OperationController.java @@ -23,6 +23,7 @@ import cn.edu.tsinghua.iot.benchmark.conf.Config; import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -30,7 +31,7 @@ public class OperationController { - private static final Logger LOGGER = null; + private static final Logger LOGGER = LoggerFactory.getLogger(OperationController.class); private static Config config = ConfigDescriptor.getInstance().getConfig(); private List proportion = new ArrayList<>(); private List operations = Operation.getNormalOperation(); @@ -108,6 +109,8 @@ public Operation getNextOperationType() { return Operation.RANGE_QUERY_ORDER_BY_TIME_DESC; case 11: return Operation.VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC; + case 12: + return Operation.GROUP_BY_QUERY_ORDER_BY_TIME_DESC; default: LOGGER.error("Unsupported operation {}, use default operation: INGESTION.", i); return Operation.INGESTION; diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBWrapper.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBWrapper.java index efa398dbd..3bb7f4ab6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBWrapper.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBWrapper.java @@ -317,6 +317,31 @@ public Status groupByQuery(GroupByQuery groupByQuery) { return status; } + @Override + public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { + Status status = null; + Operation operation = Operation.GROUP_BY_QUERY_ORDER_BY_TIME_DESC; + String device = "No Device"; + if (groupByQuery.getDeviceSchema().size() > 0) { + device = groupByQuery.getDeviceSchema().get(0).getDevice(); + } + try { + List statuses = new ArrayList<>(); + for (IDatabase database : databases) { + long start = System.nanoTime(); + status = database.groupByQueryOrderByDesc(groupByQuery); + long end = System.nanoTime(); + status.setTimeCost(end - start); + handleQueryOperation(status, operation, device); + statuses.add(status); + } + doComparisonByRecord(groupByQuery, operation, statuses); + } catch (Exception e) { + handleUnexpectedQueryException(operation, e, device); + } + return status; + } + @Override public Status latestPointQuery(LatestPointQuery latestPointQuery) { Status status = null; diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java index 7634c6881..fdc559b17 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java @@ -593,7 +593,8 @@ public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { groupByQuery.getEndTimestamp(), groupByQuery.getGranularity()); sql += ORDER_BY_TIME_DESC; - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY); + return addSomeClauseAndExecuteQueryAndGetStatus( + sql, Operation.GROUP_BY_QUERY_ORDER_BY_TIME_DESC); } /** @@ -764,7 +765,8 @@ protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation isOk.set(false); } long resultPointNum = line.get(); - if (!Operation.LATEST_POINT_QUERY.equals(operation)) { + if (!Operation.LATEST_POINT_QUERY.equals(operation) + && !config.isALIGN_BY_DEVICE()) { resultPointNum *= config.getQUERY_SENSOR_NUM(); resultPointNum *= config.getQUERY_DEVICE_NUM(); } From 67cbd2f19a02f239b64d2c6891b98536e77845df Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 17:02:53 +0800 Subject: [PATCH 10/19] test tdengion, good --- .../edu/tsinghua/iot/benchmark/tdengine3/TDengine.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index 29431e432..87d8e4be9 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -19,6 +19,7 @@ package cn.edu.tsinghua.iot.benchmark.tdengine3; +import cn.edu.tsinghua.iot.benchmark.client.operation.OperationController; import cn.edu.tsinghua.iot.benchmark.conf.Config; import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; @@ -180,6 +181,7 @@ public Double registerSchema(List schemaList) throws TsdbException private synchronized void initOnlyOnce() throws TsdbException, BrokenBarrierException, InterruptedException { + new OperationController(1).getNextOperationType(); if (!isInit.getAndSet(true)) { try (Statement statement = connection.createStatement()) { LOGGER.info("Create Database: {}", CREATE_DATABASE); @@ -504,23 +506,24 @@ private static String addWhereClause( return sql; } StringBuilder sqlBuilder = new StringBuilder(sql); - sqlBuilder.append(" WHERE"); + sqlBuilder.append(" WHERE "); if (timeRangeQuery != null) { String startTime = "" + timeRangeQuery.getStartTimestamp(); String endTime = "" + timeRangeQuery.getEndTimestamp(); sqlBuilder.append(" time >= ").append(startTime).append(" AND time <= ").append(endTime); } if (valueRangeFilter != null) { - if (!sqlBuilder.toString().endsWith("WHERE")) { + if (!sqlBuilder.toString().endsWith("WHERE ")) { sqlBuilder.append(" AND "); } double valueThreshold = valueRangeFilter.getMinValue(); for (Sensor sensor : valueRangeFilter.getSensors()) { sqlBuilder.append(sensor.getName()).append(" > ").append(valueThreshold).append(" AND "); } + sqlBuilder.delete(sqlBuilder.length() - 4, sqlBuilder.length()); } if (alignByDeviceTableNameFilter != null) { - if (!sqlBuilder.toString().endsWith("WHERE")) { + if (!sqlBuilder.toString().endsWith("WHERE ")) { sqlBuilder.append(" AND "); } sqlBuilder.append(" tbname in ("); From 7bbf5f0b1b497f692d380ae5feb51526a2ee19aa Mon Sep 17 00:00:00 2001 From: liyuheng Date: Fri, 29 Dec 2023 17:07:05 +0800 Subject: [PATCH 11/19] modify config --- configuration/conf/config.properties | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 75fd7a185..5dd3d8cb9 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -111,7 +111,7 @@ ############## 数据:设备、传感器、客户端 ################## # 设备总数 - DEVICE_NUMBER=60 +# DEVICE_NUMBER=6000 # 实际写入设备比例,(0, 1] # REAL_INSERT_RATE=1.0 @@ -298,8 +298,7 @@ # Q9 倒序范围查询(只限制起止时间)select v1... from data where time > ? and time < ? and device in ? order by time desc # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc # Q11 分组聚合查询,倒序 - OPERATION_PROPORTION=1:1:1:0:0:0:0:0:0:1:0:0 -#OPERATION_PROPORTION=1:1:1:1:1:1:1:1:1:1:1:1 +# OPERATION_PROPORTION=1:0:0:0:0:0:0:0:0:0:0:0 # 最长等待写时间,单位毫秒,即如果整个写操作在指定时间内没有返回,则终止此操作 # WRITE_OPERATION_TIMEOUT_MS=120000 @@ -371,7 +370,7 @@ # QUERY_SENSOR_NUM=1 # 每条查询语句中查询涉及到的设备数量 - QUERY_DEVICE_NUM=5 +# QUERY_DEVICE_NUM=1 # 在聚集查询中使用的聚集函数,比如count # QUERY_AGGREGATE_FUN=count @@ -389,12 +388,12 @@ # QUERY_SEED=151658 # 行数限制, -1表示不使用limit语句 - RESULT_ROW_LIMIT=10 +# RESULT_ROW_LIMIT=-1 # 是否align by device -# 实测发现,iotdb对于align by device的使用有诸多限制 -# 推荐设置OPERATION_PROPORTION不超过范围1:1:1:0:0:0:0:0:0:1:0:0 - ALIGN_BY_DEVICE=true +# 实测发现, iotdb对于align by device的使用有诸多限制, 推荐设置OPERATION_PROPORTION不超过范围1:1:1:0:0:0:0:0:0:1:0:0 +# td可设置1:1:1:1:1:1:1:1:1:1:1:0 +# ALIGN_BY_DEVICE=false ################ Workload:相关参数 ###################### # workload的缓冲区的大小 From 3ed71eddaeca4a4cac68f40a41ce31cdc0138830 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 2 Jan 2024 16:07:20 +0800 Subject: [PATCH 12/19] influxdb limit done? --- .../iot/benchmark/influxdb/InfluxDB.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java index 64d594f7e..89868c4ae 100644 --- a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java +++ b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java @@ -140,7 +140,7 @@ public Status insertOneBatch(IBatch batch) { @Override public Status preciseQuery(PreciseQuery preciseQuery) { String sql = getPreciseQuerySql(preciseQuery); - return executeQueryAndGetStatus(sql); + return addSomeClauseAndExecuteQueryAndGetStatus(sql); } /** @@ -151,7 +151,7 @@ public Status preciseQuery(PreciseQuery preciseQuery) { public Status rangeQuery(RangeQuery rangeQuery) { String rangeQueryHead = getSimpleQuerySqlHead(rangeQuery.getDeviceSchema()); String sql = addWhereTimeClause(rangeQueryHead, rangeQuery); - return executeQueryAndGetStatus(sql); + return addSomeClauseAndExecuteQueryAndGetStatus(sql); } /** @@ -167,7 +167,7 @@ public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { valueRangeQuery.getDeviceSchema(), sqlWithTimeFilter, valueRangeQuery.getValueThreshold()); - return executeQueryAndGetStatus(sqlWithValueFilter); + return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithValueFilter); } /** @@ -179,7 +179,7 @@ public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { String aggQuerySqlHead = getAggQuerySqlHead(aggRangeQuery.getDeviceSchema(), aggRangeQuery.getAggFun()); String sql = addWhereTimeClause(aggQuerySqlHead, aggRangeQuery); - return executeQueryAndGetStatus(sql); + return addSomeClauseAndExecuteQueryAndGetStatus(sql); } /** eg. SELECT count(s_3) FROM group_3 WHERE ( device = 'd_12' ) AND s_3 > -5.0. */ @@ -190,7 +190,7 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { String sql = addWhereValueClause( aggValueQuery.getDeviceSchema(), aggQuerySqlHead, aggValueQuery.getValueThreshold()); - return executeQueryAndGetStatus(sql); + return addSomeClauseAndExecuteQueryAndGetStatus(sql); } /** @@ -207,7 +207,7 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { aggRangeValueQuery.getDeviceSchema(), sqlWithTimeFilter, aggRangeValueQuery.getValueThreshold()); - return executeQueryAndGetStatus(sqlWithValueFilter); + return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithValueFilter); } /** @@ -219,14 +219,14 @@ public Status groupByQuery(GroupByQuery groupByQuery) { String sqlHeader = getAggQuerySqlHead(groupByQuery.getDeviceSchema(), groupByQuery.getAggFun()); String sqlWithTimeFilter = addWhereTimeClause(sqlHeader, groupByQuery); String sqlWithGroupBy = addGroupByClause(sqlWithTimeFilter, groupByQuery.getGranularity()); - return executeQueryAndGetStatus(sqlWithGroupBy); + return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithGroupBy); } /** eg. SELECT last(s_2) FROM group_2 WHERE ( device = 'd_8' ). */ @Override public Status latestPointQuery(LatestPointQuery latestPointQuery) { String sql = getAggQuerySqlHead(latestPointQuery.getDeviceSchema(), "last"); - return executeQueryAndGetStatus(sql); + return addSomeClauseAndExecuteQueryAndGetStatus(sql); } @Override @@ -234,7 +234,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { String rangeQueryHead = getSimpleQuerySqlHead(rangeQuery.getDeviceSchema()); String sql = addWhereTimeClause(rangeQueryHead, rangeQuery); sql = addDescClause(sql); - return executeQueryAndGetStatus(sql); + return addSomeClauseAndExecuteQueryAndGetStatus(sql); } @Override @@ -247,7 +247,7 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { sqlWithTimeFilter, valueRangeQuery.getValueThreshold()); sqlWithValueFilter = addDescClause(sqlWithValueFilter); - return executeQueryAndGetStatus(sqlWithValueFilter); + return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithValueFilter); } private String addDescClause(String sql) { @@ -289,7 +289,11 @@ private InfluxDataModel createDataModel( return model; } - private Status executeQueryAndGetStatus(String sql) { + private Status addSomeClauseAndExecuteQueryAndGetStatus(String sql) { + if (config.getRESULT_ROW_LIMIT() >= 0) { + sql += " limit " + config.getRESULT_ROW_LIMIT(); + } + LOGGER.info("do query: {}", sql); if (!config.isIS_QUIET_MODE()) { LOGGER.debug("{} query SQL: {}", Thread.currentThread().getName(), sql); } From e327c757effd3792299e15bcc386b55fbcdadb8a Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 2 Jan 2024 16:13:42 +0800 Subject: [PATCH 13/19] influxdb alignByDevice done? --- .../cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java index 89868c4ae..3ac75afa3 100644 --- a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java +++ b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java @@ -374,8 +374,10 @@ private static String addGroupByClause(String sqlHeader, long timeGranularity) { private static String getSimpleQuerySqlHead(List devices) { StringBuilder builder = new StringBuilder(); builder.append("SELECT "); + if (config.isALIGN_BY_DEVICE()) { + builder.append("device, "); + } List querySensors = devices.get(0).getSensors(); - builder.append(querySensors.get(0).getName()); for (int i = 1; i < querySensors.size(); i++) { builder.append(", ").append(querySensors.get(i).getName()); @@ -395,8 +397,10 @@ private static String getSimpleQuerySqlHead(List devices) { private static String getAggQuerySqlHead(List devices, String method) { StringBuilder builder = new StringBuilder(); builder.append("SELECT "); + if (config.isALIGN_BY_DEVICE()) { + builder.append("device, "); + } List querySensors = devices.get(0).getSensors(); - builder.append(method).append("(").append(querySensors.get(0).getName()).append(")"); for (int i = 1; i < querySensors.size(); i++) { builder From 703ca935059e019bc84ab83a63b83102dc94f1b4 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 2 Jan 2024 16:16:37 +0800 Subject: [PATCH 14/19] influxdb desc done? --- .../cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java index 3ac75afa3..24e9aa8d1 100644 --- a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java +++ b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java @@ -250,6 +250,15 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithValueFilter); } + @Override + public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { + String sql = getAggQuerySqlHead(groupByQuery.getDeviceSchema(), groupByQuery.getAggFun()); + sql = addWhereTimeClause(sql, groupByQuery); + sql = addGroupByClause(sql, groupByQuery.getGranularity()); + sql = addDescClause(sql); + return addSomeClauseAndExecuteQueryAndGetStatus(sql); + } + private String addDescClause(String sql) { return sql + " ORDER BY time DESC"; } From 694ebb337e4213ad3e5408f661763190104add2e Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 2 Jan 2024 16:54:04 +0800 Subject: [PATCH 15/19] small things --- configuration/conf/config.properties | 4 +++- .../edu/tsinghua/iot/benchmark/conf/Config.java | 5 +++++ .../iot/benchmark/conf/ConfigDescriptor.java | 16 +++++++++++----- .../iot/benchmark/iotdb110/IoTDBSessionBase.java | 2 +- .../iot/benchmark/tdengine3/TDengine.java | 8 ++++---- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 5dd3d8cb9..d9be746a0 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -297,7 +297,7 @@ # Q8 最近点查询 select time, v1... where device = ? and time = max(time) # Q9 倒序范围查询(只限制起止时间)select v1... from data where time > ? and time < ? and device in ? order by time desc # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc -# Q11 分组聚合查询,倒序 +# Q11 分组聚合查询,倒序,# 目前仅支持iotdb、influxdb v1 # OPERATION_PROPORTION=1:0:0:0:0:0:0:0:0:0:0:0 # 最长等待写时间,单位毫秒,即如果整个写操作在指定时间内没有返回,则终止此操作 @@ -388,9 +388,11 @@ # QUERY_SEED=151658 # 行数限制, -1表示不使用limit语句 +# 目前仅支持iotdb、influxdb v1、tdengine-3 # RESULT_ROW_LIMIT=-1 # 是否align by device +# 目前仅支持iotdb、influxdb v1、tdengine-3 # 实测发现, iotdb对于align by device的使用有诸多限制, 推荐设置OPERATION_PROPORTION不超过范围1:1:1:0:0:0:0:0:0:1:0:0 # td可设置1:1:1:1:1:1:1:1:1:1:1:0 # ALIGN_BY_DEVICE=false diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index 5014816f3..78fce5c4d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -320,6 +320,7 @@ public class Config { * > ? and device in ? order by time desc */ private String OPERATION_PROPORTION = "1:0:0:0:0:0:0:0:0:0:0:0"; + private final int OPERATION_PROPORTION_LEN = 12; /** The number of sensors involved in each query */ private int QUERY_SENSOR_NUM = 1; /** The number of devices involved in each query */ @@ -1201,6 +1202,10 @@ public void setOPERATION_PROPORTION(String OPERATION_PROPORTION) { this.OPERATION_PROPORTION = OPERATION_PROPORTION; } + public int getOPERATION_PROPORTION_LEN() { + return this.OPERATION_PROPORTION_LEN; + } + public int getQUERY_SENSOR_NUM() { return QUERY_SENSOR_NUM; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index f1669ab7d..68aa5451f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -32,6 +32,7 @@ import java.io.*; import java.lang.reflect.Field; import java.util.Arrays; +import java.util.Objects; import java.util.Properties; import java.util.stream.Collectors; @@ -594,26 +595,31 @@ private boolean checkConfig() { } result &= checkDeviceNumPerWrite(); result &= checkTag(); - if (!isGoodDB()) { + if (!commonlyUseDB()) { if (config.isALIGN_BY_DEVICE()) { + result = false; LOGGER.error("ALIGN_BY_DEVICE is not supported for this database"); } if (config.getRESULT_ROW_LIMIT() >= 0) { + result = false; LOGGER.error("RESULT_ROW_LIMIT is not supported for this database"); } } return result; } - private boolean isGoodDB() { + private boolean commonlyUseDB() { DBType dbType = config.getDbConfig().getDB_SWITCH().getType(); DBVersion dbVersion = config.getDbConfig().getDB_SWITCH().getVersion(); - return dbType.equals(DBType.IoTDB) - || dbVersion.equals(DBVersion.TDengine_3) - || dbVersion.equals(DBVersion.InfluxDB_2); + return Objects.equals(dbVersion, DBVersion.IOTDB_110) + || Objects.equals(dbVersion, DBVersion.TDengine_3) + || (Objects.equals(dbType, DBType.InfluxDB) && dbVersion == null); } private boolean checkOperationProportion() { + while (config.getOPERATION_PROPORTION().split(":").length != config.getOPERATION_PROPORTION_LEN()) { + config.setOPERATION_PROPORTION(config.getOPERATION_PROPORTION() + ":0"); + } String[] op = config.getOPERATION_PROPORTION().split(":"); int minOps = 0; for (String s : op) { diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java index edc9a45ff..95c3873a1 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java @@ -236,7 +236,7 @@ protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation isOk.set(false); } long resultPointNum = line.get(); - if (!Operation.LATEST_POINT_QUERY.equals(operation)) { + if (!Operation.LATEST_POINT_QUERY.equals(operation) && !config.isALIGN_BY_DEVICE()) { resultPointNum *= config.getQUERY_SENSOR_NUM(); resultPointNum *= config.getQUERY_DEVICE_NUM(); } diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index 87d8e4be9..999c8ab9c 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -51,6 +51,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; @@ -181,7 +182,6 @@ public Double registerSchema(List schemaList) throws TsdbException private synchronized void initOnlyOnce() throws TsdbException, BrokenBarrierException, InterruptedException { - new OperationController(1).getNextOperationType(); if (!isInit.getAndSet(true)) { try (Statement statement = connection.createStatement()) { LOGGER.info("Create Database: {}", CREATE_DATABASE); @@ -502,7 +502,7 @@ private static String addWhereClause( List alignByDeviceTableNameFilter) { if (timeRangeQuery == null && valueRangeFilter == null - && alignByDeviceTableNameFilter == null) { + && !alignByDeviceTableNameFilter.isEmpty()) { return sql; } StringBuilder sqlBuilder = new StringBuilder(sql); @@ -522,7 +522,7 @@ private static String addWhereClause( } sqlBuilder.delete(sqlBuilder.length() - 4, sqlBuilder.length()); } - if (alignByDeviceTableNameFilter != null) { + if (!alignByDeviceTableNameFilter.isEmpty()) { if (!sqlBuilder.toString().endsWith("WHERE ")) { sqlBuilder.append(" AND "); } @@ -540,7 +540,7 @@ private static List getTableNameFilterForAlignByDevice(RangeQuery if (config.isALIGN_BY_DEVICE()) { return rangeQuery.getDeviceSchema(); } - return null; + return Collections.emptyList(); } private static String addLimitClause(String sql, long limitation) { From 9745f6b4a9bc6c2fd54d9bdb380634efd400cc56 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 2 Jan 2024 16:55:03 +0800 Subject: [PATCH 16/19] spotless --- .../main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java | 1 + .../cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 3 ++- .../edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java | 3 ++- .../java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java | 1 - 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index 78fce5c4d..2df8af854 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -320,6 +320,7 @@ public class Config { * > ? and device in ? order by time desc */ private String OPERATION_PROPORTION = "1:0:0:0:0:0:0:0:0:0:0:0"; + private final int OPERATION_PROPORTION_LEN = 12; /** The number of sensors involved in each query */ private int QUERY_SENSOR_NUM = 1; diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 68aa5451f..07fbc60df 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -617,7 +617,8 @@ private boolean commonlyUseDB() { } private boolean checkOperationProportion() { - while (config.getOPERATION_PROPORTION().split(":").length != config.getOPERATION_PROPORTION_LEN()) { + while (config.getOPERATION_PROPORTION().split(":").length + != config.getOPERATION_PROPORTION_LEN()) { config.setOPERATION_PROPORTION(config.getOPERATION_PROPORTION() + ":0"); } String[] op = config.getOPERATION_PROPORTION().split(":"); diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java index 95c3873a1..df8939de2 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java @@ -236,7 +236,8 @@ protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation isOk.set(false); } long resultPointNum = line.get(); - if (!Operation.LATEST_POINT_QUERY.equals(operation) && !config.isALIGN_BY_DEVICE()) { + if (!Operation.LATEST_POINT_QUERY.equals(operation) + && !config.isALIGN_BY_DEVICE()) { resultPointNum *= config.getQUERY_SENSOR_NUM(); resultPointNum *= config.getQUERY_DEVICE_NUM(); } diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index 999c8ab9c..57015c4ea 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -19,7 +19,6 @@ package cn.edu.tsinghua.iot.benchmark.tdengine3; -import cn.edu.tsinghua.iot.benchmark.client.operation.OperationController; import cn.edu.tsinghua.iot.benchmark.conf.Config; import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; From 9504a4c8c1127d0289c318801a8525d7ae4fc641 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 2 Jan 2024 17:13:53 +0800 Subject: [PATCH 17/19] ohhhh --- configuration/conf/config.properties | 2 +- .../iot/benchmark/influxdb/InfluxDB.java | 24 +++++------ .../iot/benchmark/iotdb110/IoTDB.java | 26 +++++------ .../benchmark/iotdb110/IoTDBSessionBase.java | 4 +- .../iot/benchmark/tdengine3/TDengine.java | 43 +++++++------------ 5 files changed, 43 insertions(+), 56 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index d9be746a0..e9e1e0883 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -297,7 +297,7 @@ # Q8 最近点查询 select time, v1... where device = ? and time = max(time) # Q9 倒序范围查询(只限制起止时间)select v1... from data where time > ? and time < ? and device in ? order by time desc # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc -# Q11 分组聚合查询,倒序,# 目前仅支持iotdb、influxdb v1 +# Q11 分组聚合查询,倒序;目前仅支持iotdb、influxdb v1 # OPERATION_PROPORTION=1:0:0:0:0:0:0:0:0:0:0:0 # 最长等待写时间,单位毫秒,即如果整个写操作在指定时间内没有返回,则终止此操作 diff --git a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java index 24e9aa8d1..7801c64eb 100644 --- a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java +++ b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java @@ -140,7 +140,7 @@ public Status insertOneBatch(IBatch batch) { @Override public Status preciseQuery(PreciseQuery preciseQuery) { String sql = getPreciseQuerySql(preciseQuery); - return addSomeClauseAndExecuteQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** @@ -151,7 +151,7 @@ public Status preciseQuery(PreciseQuery preciseQuery) { public Status rangeQuery(RangeQuery rangeQuery) { String rangeQueryHead = getSimpleQuerySqlHead(rangeQuery.getDeviceSchema()); String sql = addWhereTimeClause(rangeQueryHead, rangeQuery); - return addSomeClauseAndExecuteQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** @@ -167,7 +167,7 @@ public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { valueRangeQuery.getDeviceSchema(), sqlWithTimeFilter, valueRangeQuery.getValueThreshold()); - return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithValueFilter); + return addTailClausesAndExecuteQueryAndGetStatus(sqlWithValueFilter); } /** @@ -179,7 +179,7 @@ public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { String aggQuerySqlHead = getAggQuerySqlHead(aggRangeQuery.getDeviceSchema(), aggRangeQuery.getAggFun()); String sql = addWhereTimeClause(aggQuerySqlHead, aggRangeQuery); - return addSomeClauseAndExecuteQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** eg. SELECT count(s_3) FROM group_3 WHERE ( device = 'd_12' ) AND s_3 > -5.0. */ @@ -190,7 +190,7 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { String sql = addWhereValueClause( aggValueQuery.getDeviceSchema(), aggQuerySqlHead, aggValueQuery.getValueThreshold()); - return addSomeClauseAndExecuteQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** @@ -207,7 +207,7 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { aggRangeValueQuery.getDeviceSchema(), sqlWithTimeFilter, aggRangeValueQuery.getValueThreshold()); - return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithValueFilter); + return addTailClausesAndExecuteQueryAndGetStatus(sqlWithValueFilter); } /** @@ -219,14 +219,14 @@ public Status groupByQuery(GroupByQuery groupByQuery) { String sqlHeader = getAggQuerySqlHead(groupByQuery.getDeviceSchema(), groupByQuery.getAggFun()); String sqlWithTimeFilter = addWhereTimeClause(sqlHeader, groupByQuery); String sqlWithGroupBy = addGroupByClause(sqlWithTimeFilter, groupByQuery.getGranularity()); - return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithGroupBy); + return addTailClausesAndExecuteQueryAndGetStatus(sqlWithGroupBy); } /** eg. SELECT last(s_2) FROM group_2 WHERE ( device = 'd_8' ). */ @Override public Status latestPointQuery(LatestPointQuery latestPointQuery) { String sql = getAggQuerySqlHead(latestPointQuery.getDeviceSchema(), "last"); - return addSomeClauseAndExecuteQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } @Override @@ -234,7 +234,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { String rangeQueryHead = getSimpleQuerySqlHead(rangeQuery.getDeviceSchema()); String sql = addWhereTimeClause(rangeQueryHead, rangeQuery); sql = addDescClause(sql); - return addSomeClauseAndExecuteQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } @Override @@ -247,7 +247,7 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { sqlWithTimeFilter, valueRangeQuery.getValueThreshold()); sqlWithValueFilter = addDescClause(sqlWithValueFilter); - return addSomeClauseAndExecuteQueryAndGetStatus(sqlWithValueFilter); + return addTailClausesAndExecuteQueryAndGetStatus(sqlWithValueFilter); } @Override @@ -256,7 +256,7 @@ public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { sql = addWhereTimeClause(sql, groupByQuery); sql = addGroupByClause(sql, groupByQuery.getGranularity()); sql = addDescClause(sql); - return addSomeClauseAndExecuteQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } private String addDescClause(String sql) { @@ -298,7 +298,7 @@ private InfluxDataModel createDataModel( return model; } - private Status addSomeClauseAndExecuteQueryAndGetStatus(String sql) { + private Status addTailClausesAndExecuteQueryAndGetStatus(String sql) { if (config.getRESULT_ROW_LIMIT() >= 0) { sql += " limit " + config.getRESULT_ROW_LIMIT(); } diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java index fdc559b17..b1bae6a4c 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java @@ -425,7 +425,7 @@ public Status insertOneBatch(IBatch batch) throws DBConnectException { public Status preciseQuery(PreciseQuery preciseQuery) { String strTime = preciseQuery.getTimestamp() + ""; String sql = getSimpleQuerySqlHead(preciseQuery.getDeviceSchema()) + " WHERE time = " + strTime; - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.PRECISE_QUERY); + return addTailClausesAndExecuteQueryAndGetStatus(sql, Operation.PRECISE_QUERY); } /** @@ -442,7 +442,7 @@ public Status rangeQuery(RangeQuery rangeQuery) { rangeQuery.getDeviceSchema(), rangeQuery.getStartTimestamp(), rangeQuery.getEndTimestamp()); - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.RANGE_QUERY); + return addTailClausesAndExecuteQueryAndGetStatus(sql, Operation.RANGE_QUERY); } /** @@ -455,7 +455,7 @@ public Status rangeQuery(RangeQuery rangeQuery) { @Override public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { String sql = getValueRangeQuerySql(valueRangeQuery); - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.VALUE_RANGE_QUERY); + return addTailClausesAndExecuteQueryAndGetStatus(sql, Operation.VALUE_RANGE_QUERY); } /** @@ -472,7 +472,7 @@ public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { String sql = addWhereTimeClause( aggQuerySqlHead, aggRangeQuery.getStartTimestamp(), aggRangeQuery.getEndTimestamp()); - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.AGG_RANGE_QUERY); + return addTailClausesAndExecuteQueryAndGetStatus(sql, Operation.AGG_RANGE_QUERY); } /** @@ -491,7 +491,7 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { + getValueFilterClause( aggValueQuery.getDeviceSchema(), (int) aggValueQuery.getValueThreshold()) .substring(4); - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.AGG_VALUE_QUERY); + return addTailClausesAndExecuteQueryAndGetStatus(sql, Operation.AGG_VALUE_QUERY); } /** @@ -514,7 +514,7 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { sql += getValueFilterClause( aggRangeValueQuery.getDeviceSchema(), (int) aggRangeValueQuery.getValueThreshold()); - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.AGG_RANGE_VALUE_QUERY); + return addTailClausesAndExecuteQueryAndGetStatus(sql, Operation.AGG_RANGE_VALUE_QUERY); } /** @@ -534,7 +534,7 @@ public Status groupByQuery(GroupByQuery groupByQuery) { groupByQuery.getStartTimestamp(), groupByQuery.getEndTimestamp(), groupByQuery.getGranularity()); - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY); + return addTailClausesAndExecuteQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY); } /** @@ -546,7 +546,7 @@ public Status groupByQuery(GroupByQuery groupByQuery) { @Override public Status latestPointQuery(LatestPointQuery latestPointQuery) { String aggQuerySqlHead = getLatestPointQuerySql(latestPointQuery.getDeviceSchema()); - return addSomeClauseAndExecuteQueryAndGetStatus(aggQuerySqlHead, Operation.LATEST_POINT_QUERY); + return addTailClausesAndExecuteQueryAndGetStatus(aggQuerySqlHead, Operation.LATEST_POINT_QUERY); } /** @@ -564,7 +564,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { rangeQuery.getStartTimestamp(), rangeQuery.getEndTimestamp()) + ORDER_BY_TIME_DESC; - return addSomeClauseAndExecuteQueryAndGetStatus(sql, Operation.RANGE_QUERY_ORDER_BY_TIME_DESC); + return addTailClausesAndExecuteQueryAndGetStatus(sql, Operation.RANGE_QUERY_ORDER_BY_TIME_DESC); } /** @@ -577,7 +577,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { @Override public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { String sql = getValueRangeQuerySql(valueRangeQuery) + ORDER_BY_TIME_DESC; - return addSomeClauseAndExecuteQueryAndGetStatus( + return addTailClausesAndExecuteQueryAndGetStatus( sql, Operation.VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC); } @@ -593,7 +593,7 @@ public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { groupByQuery.getEndTimestamp(), groupByQuery.getGranularity()); sql += ORDER_BY_TIME_DESC; - return addSomeClauseAndExecuteQueryAndGetStatus( + return addTailClausesAndExecuteQueryAndGetStatus( sql, Operation.GROUP_BY_QUERY_ORDER_BY_TIME_DESC); } @@ -714,7 +714,7 @@ protected String getDevicePath(DeviceSchema deviceSchema) { return name.toString(); } - protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation operation) { + protected Status addTailClausesAndExecuteQueryAndGetStatus(String sql, Operation operation) { if (config.getRESULT_ROW_LIMIT() >= 0) { sql += " limit " + config.getRESULT_ROW_LIMIT(); } @@ -766,7 +766,7 @@ protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation } long resultPointNum = line.get(); if (!Operation.LATEST_POINT_QUERY.equals(operation) - && !config.isALIGN_BY_DEVICE()) { + || !config.isALIGN_BY_DEVICE()) { resultPointNum *= config.getQUERY_SENSOR_NUM(); resultPointNum *= config.getQUERY_DEVICE_NUM(); } diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java index df8939de2..e02113a79 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDBSessionBase.java @@ -169,7 +169,7 @@ public Status insertOneBatchByRecords(IBatch batch) { } @Override - protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation operation) { + protected Status addTailClausesAndExecuteQueryAndGetStatus(String sql, Operation operation) { if (config.getRESULT_ROW_LIMIT() >= 0) { sql += " limit " + config.getRESULT_ROW_LIMIT(); } @@ -237,7 +237,7 @@ protected Status addSomeClauseAndExecuteQueryAndGetStatus(String sql, Operation } long resultPointNum = line.get(); if (!Operation.LATEST_POINT_QUERY.equals(operation) - && !config.isALIGN_BY_DEVICE()) { + || !config.isALIGN_BY_DEVICE()) { resultPointNum *= config.getQUERY_SENSOR_NUM(); resultPointNum *= config.getQUERY_DEVICE_NUM(); } diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index 57015c4ea..819c707f3 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -268,7 +268,7 @@ private String getInsertOneRecordSql( @Override public Status preciseQuery(PreciseQuery preciseQuery) { String sql = getPreciseQuerySql(preciseQuery); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** @@ -281,8 +281,7 @@ public Status rangeQuery(RangeQuery rangeQuery) { String sql = addWhereClause( rangeQueryHead, rangeQuery, null, getTableNameFilterForAlignByDevice(rangeQuery)); - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** @@ -300,8 +299,7 @@ public Status valueRangeQuery(ValueRangeQuery valueRangeQuery) { valueRangeQuery.getValueThreshold(), valueRangeQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(valueRangeQuery)); - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** @@ -318,8 +316,7 @@ public Status aggRangeQuery(AggRangeQuery aggRangeQuery) { aggRangeQuery, null, getTableNameFilterForAlignByDevice(aggRangeQuery)); - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** eg. SELECT count(s_3) FROM group_3 WHERE ( device = 'd_12' ) AND s_3 > -5.0. */ @@ -335,8 +332,7 @@ public Status aggValueQuery(AggValueQuery aggValueQuery) { aggValueQuery.getValueThreshold(), aggValueQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(aggValueQuery)); - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** @@ -355,8 +351,7 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { aggRangeValueQuery.getValueThreshold(), aggRangeValueQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(aggRangeValueQuery)); - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** @@ -370,15 +365,14 @@ public Status groupByQuery(GroupByQuery groupByQuery) { addWhereClause( sqlHeader, groupByQuery, null, getTableNameFilterForAlignByDevice(groupByQuery)); sql = addGroupByClause(sql, groupByQuery.getGranularity()); - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } /** eg. SELECT last(s_2) FROM group_2 WHERE ( device = 'd_8' ). */ @Override public Status latestPointQuery(LatestPointQuery latestPointQuery) { String sql = getAggQuerySqlHead(latestPointQuery.getDeviceSchema(), "last"); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } @Override @@ -388,8 +382,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { addWhereClause( rangeQueryHead, rangeQuery, null, getTableNameFilterForAlignByDevice(rangeQuery)); sql += ORDER_BY_TIME_DESC; - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } @Override @@ -404,8 +397,7 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { valueRangeQuery.getDeviceSchema().get(0).getSensors()), getTableNameFilterForAlignByDevice(valueRangeQuery)); sql += ORDER_BY_TIME_DESC; - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } @Override @@ -416,8 +408,7 @@ public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { sqlHeader, groupByQuery, null, getTableNameFilterForAlignByDevice(groupByQuery)); sql = addGroupByClause(sql, groupByQuery.getGranularity()); sql += ORDER_BY_TIME_DESC; - sql = addLimitClause(sql, config.getRESULT_ROW_LIMIT()); - return executeQueryAndGetStatus(sql); + return addTailClausesAndExecuteQueryAndGetStatus(sql); } private String getPreciseQuerySql(PreciseQuery preciseQuery) { @@ -462,7 +453,10 @@ private static String generateConstrainForDevices(List devices) { return builder.toString(); } - private Status executeQueryAndGetStatus(String sql) { + private Status addTailClausesAndExecuteQueryAndGetStatus(String sql) { + if (config.getRESULT_ROW_LIMIT() >= 0) { + sql += " limit " + config.getRESULT_ROW_LIMIT(); + } if (!config.isIS_QUIET_MODE()) { LOGGER.info("{} query SQL: {}", Thread.currentThread().getName(), sql); } @@ -542,13 +536,6 @@ private static List getTableNameFilterForAlignByDevice(RangeQuery return Collections.emptyList(); } - private static String addLimitClause(String sql, long limitation) { - if (config.getRESULT_ROW_LIMIT() >= 0) { - sql += " limit " + limitation; - } - return sql; - } - /** * generate aggregation query header. * From 48c95e71e90233e07804ff5654b7797df232e5ff Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 2 Jan 2024 18:15:24 +0800 Subject: [PATCH 18/19] ohhhh --- configuration/conf/config.properties | 6 +++--- .../cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index e9e1e0883..872cbd1cf 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -297,7 +297,7 @@ # Q8 最近点查询 select time, v1... where device = ? and time = max(time) # Q9 倒序范围查询(只限制起止时间)select v1... from data where time > ? and time < ? and device in ? order by time desc # Q10 倒序带值过滤的范围查询 select v1... from data where time > ? and time < ? and v1 > ? and device in ? order by time desc -# Q11 分组聚合查询,倒序;目前仅支持iotdb、influxdb v1 +# Q11 分组聚合查询,倒序;目前仅支持iotdb、tdengine-3.0、influxdb v1 # OPERATION_PROPORTION=1:0:0:0:0:0:0:0:0:0:0:0 # 最长等待写时间,单位毫秒,即如果整个写操作在指定时间内没有返回,则终止此操作 @@ -393,8 +393,8 @@ # 是否align by device # 目前仅支持iotdb、influxdb v1、tdengine-3 -# 实测发现, iotdb对于align by device的使用有诸多限制, 推荐设置OPERATION_PROPORTION不超过范围1:1:1:0:0:0:0:0:0:1:0:0 -# td可设置1:1:1:1:1:1:1:1:1:1:1:0 +# iotdb支持OPERATION_PROPORTION=1:1:1:0:0:0:0:0:0:1:0:0 +# tdengine支持1:1:1:1:1:1:1:1:1:1:1:1 # ALIGN_BY_DEVICE=false ################ Workload:相关参数 ###################### diff --git a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java index 7801c64eb..42aade911 100644 --- a/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java +++ b/influxdb/src/main/java/cn/edu/tsinghua/iot/benchmark/influxdb/InfluxDB.java @@ -302,9 +302,8 @@ private Status addTailClausesAndExecuteQueryAndGetStatus(String sql) { if (config.getRESULT_ROW_LIMIT() >= 0) { sql += " limit " + config.getRESULT_ROW_LIMIT(); } - LOGGER.info("do query: {}", sql); if (!config.isIS_QUIET_MODE()) { - LOGGER.debug("{} query SQL: {}", Thread.currentThread().getName(), sql); + LOGGER.info("{} query SQL: {}", Thread.currentThread().getName(), sql); } QueryResult results = influxDbInstance.query(new Query(sql, influxDbName)); From 4c787dec672594996e132617a1f259fba991ef09 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 2 Jan 2024 18:26:35 +0800 Subject: [PATCH 19/19] tdengine q11 done --- .../java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index 819c707f3..85b4cd6f7 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -67,6 +67,7 @@ public class TDengine implements IDatabase { private static final String USE_DB = "use %s"; private static final String SUPER_TABLE_NAME = "device"; private static final String ORDER_BY_TIME_DESC = " order by time desc "; + private static final String ORDER_BY_WSTART_DESC = " order by _wstart desc "; private static final AtomicBoolean isInit = new AtomicBoolean(false); private final String CREATE_STABLE; @@ -407,7 +408,7 @@ public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { addWhereClause( sqlHeader, groupByQuery, null, getTableNameFilterForAlignByDevice(groupByQuery)); sql = addGroupByClause(sql, groupByQuery.getGranularity()); - sql += ORDER_BY_TIME_DESC; + sql += ORDER_BY_WSTART_DESC; return addTailClausesAndExecuteQueryAndGetStatus(sql); }