From 8625c658c719270e1d00c5015f02a34b8ef438a2 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Tue, 17 Oct 2023 13:50:12 +0800 Subject: [PATCH 1/8] save --- configuration/conf/config.properties | 12 ++++ .../tsinghua/iot/benchmark/conf/Config.java | 59 ++++++++++++------- .../iot/benchmark/conf/ConfigDescriptor.java | 27 ++++++++- .../iot/benchmark/schema/MetaUtil.java | 37 ++++++++++++ .../schemaImpl/GenerateMetaDataSchema.java | 2 +- .../schema/schemaImpl/RealMetaDataSchema.java | 2 +- .../iot/benchmark/source/CSVDataReader.java | 2 +- .../iot/benchmark/source/CopyDataReader.java | 2 +- .../workload/GenerateQueryWorkLoad.java | 6 +- .../workload/SingletonWorkDataWorkLoad.java | 2 +- .../workload/SyntheticDataWorkLoad.java | 5 +- .../iot/benchmark/iotdb012/IoTDB.java | 3 +- .../iot/benchmark/iotdb013/IoTDB.java | 3 +- .../iot/benchmark/iotdb100/IoTDB.java | 3 +- .../iot/benchmark/iotdb110/IoTDB.java | 3 +- .../timescaledbCluster/TimescaleDB.java | 8 +-- .../benchmark/timescaledb/TimescaleDB.java | 8 +-- 17 files changed, 142 insertions(+), 42 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 877e9c3ae..3be68384c 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -143,6 +143,18 @@ # 定义设备标签值，所有设备相同，使用逗号分开，目前支持iotdb-0.12, iotdb-0.13, influxdb-2.x, timescaledb, tdengine # DEVICE_TAGS= +# 标签名数量 +# TAG_NUMBER_=3 + +# 标签名前缀 +# TAG_KEY_PREFIX=tag_ + +# 标签值前缀 +# TAG_VALUE_PREFIX=value_ + +# 每个标签值的取值种数，使用逗号分开，设置个数应与标签名数量相等。例如TAG_NUMBER_=2，TAG_VALUE_RANGE=50,10，说明第一个标签名可能有50种值，第二个标签名可能有10种值。 +# TAG_VALUE_RANGE=50,50,10 + ############## 被测系统为IoTDB时扩展参数 ################## # 是否使用thrift压缩，需要在iotdb的配置文件iotdb-common.properties中设置rpc_thrift_compression_enable=true # ENABLE_THRIFT_COMPRESSION=false diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index d2d0a43f9..35afe5cdf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -182,9 +182,11 @@ public class Config { /** name prefix of sensor */ private String SENSOR_NAME_PREFIX = "s_"; /** name prefix of tag */ - private String TAG_NAME_PREFIX = "tag_"; - /** The tags of device. Each device is same. */ - private Map DEVICE_TAGS = new LinkedHashMap<>(); + private int TAG_NUMBER = 0; + + private String TAG_KEY_PREFIX = "tag_"; + private String TAG_VALUE_PREFIX = "value_"; + private List TAG_VALUE_CARDINALITY = new ArrayList<>(); // 设备、传感器、客户端：生成数据的规律 /** 线性 默认 9个 */ @@ -844,27 +846,41 @@ public void setCLIENT_NUMBER(int CLIENT_NUMBER) { this.CLIENT_NUMBER = CLIENT_NUMBER; } - public String getTAG_NAME_PREFIX() { - return TAG_NAME_PREFIX; + public Map getDEVICE_TAGS() { + // TODO: remove this + return null; } - public void setTAG_NAME_PREFIX(String TAG_NAME_PREFIX) { - this.TAG_NAME_PREFIX = TAG_NAME_PREFIX; + public int getTAG_NUMBER() { + return TAG_NUMBER; } - public Map getDEVICE_TAGS() { - return new LinkedHashMap<>(DEVICE_TAGS); + public void setTAG_NUMBER(int TAG_NUMBER) { + this.TAG_NUMBER = TAG_NUMBER; } - public void setDEVICE_TAGS(String DEVICE_TAGS) { - if (DEVICE_TAGS.length() != 0) { - this.DEVICE_TAGS = new LinkedHashMap<>(); - int tagIndex = 0; - for (String tagValue : DEVICE_TAGS.split(",")) { - this.DEVICE_TAGS.put(TAG_NAME_PREFIX + tagIndex, tagValue); - tagIndex++; - } - } + public String getTAG_KEY_PREFIX() { + return TAG_KEY_PREFIX; + } + + public void setTAG_KEY_PREFIX(String TAG_KEY_PREFIX) { + this.TAG_KEY_PREFIX = TAG_KEY_PREFIX; + } + + public String getTAG_VALUE_PREFIX() { + return TAG_VALUE_PREFIX; + } + + public void setTAG_VALUE_PREFIX(String TAG_VALUE_PREFIX) { + this.TAG_VALUE_PREFIX = TAG_VALUE_PREFIX; + } + + public List getTAG_VALUE_CARDINALITY() { + return TAG_VALUE_CARDINALITY; + } + + public void setTAG_VALUE_CARDINALITY(List TAG_VALUE_CARDINALITY) { + this.TAG_VALUE_CARDINALITY = TAG_VALUE_CARDINALITY; } public String getGROUP_NAME_PREFIX() { @@ -1785,8 +1801,11 @@ public ConfigProperties getAllConfigProperties() { configProperties.addProperty("Extern Param", "GROUP_NAME_PREFIX", this.GROUP_NAME_PREFIX); configProperties.addProperty("Extern Param", "DEVICE_NAME_PREFIX", this.DEVICE_NAME_PREFIX); configProperties.addProperty("Extern Param", "SENSOR_NAME_PREFIX", this.SENSOR_NAME_PREFIX); - configProperties.addProperty("Extern Param", "TAG_NAME_PREFIX", this.TAG_NAME_PREFIX); - configProperties.addProperty("Extern Param", "DEVICE_TAGS", this.DEVICE_TAGS); + configProperties.addProperty("Extern Param", "TAG_NUMBER", this.TAG_NUMBER); + configProperties.addProperty("Extern Param", "TAG_KEY_PREFIX", this.TAG_KEY_PREFIX); + configProperties.addProperty("Extern Param", "TAG_VALUE_PREFIX", this.TAG_VALUE_PREFIX); + configProperties.addProperty( + "Extern Param", "TAG_VALUE_CARDINALITY", this.TAG_VALUE_CARDINALITY); configProperties.addProperty( "Extern Param", "ENABLE_THRIFT_COMPRESSION", this.ENABLE_THRIFT_COMPRESSION); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index e9bfb523c..0962c556f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -33,6 +33,7 @@ import java.lang.reflect.Field; import java.util.Arrays; import java.util.Properties; +import java.util.stream.Collectors; import static cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode.INSERT_USE_SESSION_RECORDS; @@ -248,8 +249,21 @@ private void loadProps() { properties.getProperty("DEVICE_NAME_PREFIX", config.getDEVICE_NAME_PREFIX())); config.setSENSOR_NAME_PREFIX( properties.getProperty("SENSOR_NAME_PREFIX", config.getSENSOR_NAME_PREFIX())); - config.setDEVICE_TAGS(properties.getProperty("DEVICE_TAGS", "")); - + config.setTAG_NUMBER( + Integer.parseInt(properties.getProperty("TAG_NUMBER", config.getTAG_NUMBER() + ""))); + config.setTAG_KEY_PREFIX( + properties.getProperty("TAG_KEY_PREFIX", config.getTAG_KEY_PREFIX())); + config.setTAG_VALUE_PREFIX( + properties.getProperty("TAG_VALUE_PREFIX", config.getTAG_VALUE_PREFIX())); + config.setTAG_VALUE_CARDINALITY( + Arrays.stream( + properties + .getProperty( + "TAG_VALUE_CARDINALITY", config.getTAG_VALUE_CARDINALITY() + "") + .split(",")) + .mapToInt(Integer::parseInt) + .boxed() + .collect(Collectors.toList())); config.setBENCHMARK_CLUSTER( Boolean.parseBoolean( properties.getProperty("BENCHMARK_CLUSTER", config.isBENCHMARK_CLUSTER() + ""))); @@ -596,6 +610,15 @@ private boolean checkConfig() { "The combination of \"DEVICE_NUM_PER_WRITE > 1\" and \"IS_SENSOR_TS_ALIGNMENT == false\" is not supported"); result = false; } + // check tag + if (config.getTAG_NUMBER() != config.getTAG_VALUE_CARDINALITY().size()) { + LOGGER.error( + "TAG_NUMBER must be equal to TAG_VALUE_CARDINALITY's size. Currently, " + + "TAG_NUMBER = {}, TAG_VALUE_CARDINALITY = {}.", + config.getTAG_NUMBER(), + config.getTAG_VALUE_CARDINALITY().size()); + result = false; + } return result; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 60d6fdd18..054756409 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -5,11 +5,30 @@ import cn.edu.tsinghua.iot.benchmark.conf.Constants; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class MetaUtil { private static Config config = ConfigDescriptor.getInstance().getConfig(); + private static final String TAG_KEY_PREFIX = config.getTAG_KEY_PREFIX(); + private static final String TAG_VALUE_PREFIX = config.getTAG_VALUE_PREFIX(); + private static final int TAG_NUMBER = config.getTAG_NUMBER(); + private static final List TAG_VALUE_CARDINALITY = config.getTAG_VALUE_CARDINALITY(); + private static final int[] LEVEL_CARDINALITY = new int[TAG_VALUE_CARDINALITY.size() + 1]; + + static { + int idx = TAG_VALUE_CARDINALITY.size(); + int sum = 1; + LEVEL_CARDINALITY[idx--] = 1; + for (; idx >= 0; idx--) { + sum *= TAG_VALUE_CARDINALITY.get(idx); + LEVEL_CARDINALITY[idx] = sum; + } + } + private static List> CLIENT_FILES; /** Used under cluster mode of benchmark */ @@ -72,4 +91,22 @@ public static List> getClientFiles() { public static void setClientFiles(List> clientFiles) { CLIENT_FILES = clientFiles; } + + public static Map getTag(String deviceName) { + if (TAG_NUMBER == 0) { + return Collections.emptyMap(); + } + int id = deviceName.hashCode(); + Map res = new HashMap<>(); + for (int i = 0; i < LEVEL_CARDINALITY.length; i++) { + id = id % LEVEL_CARDINALITY[i]; + int tagValueId = id / LEVEL_CARDINALITY[i + 1]; + res.put(TAG_KEY_PREFIX + i, TAG_VALUE_PREFIX + tagValueId); + } + return res; + } + + public static Map getTag(int deviceId) { + return getTag(getDeviceName(deviceId)); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index c5658603b..55c04664d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -52,7 +52,7 @@ public boolean createMetaDataSchema() { (clientId < leftClientDeviceNum) ? eachClientDeviceNum + 1 : eachClientDeviceNum; List deviceSchemaList = new ArrayList<>(); for (int d = 0; d < deviceNumber; d++) { - DeviceSchema deviceSchema = new DeviceSchema(deviceId, sensors, config.getDEVICE_TAGS()); + DeviceSchema deviceSchema = new DeviceSchema(deviceId, sensors, MetaUtil.getTag(deviceId)); NAME_DATA_SCHEMA.put(deviceSchema.getDevice(), deviceSchema); GROUPS.add(deviceSchema.getGroup()); deviceSchemaList.add(deviceSchema); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/RealMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/RealMetaDataSchema.java index ca28d3100..a9c9389d0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/RealMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/RealMetaDataSchema.java @@ -73,7 +73,7 @@ protected boolean createMetaDataSchema() { MetaUtil.getGroupIdFromDeviceName(deviceName), deviceName, sensors, - config.getDEVICE_TAGS()); + MetaUtil.getTag(deviceName)); NAME_DATA_SCHEMA.put(deviceName, deviceSchema); GROUPS.add(deviceSchema.getGroup()); deviceSchemaList.add(deviceSchema); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java index 449fd249d..ca45d3bc2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java @@ -80,7 +80,7 @@ public IBatch nextBatch() { MetaUtil.getGroupIdFromDeviceName(deviceName), deviceName, sensors, - config.getDEVICE_TAGS()); + MetaUtil.getTag(deviceName)); firstLine = false; continue; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CopyDataReader.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CopyDataReader.java index 0c7b4c5ef..54dcc9e18 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CopyDataReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CopyDataReader.java @@ -90,7 +90,7 @@ public CopyDataReader(List files) { MetaUtil.getGroupIdFromDeviceName(deviceName), deviceName, sensors, - config.getDEVICE_TAGS()); + MetaUtil.getTag(deviceName)); firstLine = false; continue; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java index 28e8e92ec..ce9396b19 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java @@ -25,6 +25,7 @@ import cn.edu.tsinghua.iot.benchmark.entity.Sensor; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBSwitch; import cn.edu.tsinghua.iot.benchmark.utils.TimeUtils; @@ -165,7 +166,7 @@ public DeviceQuery getDeviceQuery() { return null; } DeviceSchema deviceSchema = - new DeviceSchema(deviceId, config.getSENSORS(), config.getDEVICE_TAGS()); + new DeviceSchema(deviceId, config.getSENSORS(), MetaUtil.getTag(deviceId)); return new DeviceQuery(deviceSchema); } @@ -234,7 +235,8 @@ private List getQueryDeviceSchemaList(boolean typeAllow) throws Wo if (querySensors.size() != config.getQUERY_SENSOR_NUM()) { continue; } - DeviceSchema deviceSchema = new DeviceSchema(deviceId, querySensors, config.getDEVICE_TAGS()); + DeviceSchema deviceSchema = + new DeviceSchema(deviceId, querySensors, MetaUtil.getTag(deviceId)); queryDevices.add(deviceSchema); } if (queryDevices.size() != config.getQUERY_DEVICE_NUM()) { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java index 700dc415f..c7919514a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java @@ -89,7 +89,7 @@ public IBatch getOneBatch() throws WorkloadException { new DeviceSchema( MetaUtil.getDeviceId((int) curLoop % config.getDEVICE_NUMBER()), sensors, - config.getDEVICE_TAGS()); + MetaUtil.getTag((int) curLoop % config.getDEVICE_NUMBER())); // create data of batch List records = new ArrayList<>(); for (long batchOffset = 0; batchOffset < recordsNumPerDevice; batchOffset++) { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java index e7d321d11..d717a0817 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java @@ -25,6 +25,7 @@ import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import java.util.*; @@ -47,7 +48,9 @@ public SyntheticDataWorkLoad(List deviceSchemas) { for (Sensor sensor : schema.getSensors()) { DeviceSchema deviceSchema = new DeviceSchema( - schema.getDeviceId(), Collections.singletonList(sensor), config.getDEVICE_TAGS()); + schema.getDeviceId(), + Collections.singletonList(sensor), + MetaUtil.getTag(schema.getDeviceId())); maxTimestampIndexMap.put(deviceSchema, 0L); } } diff --git a/iotdb-0.12/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java b/iotdb-0.12/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java index 430471830..80421ebfb 100644 --- a/iotdb-0.12/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java +++ b/iotdb-0.12/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java @@ -36,6 +36,7 @@ import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -599,7 +600,7 @@ private String addGroupByClause(String prefix, long start, long end, long granul protected String getDevicePath(DeviceSchema deviceSchema) { StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { + for (Map.Entry pair : MetaUtil.getTag(deviceSchema.getDevice()).entrySet()) { name.append(".").append(pair.getValue()); } name.append(".").append(deviceSchema.getDevice()); diff --git a/iotdb-0.13/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java b/iotdb-0.13/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java index 500724d5b..c8d5f1996 100644 --- a/iotdb-0.13/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java +++ b/iotdb-0.13/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java @@ -39,6 +39,7 @@ import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -652,7 +653,7 @@ private String addGroupByClause(String prefix, long start, long end, long granul protected String getDevicePath(DeviceSchema deviceSchema) { StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { + for (Map.Entry pair : MetaUtil.getTag(deviceSchema.getDevice()).entrySet()) { name.append(".").append(pair.getValue()); } name.append(".").append(deviceSchema.getDevice()); diff --git a/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java b/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java index c4c465708..02b5488c4 100644 --- a/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java +++ b/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java @@ -39,6 +39,7 @@ import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -652,7 +653,7 @@ private String addGroupByClause(String prefix, long start, long end, long granul protected String getDevicePath(DeviceSchema deviceSchema) { StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { + for (Map.Entry pair : MetaUtil.getTag(deviceSchema.getDevice()).entrySet()) { name.append(".").append(pair.getValue()); } name.append(".").append(deviceSchema.getDevice()); diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java index f31dfd584..b5a07a738 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java @@ -39,6 +39,7 @@ import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -676,7 +677,7 @@ private String addGroupByClause(String prefix, long start, long end, long granul protected String getDevicePath(DeviceSchema deviceSchema) { StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { + for (Map.Entry pair : MetaUtil.getTag(deviceSchema.getDevice()).entrySet()) { name.append(".").append(pair.getValue()); } name.append(".").append(deviceSchema.getDevice()); diff --git a/timescaledb-cluster/src/main/java/cn/edu/tsinghua/iot/benchmark/timescaledbCluster/TimescaleDB.java b/timescaledb-cluster/src/main/java/cn/edu/tsinghua/iot/benchmark/timescaledbCluster/TimescaleDB.java index 52798403e..d0f5244d6 100644 --- a/timescaledb-cluster/src/main/java/cn/edu/tsinghua/iot/benchmark/timescaledbCluster/TimescaleDB.java +++ b/timescaledb-cluster/src/main/java/cn/edu/tsinghua/iot/benchmark/timescaledbCluster/TimescaleDB.java @@ -607,8 +607,8 @@ private String getCreateTableSql(String tableName, List sensors) { StringBuilder sqlBuilder = new StringBuilder("CREATE TABLE ").append(tableName).append(" ("); sqlBuilder.append( "time BIGINT NOT NULL, location TEXT NOT NULL, sGroup TEXT NOT NULL, device TEXT NOT NULL"); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { - sqlBuilder.append(", ").append(pair.getKey()).append(" TEXT NOT NULL"); + for (int i = 0; i < config.getTAG_NUMBER(); i++) { + sqlBuilder.append(", ").append(config.getTAG_KEY_PREFIX()).append(i).append(" TEXT NOT NULL"); } for (int i = 0; i < sensors.size(); i++) { sqlBuilder @@ -619,8 +619,8 @@ private String getCreateTableSql(String tableName, List sensors) { .append(" NULL "); } sqlBuilder.append(",UNIQUE (time, location, sGroup, device"); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { - sqlBuilder.append(", ").append(pair.getKey()); + for (int i = 0; i < config.getTAG_NUMBER(); i++) { + sqlBuilder.append(", ").append(config.getTAG_KEY_PREFIX()).append(i); } sqlBuilder.append("));"); return sqlBuilder.toString(); diff --git a/timescaledb/src/main/java/cn/edu/tsinghua/iot/benchmark/timescaledb/TimescaleDB.java b/timescaledb/src/main/java/cn/edu/tsinghua/iot/benchmark/timescaledb/TimescaleDB.java index 22444a19d..4450334d1 100644 --- a/timescaledb/src/main/java/cn/edu/tsinghua/iot/benchmark/timescaledb/TimescaleDB.java +++ b/timescaledb/src/main/java/cn/edu/tsinghua/iot/benchmark/timescaledb/TimescaleDB.java @@ -603,8 +603,8 @@ private static void addOrderByClause(StringBuilder builder) { private String getCreateTableSql(String tableName, List sensors) { StringBuilder sqlBuilder = new StringBuilder("CREATE TABLE ").append(tableName).append(" ("); sqlBuilder.append("time BIGINT NOT NULL, sGroup TEXT NOT NULL, device TEXT NOT NULL"); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { - sqlBuilder.append(", ").append(pair.getKey()).append(" TEXT NOT NULL"); + for (int i = 0; i < config.getTAG_NUMBER(); i++) { + sqlBuilder.append(", ").append(config.getTAG_KEY_PREFIX()).append(i).append(" TEXT NOT NULL"); } for (int i = 0; i < sensors.size(); i++) { sqlBuilder @@ -615,8 +615,8 @@ private String getCreateTableSql(String tableName, List sensors) { .append(" NULL "); } sqlBuilder.append(",UNIQUE (time, sGroup, device"); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { - sqlBuilder.append(", ").append(pair.getKey()); + for (int i = 0; i < config.getTAG_NUMBER(); i++) { + sqlBuilder.append(", ").append(config.getTAG_KEY_PREFIX()).append(i); } sqlBuilder.append("));"); return sqlBuilder.toString(); From d17aa02ee15966c6d1f14eedb0a1ec54ae9cd8bb Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Tue, 17 Oct 2023 14:46:34 +0800 Subject: [PATCH 2/8] done --- configuration/conf/config.properties | 6 ++--- .../tsinghua/iot/benchmark/conf/Config.java | 5 ----- .../iot/benchmark/conf/ConfigDescriptor.java | 1 + .../iot/benchmark/schema/MetaUtil.java | 2 +- .../iot/benchmark/tdengine3/TDengine.java | 22 +++++++++---------- .../iot/benchmark/tdengine/TDengine.java | 21 +++++++++--------- 6 files changed, 27 insertions(+), 30 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 3be68384c..517dc1699 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -144,7 +144,7 @@ # DEVICE_TAGS= # 标签名数量 -# TAG_NUMBER_=3 +# TAG_NUMBER=0 # 标签名前缀 # TAG_KEY_PREFIX=tag_ @@ -152,8 +152,8 @@ # 标签值前缀 # TAG_VALUE_PREFIX=value_ -# 每个标签值的取值种数，使用逗号分开，设置个数应与标签名数量相等。例如TAG_NUMBER_=2，TAG_VALUE_RANGE=50,10，说明第一个标签名可能有50种值，第二个标签名可能有10种值。 -# TAG_VALUE_RANGE=50,50,10 +# 每个标签值的取值种数，使用逗号分开，设置个数应与标签名数量相等。例如TAG_NUMBER=2，TAG_VALUE_CARDINALITY=50,10，说明第一个标签名可能有50种值，第二个标签名可能有10种值 +# TAG_VALUE_CARDINALITY= ############## 被测系统为IoTDB时扩展参数 ################## # 是否使用thrift压缩，需要在iotdb的配置文件iotdb-common.properties中设置rpc_thrift_compression_enable=true diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index 35afe5cdf..c9b4832b4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -846,11 +846,6 @@ public void setCLIENT_NUMBER(int CLIENT_NUMBER) { this.CLIENT_NUMBER = CLIENT_NUMBER; } - public Map getDEVICE_TAGS() { - // TODO: remove this - return null; - } - public int getTAG_NUMBER() { return TAG_NUMBER; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 0962c556f..9fdc0183e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -261,6 +261,7 @@ private void loadProps() { .getProperty( "TAG_VALUE_CARDINALITY", config.getTAG_VALUE_CARDINALITY() + "") .split(",")) + .filter(s -> !s.isEmpty()) .mapToInt(Integer::parseInt) .boxed() .collect(Collectors.toList())); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 054756409..ed6270522 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -98,7 +98,7 @@ public static Map getTag(String deviceName) { } int id = deviceName.hashCode(); Map res = new HashMap<>(); - for (int i = 0; i < LEVEL_CARDINALITY.length; i++) { + for (int i = 0; i < LEVEL_CARDINALITY.length - 1; i++) { id = id % LEVEL_CARDINALITY[i]; int tagValueId = id / LEVEL_CARDINALITY[i + 1]; res.put(TAG_KEY_PREFIX + i, TAG_VALUE_PREFIX + tagValueId); diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index c64b31aa4..d0afd3756 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -26,6 +26,7 @@ import cn.edu.tsinghua.iot.benchmark.entity.Sensor; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -47,8 +48,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; @@ -82,9 +83,9 @@ public TDengine(DBConfig dbConfig) { "create stable if not exists %s (time timestamp, %s) tags(device binary(20)"); StringBuilder createTable = new StringBuilder("create table if not exists %s using %s tags('%s'"); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { - createStable.append(", ").append(pair.getKey()).append(" binary(20)"); - createTable.append(", '").append(pair.getValue()).append("'"); + for (int i = 0; i < config.getTAG_NUMBER(); i++) { + createStable.append(", ").append(config.getTAG_KEY_PREFIX()).append(i).append(" binary(20)"); + createTable.append(", '").append("%s").append("'"); } createStable.append(")"); createTable.append(")"); @@ -157,13 +158,12 @@ public Double registerSchema(List schemaList) throws TsdbException statement.execute(String.format(USE_DB, testDatabaseName)); for (DeviceSchema deviceSchema : schemaList) { synchronized (TDengine.class) { - String createTable = - String.format( - CREATE_TABLE, - deviceSchema.getDevice(), - SUPER_TABLE_NAME, - deviceSchema.getDevice()); - statement.execute(createTable); + List params = new ArrayList<>(); + params.add(deviceSchema.getDevice()); + params.add(SUPER_TABLE_NAME); + params.add(deviceSchema.getDevice()); + params.addAll(MetaUtil.getTag(deviceSchema.getDeviceId()).values()); + statement.execute(String.format(CREATE_TABLE, params.toArray())); } } } catch (SQLException | BrokenBarrierException | InterruptedException e) { diff --git a/tdengine/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine/TDengine.java b/tdengine/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine/TDengine.java index e9d4ecd05..549fb3a5f 100644 --- a/tdengine/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine/TDengine.java +++ b/tdengine/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine/TDengine.java @@ -26,6 +26,7 @@ import cn.edu.tsinghua.iot.benchmark.entity.Sensor; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -47,8 +48,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; -import java.util.Map; public class TDengine implements IDatabase { private static final Config config = ConfigDescriptor.getInstance().getConfig(); @@ -77,9 +78,9 @@ public TDengine(DBConfig dbConfig) { "create stable if not exists %s (time timestamp, %s) tags(device binary(20)"); StringBuilder createTable = new StringBuilder("create table if not exists %s using %s tags('%s'"); - for (Map.Entry pair : config.getDEVICE_TAGS().entrySet()) { - createStable.append(", ").append(pair.getKey()).append(" binary(20)"); - createTable.append(", '").append(pair.getValue()).append("'"); + for (int i = 0; i < config.getTAG_NUMBER(); i++) { + createStable.append(", ").append(config.getTAG_KEY_PREFIX()).append(i).append(" binary(20)"); + createTable.append(", '").append("%s").append("'"); } createStable.append(")"); createTable.append(")"); @@ -166,12 +167,12 @@ public Double registerSchema(List schemaList) throws TsdbException // create tables statement.execute(String.format(USE_DB, testDatabaseName)); for (DeviceSchema deviceSchema : schemaList) { - statement.execute( - String.format( - CREATE_TABLE, - deviceSchema.getDevice(), - SUPER_TABLE_NAME, - deviceSchema.getDevice())); + List params = new ArrayList<>(); + params.add(deviceSchema.getDevice()); + params.add(SUPER_TABLE_NAME); + params.add(deviceSchema.getDevice()); + params.addAll(MetaUtil.getTag(deviceSchema.getDeviceId()).values()); + statement.execute(String.format(CREATE_TABLE, params.toArray())); } } catch (SQLException e) { // ignore if already has the time series From 61a0b67d64c10e082e3d5a71ff1b3f11380c5c30 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Thu, 19 Oct 2023 17:53:16 +0800 Subject: [PATCH 3/8] done --- configuration/conf/config.properties | 6 ------ .../iot/benchmark/conf/ConfigDescriptor.java | 21 +++++++++++-------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 517dc1699..f657b7ea4 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -137,12 +137,6 @@ # 传感器名称前缀 # SENSOR_NAME_PREFIX=s_ -# 标签名的前缀 -# TAG_NAME_PREFIX=tag_ - -# 定义设备标签值，所有设备相同，使用逗号分开，目前支持iotdb-0.12, iotdb-0.13, influxdb-2.x, timescaledb, tdengine -# DEVICE_TAGS= - # 标签名数量 # TAG_NUMBER=0 diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 9fdc0183e..8fcbef093 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -256,15 +256,18 @@ private void loadProps() { config.setTAG_VALUE_PREFIX( properties.getProperty("TAG_VALUE_PREFIX", config.getTAG_VALUE_PREFIX())); config.setTAG_VALUE_CARDINALITY( - Arrays.stream( - properties - .getProperty( - "TAG_VALUE_CARDINALITY", config.getTAG_VALUE_CARDINALITY() + "") - .split(",")) - .filter(s -> !s.isEmpty()) - .mapToInt(Integer::parseInt) - .boxed() - .collect(Collectors.toList())); + Arrays.stream( + properties + .getProperty( + "TAG_VALUE_CARDINALITY", + config.getTAG_VALUE_CARDINALITY().stream() + .map(Object::toString) + .collect(Collectors.joining(","))) + .split(",")) + .filter(s -> !s.isEmpty()) + .mapToInt(Integer::parseInt) + .boxed() + .collect(Collectors.toList())); config.setBENCHMARK_CLUSTER( Boolean.parseBoolean( properties.getProperty("BENCHMARK_CLUSTER", config.isBENCHMARK_CLUSTER() + ""))); From 132ec8335ea7427a6e4fe03be1dfb1cc0f593eb6 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Thu, 19 Oct 2023 17:53:36 +0800 Subject: [PATCH 4/8] spotless --- .../iot/benchmark/conf/ConfigDescriptor.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 8fcbef093..1e1a99807 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -256,18 +256,18 @@ private void loadProps() { config.setTAG_VALUE_PREFIX( properties.getProperty("TAG_VALUE_PREFIX", config.getTAG_VALUE_PREFIX())); config.setTAG_VALUE_CARDINALITY( - Arrays.stream( - properties - .getProperty( - "TAG_VALUE_CARDINALITY", - config.getTAG_VALUE_CARDINALITY().stream() - .map(Object::toString) - .collect(Collectors.joining(","))) - .split(",")) - .filter(s -> !s.isEmpty()) - .mapToInt(Integer::parseInt) - .boxed() - .collect(Collectors.toList())); + Arrays.stream( + properties + .getProperty( + "TAG_VALUE_CARDINALITY", + config.getTAG_VALUE_CARDINALITY().stream() + .map(Object::toString) + .collect(Collectors.joining(","))) + .split(",")) + .filter(s -> !s.isEmpty()) + .mapToInt(Integer::parseInt) + .boxed() + .collect(Collectors.toList())); config.setBENCHMARK_CLUSTER( Boolean.parseBoolean( properties.getProperty("BENCHMARK_CLUSTER", config.isBENCHMARK_CLUSTER() + ""))); From 0c85eae17f1b0b4cccd60fc625ebd262e47dce3f Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Mon, 23 Oct 2023 18:16:07 +0800 Subject: [PATCH 5/8] Save --- .../cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index ed6270522..2e9c16511 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -17,11 +17,11 @@ public class MetaUtil { private static final String TAG_VALUE_PREFIX = config.getTAG_VALUE_PREFIX(); private static final int TAG_NUMBER = config.getTAG_NUMBER(); private static final List TAG_VALUE_CARDINALITY = config.getTAG_VALUE_CARDINALITY(); - private static final int[] LEVEL_CARDINALITY = new int[TAG_VALUE_CARDINALITY.size() + 1]; + private static final long[] LEVEL_CARDINALITY = new long[TAG_VALUE_CARDINALITY.size() + 1]; static { int idx = TAG_VALUE_CARDINALITY.size(); - int sum = 1; + long sum = 1; LEVEL_CARDINALITY[idx--] = 1; for (; idx >= 0; idx--) { sum *= TAG_VALUE_CARDINALITY.get(idx); @@ -96,11 +96,11 @@ public static Map getTag(String deviceName) { if (TAG_NUMBER == 0) { return Collections.emptyMap(); } - int id = deviceName.hashCode(); + long id = Math.abs(deviceName.hashCode()); Map res = new HashMap<>(); for (int i = 0; i < LEVEL_CARDINALITY.length - 1; i++) { id = id % LEVEL_CARDINALITY[i]; - int tagValueId = id / LEVEL_CARDINALITY[i + 1]; + long tagValueId = id / LEVEL_CARDINALITY[i + 1]; res.put(TAG_KEY_PREFIX + i, TAG_VALUE_PREFIX + tagValueId); } return res; From 55e62d715ecb7c7336525c850c7d4f85d27d8171 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Wed, 25 Oct 2023 09:43:24 +0800 Subject: [PATCH 6/8] optimize --- .../java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java | 3 +-- .../java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java | 3 +-- .../java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java | 3 +-- .../java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/iotdb-0.12/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java b/iotdb-0.12/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java index 80421ebfb..e3d855d09 100644 --- a/iotdb-0.12/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java +++ b/iotdb-0.12/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb012/IoTDB.java @@ -36,7 +36,6 @@ import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; import cn.edu.tsinghua.iot.benchmark.measurement.Status; -import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -600,7 +599,7 @@ private String addGroupByClause(String prefix, long start, long end, long granul protected String getDevicePath(DeviceSchema deviceSchema) { StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : MetaUtil.getTag(deviceSchema.getDevice()).entrySet()) { + for (Map.Entry pair : deviceSchema.getTags().entrySet()) { name.append(".").append(pair.getValue()); } name.append(".").append(deviceSchema.getDevice()); diff --git a/iotdb-0.13/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java b/iotdb-0.13/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java index c8d5f1996..ba4064fb5 100644 --- a/iotdb-0.13/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java +++ b/iotdb-0.13/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb013/IoTDB.java @@ -39,7 +39,6 @@ import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; import cn.edu.tsinghua.iot.benchmark.measurement.Status; -import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -653,7 +652,7 @@ private String addGroupByClause(String prefix, long start, long end, long granul protected String getDevicePath(DeviceSchema deviceSchema) { StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : MetaUtil.getTag(deviceSchema.getDevice()).entrySet()) { + for (Map.Entry pair : deviceSchema.getTags().entrySet()) { name.append(".").append(pair.getValue()); } name.append(".").append(deviceSchema.getDevice()); diff --git a/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java b/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java index 02b5488c4..836c92ea9 100644 --- a/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java +++ b/iotdb-1.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb100/IoTDB.java @@ -39,7 +39,6 @@ import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; import cn.edu.tsinghua.iot.benchmark.measurement.Status; -import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -653,7 +652,7 @@ private String addGroupByClause(String prefix, long start, long end, long granul protected String getDevicePath(DeviceSchema deviceSchema) { StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : MetaUtil.getTag(deviceSchema.getDevice()).entrySet()) { + for (Map.Entry pair : deviceSchema.getTags().entrySet()) { name.append(".").append(pair.getValue()); } name.append(".").append(deviceSchema.getDevice()); diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java index b5a07a738..e01a439bf 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java @@ -39,7 +39,6 @@ import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.DBConnectException; import cn.edu.tsinghua.iot.benchmark.measurement.Status; -import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.IDatabase; @@ -677,7 +676,7 @@ private String addGroupByClause(String prefix, long start, long end, long granul protected String getDevicePath(DeviceSchema deviceSchema) { StringBuilder name = new StringBuilder(ROOT_SERIES_NAME); name.append(".").append(deviceSchema.getGroup()); - for (Map.Entry pair : MetaUtil.getTag(deviceSchema.getDevice()).entrySet()) { + for (Map.Entry pair : deviceSchema.getTags().entrySet()) { name.append(".").append(pair.getValue()); } name.append(".").append(deviceSchema.getDevice()); From 7f6f8ea5edf4f7a65077edb7cd35eedfebcf487c Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Tue, 14 Nov 2023 17:40:59 +0800 Subject: [PATCH 7/8] fix review --- configuration/conf/config.properties | 2 +- .../iot/benchmark/schema/MetaUtil.java | 32 +++++++++++++------ .../schemaImpl/GenerateMetaDataSchema.java | 2 +- .../schema/schemaImpl/RealMetaDataSchema.java | 2 +- .../iot/benchmark/source/CSVDataReader.java | 2 +- .../iot/benchmark/source/CopyDataReader.java | 2 +- .../workload/GenerateQueryWorkLoad.java | 4 +-- .../workload/SingletonWorkDataWorkLoad.java | 2 +- .../workload/SyntheticDataWorkLoad.java | 2 +- .../iot/benchmark/tdengine3/TDengine.java | 2 +- .../iot/benchmark/tdengine/TDengine.java | 2 +- 11 files changed, 34 insertions(+), 20 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index f657b7ea4..964392b66 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -146,7 +146,7 @@ # 标签值前缀 # TAG_VALUE_PREFIX=value_ -# 每个标签值的取值种数，使用逗号分开，设置个数应与标签名数量相等。例如TAG_NUMBER=2，TAG_VALUE_CARDINALITY=50,10，说明第一个标签名可能有50种值，第二个标签名可能有10种值 +# 每个标签值的取值种数，使用逗号分开，设置个数应与标签名数量相等。例如TAG_NUMBER=2，TAG_VALUE_CARDINALITY=50,10，说明第一个标签名可能有50种值，第二个标签名可能有10种值。d_0的中间路径将为value_0.value_0.d_0。 # TAG_VALUE_CARDINALITY= ############## 被测系统为IoTDB时扩展参数 ################## diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 2e9c16511..0d640f62e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -5,6 +5,7 @@ import cn.edu.tsinghua.iot.benchmark.conf.Constants; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -17,15 +18,16 @@ public class MetaUtil { private static final String TAG_VALUE_PREFIX = config.getTAG_VALUE_PREFIX(); private static final int TAG_NUMBER = config.getTAG_NUMBER(); private static final List TAG_VALUE_CARDINALITY = config.getTAG_VALUE_CARDINALITY(); - private static final long[] LEVEL_CARDINALITY = new long[TAG_VALUE_CARDINALITY.size() + 1]; + private static final List LEVEL_CARDINALITY = + Arrays.asList(new Long[TAG_VALUE_CARDINALITY.size() + 1]); static { int idx = TAG_VALUE_CARDINALITY.size(); long sum = 1; - LEVEL_CARDINALITY[idx--] = 1; + LEVEL_CARDINALITY.set(idx--, 1L); for (; idx >= 0; idx--) { sum *= TAG_VALUE_CARDINALITY.get(idx); - LEVEL_CARDINALITY[idx] = sum; + LEVEL_CARDINALITY.set(idx, sum); } } @@ -92,21 +94,33 @@ public static void setClientFiles(List> clientFiles) { CLIENT_FILES = clientFiles; } - public static Map getTag(String deviceName) { + /** + * Get tags pair by deviceName. + * + * @param deviceName deviceName + * @return tags pair + */ + public static Map getTags(String deviceName) { if (TAG_NUMBER == 0) { return Collections.emptyMap(); } long id = Math.abs(deviceName.hashCode()); Map res = new HashMap<>(); - for (int i = 0; i < LEVEL_CARDINALITY.length - 1; i++) { - id = id % LEVEL_CARDINALITY[i]; - long tagValueId = id / LEVEL_CARDINALITY[i + 1]; + for (int i = 0; i < LEVEL_CARDINALITY.size() - 1; i++) { + id = id % LEVEL_CARDINALITY.get(i); + long tagValueId = id / LEVEL_CARDINALITY.get(i + 1); res.put(TAG_KEY_PREFIX + i, TAG_VALUE_PREFIX + tagValueId); } return res; } - public static Map getTag(int deviceId) { - return getTag(getDeviceName(deviceId)); + /** + * Get tags pair by deviceId. + * + * @param deviceId deviceId + * @return tags pair + */ + public static Map getTags(int deviceId) { + return getTags(getDeviceName(deviceId)); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index 55c04664d..e6374b9e5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -52,7 +52,7 @@ public boolean createMetaDataSchema() { (clientId < leftClientDeviceNum) ? eachClientDeviceNum + 1 : eachClientDeviceNum; List deviceSchemaList = new ArrayList<>(); for (int d = 0; d < deviceNumber; d++) { - DeviceSchema deviceSchema = new DeviceSchema(deviceId, sensors, MetaUtil.getTag(deviceId)); + DeviceSchema deviceSchema = new DeviceSchema(deviceId, sensors, MetaUtil.getTags(deviceId)); NAME_DATA_SCHEMA.put(deviceSchema.getDevice(), deviceSchema); GROUPS.add(deviceSchema.getGroup()); deviceSchemaList.add(deviceSchema); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/RealMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/RealMetaDataSchema.java index a9c9389d0..b4c95522a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/RealMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/RealMetaDataSchema.java @@ -73,7 +73,7 @@ protected boolean createMetaDataSchema() { MetaUtil.getGroupIdFromDeviceName(deviceName), deviceName, sensors, - MetaUtil.getTag(deviceName)); + MetaUtil.getTags(deviceName)); NAME_DATA_SCHEMA.put(deviceName, deviceSchema); GROUPS.add(deviceSchema.getGroup()); deviceSchemaList.add(deviceSchema); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java index ca45d3bc2..6bdb3b80b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CSVDataReader.java @@ -80,7 +80,7 @@ public IBatch nextBatch() { MetaUtil.getGroupIdFromDeviceName(deviceName), deviceName, sensors, - MetaUtil.getTag(deviceName)); + MetaUtil.getTags(deviceName)); firstLine = false; continue; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CopyDataReader.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CopyDataReader.java index 54dcc9e18..54d993fff 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CopyDataReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/source/CopyDataReader.java @@ -90,7 +90,7 @@ public CopyDataReader(List files) { MetaUtil.getGroupIdFromDeviceName(deviceName), deviceName, sensors, - MetaUtil.getTag(deviceName)); + MetaUtil.getTags(deviceName)); firstLine = false; continue; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java index ce9396b19..a214f9327 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java @@ -166,7 +166,7 @@ public DeviceQuery getDeviceQuery() { return null; } DeviceSchema deviceSchema = - new DeviceSchema(deviceId, config.getSENSORS(), MetaUtil.getTag(deviceId)); + new DeviceSchema(deviceId, config.getSENSORS(), MetaUtil.getTags(deviceId)); return new DeviceQuery(deviceSchema); } @@ -236,7 +236,7 @@ private List getQueryDeviceSchemaList(boolean typeAllow) throws Wo continue; } DeviceSchema deviceSchema = - new DeviceSchema(deviceId, querySensors, MetaUtil.getTag(deviceId)); + new DeviceSchema(deviceId, querySensors, MetaUtil.getTags(deviceId)); queryDevices.add(deviceSchema); } if (queryDevices.size() != config.getQUERY_DEVICE_NUM()) { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java index c7919514a..e09fe96e2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java @@ -89,7 +89,7 @@ public IBatch getOneBatch() throws WorkloadException { new DeviceSchema( MetaUtil.getDeviceId((int) curLoop % config.getDEVICE_NUMBER()), sensors, - MetaUtil.getTag((int) curLoop % config.getDEVICE_NUMBER())); + MetaUtil.getTags((int) curLoop % config.getDEVICE_NUMBER())); // create data of batch List records = new ArrayList<>(); for (long batchOffset = 0; batchOffset < recordsNumPerDevice; batchOffset++) { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java index d717a0817..85b2482f7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java @@ -50,7 +50,7 @@ public SyntheticDataWorkLoad(List deviceSchemas) { new DeviceSchema( schema.getDeviceId(), Collections.singletonList(sensor), - MetaUtil.getTag(schema.getDeviceId())); + MetaUtil.getTags(schema.getDeviceId())); maxTimestampIndexMap.put(deviceSchema, 0L); } } diff --git a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java index d0afd3756..c79fe8669 100644 --- a/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java +++ b/tdengine-3.0/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine3/TDengine.java @@ -162,7 +162,7 @@ public Double registerSchema(List schemaList) throws TsdbException params.add(deviceSchema.getDevice()); params.add(SUPER_TABLE_NAME); params.add(deviceSchema.getDevice()); - params.addAll(MetaUtil.getTag(deviceSchema.getDeviceId()).values()); + params.addAll(MetaUtil.getTags(deviceSchema.getDeviceId()).values()); statement.execute(String.format(CREATE_TABLE, params.toArray())); } } diff --git a/tdengine/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine/TDengine.java b/tdengine/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine/TDengine.java index 549fb3a5f..1b0f72d86 100644 --- a/tdengine/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine/TDengine.java +++ b/tdengine/src/main/java/cn/edu/tsinghua/iot/benchmark/tdengine/TDengine.java @@ -171,7 +171,7 @@ public Double registerSchema(List schemaList) throws TsdbException params.add(deviceSchema.getDevice()); params.add(SUPER_TABLE_NAME); params.add(deviceSchema.getDevice()); - params.addAll(MetaUtil.getTag(deviceSchema.getDeviceId()).values()); + params.addAll(MetaUtil.getTags(deviceSchema.getDeviceId()).values()); statement.execute(String.format(CREATE_TABLE, params.toArray())); } } catch (SQLException e) { From 199eed001cbd4f1432c69f5018adfa823759407b Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Tue, 14 Nov 2023 17:57:41 +0800 Subject: [PATCH 8/8] fix --- configuration/conf/config.properties | 2 +- .../java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 964392b66..415548437 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -146,7 +146,7 @@ # 标签值前缀 # TAG_VALUE_PREFIX=value_ -# 每个标签值的取值种数，使用逗号分开，设置个数应与标签名数量相等。例如TAG_NUMBER=2，TAG_VALUE_CARDINALITY=50,10，说明第一个标签名可能有50种值，第二个标签名可能有10种值。d_0的中间路径将为value_0.value_0.d_0。 +# 每个标签值的取值种数，使用逗号分开，设置个数应与标签名数量相等。例如TAG_NUMBER=2，TAG_VALUE_CARDINALITY=50,10，说明第一个标签名可能有50种值，第二个标签名可能有10种值。d_0的中间路径将为value_0.value_0.d_0。该配置不影响总点数。 # TAG_VALUE_CARDINALITY= ############## 被测系统为IoTDB时扩展参数 ################## diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 0d640f62e..e3d5efc9a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -95,7 +95,8 @@ public static void setClientFiles(List> clientFiles) { } /** - * Get tags pair by deviceName. + * Get tags pair by deviceName. Tags are allocated based on hashCode to ensure an even number of + * devices under each tag as much as possible. * * @param deviceName deviceName * @return tags pair @@ -115,7 +116,8 @@ public static Map getTags(String deviceName) { } /** - * Get tags pair by deviceId. + * Get tags pair by deviceId. Tags are allocated based on hashCode to ensure an even number of + * devices under each tag as much as possible. * * @param deviceId deviceId * @return tags pair