From e9e0120107ae1768f3d3477b89364b24b1ee8202 Mon Sep 17 00:00:00 2001 From: YuFengLiu <38746920+l2280212@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:13:08 +0800 Subject: [PATCH] Support "register schema only" by set LOOP=0 (#398) * fix code smell in App.java * fix some code smell in mode * delete some unuseful para in workLoad to fix one oom error while device number is too big * make create template more efficient * spotless * fix by reviewed * when loop=0 bm can run * spotless * fixed by review --- .../cn/edu/tsinghua/iot/benchmark/App.java | 10 +++------- .../iot/benchmark/conf/ConfigDescriptor.java | 2 +- .../tsinghua/iot/benchmark/mode/BaseMode.java | 2 +- .../iot/benchmark/mode/GenerateDataMode.java | 6 +++--- .../mode/TestWithDefaultPathMode.java | 13 ++++-------- .../benchmark/mode/VerificationQueryMode.java | 8 ++------ .../benchmark/mode/VerificationWriteMode.java | 17 ++++++---------- .../workload/SingletonWorkDataWorkLoad.java | 6 ------ .../workload/SyntheticDataWorkLoad.java | 18 ----------------- .../iot/benchmark/iotdb110/IoTDB.java | 20 ++++++++----------- 10 files changed, 28 insertions(+), 74 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/App.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/App.java index 97383349a..939b76430 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/App.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/App.java @@ -33,17 +33,13 @@ import java.sql.SQLException; public class App { - private static Logger LOGGER = LoggerFactory.getLogger(Config.class); + private static final Logger LOGGER = LoggerFactory.getLogger(App.class); public static void main(String[] args) throws SQLException { long initialHeapSize = Runtime.getRuntime().totalMemory(); long maxHeapSize = Runtime.getRuntime().maxMemory(); LOGGER.info( - "Initial Heap Size: " - + initialHeapSize - + "bytes, Max Heap Size: " - + maxHeapSize - + "bytes."); + "Initial Heap Size: {} bytes, Max Heap Size: {} bytes. ", initialHeapSize, maxHeapSize); if (args == null || args.length == 0) { args = new String[] {"-cf", "configuration/conf"}; @@ -54,7 +50,7 @@ public static void main(String[] args) throws SQLException { } Runtime.getRuntime().addShutdownHook(new CSVShutdownHook()); Config config = ConfigDescriptor.getInstance().getConfig(); - BaseMode baseMode = null; + BaseMode baseMode; switch (config.getBENCHMARK_WORK_MODE()) { case TEST_WITH_DEFAULT_PATH: baseMode = new TestWithDefaultPathMode(); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 32a128a59..28a30deda 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -628,7 +628,7 @@ private boolean checkOperationProportion() { minOps++; } } - if (minOps > config.getLOOP()) { + if (config.getLOOP() != 0 && minOps > config.getLOOP()) { LOGGER.error("Loop is too small that can't meet the need of OPERATION_PROPORTION"); return false; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/BaseMode.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/BaseMode.java index 4cbd14bb4..4f90e593b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/BaseMode.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/BaseMode.java @@ -179,7 +179,7 @@ protected static void finalMeasure( // must call calculateMetrics() before using the Metrics try { measurement.calculateMetrics(operations); - if (operations.size() != 0) { + if (!operations.isEmpty()) { measurement.showMeasurements(operations); measurement.showMetrics(operations); } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/GenerateDataMode.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/GenerateDataMode.java index caa76c367..0230bceee 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/GenerateDataMode.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/GenerateDataMode.java @@ -40,8 +40,8 @@ protected boolean preCheck() { @Override protected void postCheck() { - LOGGER.info("Data Location: " + config.getFILE_PATH()); - LOGGER.info("Schema Location: " + FileUtils.union(config.getFILE_PATH(), "schema.txt")); - LOGGER.info("Generate Info Location: " + FileUtils.union(config.getFILE_PATH(), "info.txt")); + LOGGER.info("Data Location: {}", config.getFILE_PATH()); + LOGGER.info("Schema Location: {}", FileUtils.union(config.getFILE_PATH(), "schema.txt")); + LOGGER.info("Generate Info Location: {}", FileUtils.union(config.getFILE_PATH(), "info.txt")); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java index c7b922383..94592d0f6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java @@ -39,21 +39,16 @@ protected boolean preCheck() { PersistenceFactory persistenceFactory = new PersistenceFactory(); TestDataPersistence recorder = persistenceFactory.getPersistence(); recorder.saveTestConfig(); - List dbConfigs = new ArrayList<>(); dbConfigs.add(config.getDbConfig()); if (config.isIS_DOUBLE_WRITE()) { dbConfigs.add(config.getANOTHER_DBConfig()); } - if (config.isIS_DELETE_DATA()) { - if (!cleanUpData(dbConfigs, measurement)) { - return false; - } + if (config.isIS_DELETE_DATA() && (!cleanUpData(dbConfigs, measurement))) { + return false; } - if (config.isCREATE_SCHEMA()) { - if (!registerSchema(measurement)) { - return false; - } + if (config.isCREATE_SCHEMA() && (!registerSchema(measurement))) { + return false; } return true; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/VerificationQueryMode.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/VerificationQueryMode.java index 6127631c9..7ea33fb1b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/VerificationQueryMode.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/VerificationQueryMode.java @@ -20,18 +20,14 @@ package cn.edu.tsinghua.iot.benchmark.mode; import cn.edu.tsinghua.iot.benchmark.client.operation.Operation; -import cn.edu.tsinghua.iot.benchmark.conf.Config; -import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import cn.edu.tsinghua.iot.benchmark.measurement.Measurement; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; public class VerificationQueryMode extends BaseMode { - private static final Config config = ConfigDescriptor.getInstance().getConfig(); - @Override protected boolean preCheck() { return true; @@ -45,6 +41,6 @@ protected void postCheck() { threadsMeasurements, start, dataClients, - Arrays.asList(Operation.VERIFICATION_QUERY)); + Collections.singletonList(Operation.VERIFICATION_QUERY)); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/VerificationWriteMode.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/VerificationWriteMode.java index efc6143e4..690a09157 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/VerificationWriteMode.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/VerificationWriteMode.java @@ -26,11 +26,10 @@ import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; public class VerificationWriteMode extends BaseMode { - private static final Config config = ConfigDescriptor.getInstance().getConfig(); @Override @@ -40,15 +39,11 @@ protected boolean preCheck() { if (config.isIS_DOUBLE_WRITE()) { dbConfigs.add(config.getANOTHER_DBConfig()); } - if (config.isIS_DELETE_DATA()) { - if (!cleanUpData(dbConfigs, measurement)) { - return false; - } + if (config.isIS_DELETE_DATA() && (!cleanUpData(dbConfigs, measurement))) { + return false; } - if (config.isCREATE_SCHEMA()) { - if (!registerSchema(measurement)) { - return false; - } + if (config.isCREATE_SCHEMA() && (!registerSchema(measurement))) { + return false; } return true; } @@ -61,6 +56,6 @@ protected void postCheck() { threadsMeasurements, start, dataClients, - new ArrayList<>(Arrays.asList(Operation.INGESTION))); + new ArrayList<>(Collections.singletonList(Operation.INGESTION))); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java index ec7ce0af5..f14de2fa1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SingletonWorkDataWorkLoad.java @@ -31,13 +31,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; public class SingletonWorkDataWorkLoad extends GenerateDataWorkLoad { private static final List SENSORS = Collections.synchronizedList(config.getSENSORS()); - private ConcurrentHashMap deviceMaxTimeIndexMap; private static SingletonWorkDataWorkLoad singletonWorkDataWorkLoad = null; private static final AtomicInteger sensorIndex = new AtomicInteger(); private final AtomicLong insertLoop = new AtomicLong(0); @@ -47,10 +45,6 @@ private SingletonWorkDataWorkLoad() { long startIndex = (long) (config.getLOOP() * config.getOUT_OF_ORDER_RATIO()); this.insertLoop.set(startIndex); } - deviceMaxTimeIndexMap = new ConcurrentHashMap<>(); - for (int i = 0; i < config.getDEVICE_NUMBER(); i++) { - deviceMaxTimeIndexMap.put(MetaUtil.getDeviceId(i), new AtomicLong(0)); - } } public static SingletonWorkDataWorkLoad getInstance() { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java index 85b2482f7..83b1bc8d0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/SyntheticDataWorkLoad.java @@ -25,14 +25,11 @@ import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; -import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; import java.util.*; public class SyntheticDataWorkLoad extends GenerateDataWorkLoad { - - private final Map maxTimestampIndexMap; private long insertLoop = 0; private int deviceIndex = 0; private int sensorIndex = 0; @@ -40,21 +37,6 @@ public class SyntheticDataWorkLoad extends GenerateDataWorkLoad { public SyntheticDataWorkLoad(List deviceSchemas) { this.deviceSchemas = deviceSchemas; - maxTimestampIndexMap = new HashMap<>(); - for (DeviceSchema schema : deviceSchemas) { - if (config.isIS_SENSOR_TS_ALIGNMENT()) { - maxTimestampIndexMap.put(schema, 0L); - } else { - for (Sensor sensor : schema.getSensors()) { - DeviceSchema deviceSchema = - new DeviceSchema( - schema.getDeviceId(), - Collections.singletonList(sensor), - MetaUtil.getTags(schema.getDeviceId())); - maxTimestampIndexMap.put(deviceSchema, 0L); - } - } - } this.deviceSchemaSize = deviceSchemas.size(); } diff --git a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java index 4f17bf64f..0a34d985c 100644 --- a/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java +++ b/iotdb-1.1/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb110/IoTDB.java @@ -305,18 +305,14 @@ private void registerStorageGroups(Session metaSession, List s private void activateTemplate(Session metaSession, List schemaList) { List someDevicePaths = new ArrayList<>(); AtomicLong activatedDeviceCount = new AtomicLong(); - schemaList.stream() - .map(schema -> ROOT_SERIES_NAME + "." + schema.getDeviceSchema().getDevicePath()) - .forEach( - path -> { - someDevicePaths.add(path); - if (someDevicePaths.size() >= ACTIVATE_TEMPLATE_THRESHOLD) { - activateTemplateForSomeDevices( - metaSession, someDevicePaths, activatedDeviceCount.get()); - activatedDeviceCount.addAndGet(someDevicePaths.size()); - someDevicePaths.clear(); - } - }); + for (TimeseriesSchema timeseriesSchema : schemaList) { + someDevicePaths.add(timeseriesSchema.getDeviceId()); + if (someDevicePaths.size() >= ACTIVATE_TEMPLATE_THRESHOLD) { + activateTemplateForSomeDevices(metaSession, someDevicePaths, activatedDeviceCount.get()); + activatedDeviceCount.addAndGet(someDevicePaths.size()); + someDevicePaths.clear(); + } + } if (!someDevicePaths.isEmpty()) { activateTemplateForSomeDevices(metaSession, someDevicePaths, activatedDeviceCount.get()); }