From a4a0e2948a86534dbdca6a255a5063e8366ddcde Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 22 Jul 2024 09:41:17 +0800 Subject: [PATCH 01/33] fix memory concurrency problem --- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 3 +-- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 3 +-- .../storageengine/rescon/memory/SystemInfo.java | 15 ++++++++------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index e9173cc5888a..6a0b9d197dfb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -193,8 +193,7 @@ public synchronized void init() throws MetadataException { if (config.getSchemaRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS)) { long memCost = config.getSchemaRatisConsensusLogAppenderBufferSizeMax(); - if (!SystemInfo.getInstance() - .addDirectBufferMemoryCost(config.getSchemaRatisConsensusLogAppenderBufferSizeMax())) { + if (!SystemInfo.getInstance().addDirectBufferMemoryCost(memCost)) { throw new MetadataException( "Total allocated memory for direct buffer will be " + (SystemInfo.getInstance().getDirectBufferMemoryCost() + memCost) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 4dbfaf5c3bf6..9c7a1b5ace82 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -190,8 +190,7 @@ public synchronized void init() throws MetadataException { if (config.getSchemaRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS)) { long memCost = config.getSchemaRatisConsensusLogAppenderBufferSizeMax(); - if (!SystemInfo.getInstance() - .addDirectBufferMemoryCost(config.getSchemaRatisConsensusLogAppenderBufferSizeMax())) { + if (!SystemInfo.getInstance().addDirectBufferMemoryCost(memCost)) { throw new MetadataException( "Total allocated memory for direct buffer will be " + (SystemInfo.getInstance().getDirectBufferMemoryCost() + memCost) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java index 4b0f56f858d6..a64a4fe24bda 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java @@ -40,6 +40,7 @@ import java.util.PriorityQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -196,15 +197,15 @@ public synchronized void resetFlushingMemTableCost(long flushingMemTableCost) { } public boolean addDirectBufferMemoryCost(long size) { - while (true) { - long memCost = directBufferMemoryCost.get(); + AtomicBoolean result = new AtomicBoolean(false); + directBufferMemoryCost.updateAndGet(memCost -> { if (memCost + size > totalDirectBufferMemorySizeLimit) { - return false; + return memCost; } - if (directBufferMemoryCost.compareAndSet(memCost, memCost + size)) { - return true; - } - } + result.set(true); + return memCost + size; + }); + return result.get(); } public void decreaseDirectBufferMemoryCost(long size) { From 7307bc5821f92641036764aa8c5d5e463fcf193d Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 26 Dec 2024 14:33:55 +0800 Subject: [PATCH 02/33] add memory related metric name and tag --- .../org/apache/iotdb/commons/service/metric/enums/Metric.java | 2 ++ .../org/apache/iotdb/commons/service/metric/enums/Tag.java | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 0894f33dc39b..6d104a643763 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -186,6 +186,8 @@ public enum Metric { LOAD_POINT_COUNT("load_point_count"), MEMTABLE_POINT_COUNT("memtable_point_count"), BINARY_ALLOCATOR("binary_allocator"), + // memory related + MEMORY_SIZE("memory_size"), ; final String value; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java index f3062737d924..30ee06e8484c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java @@ -30,7 +30,9 @@ public enum Tag { OPERATION("operation"), INTERFACE("interface"), CREATION_TIME("creation_time"), - INDEX("index"); + INDEX("index"), + MODULE("module") + ; final String value; From b5d88a6590bf8c3955cadc189f74146a2df1b789 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 26 Dec 2024 15:07:23 +0800 Subject: [PATCH 03/33] add memory threshold code --- .../metrics/DataNodeMetricsHelper.java | 82 +++++---- .../memory/ThresholdMemoryMetrics.java | 174 ++++++++++++++++++ .../commons/service/metric/enums/Metric.java | 2 +- .../commons/service/metric/enums/Tag.java | 4 +- 4 files changed, 220 insertions(+), 42 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java index 17463aa357f2..dda0a403bc52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; +import org.apache.iotdb.db.service.metrics.memory.ThresholdMemoryMetrics; import org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesNumberMetricsSet; import org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesSizeMetricsSet; import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet; @@ -58,52 +59,56 @@ public class DataNodeMetricsHelper { /** Bind predefined metric sets into DataNode. */ public static void bind() { - MetricService.getInstance().addMetricSet(new UpTimeMetrics()); - MetricService.getInstance().addMetricSet(new JvmMetrics()); - MetricService.getInstance().addMetricSet(ThreadPoolMetrics.getInstance()); - MetricService.getInstance().addMetricSet(new LogbackMetrics()); - MetricService.getInstance().addMetricSet(FileMetrics.getInstance()); - MetricService.getInstance().addMetricSet(CompactionMetrics.getInstance()); - MetricService.getInstance().addMetricSet(new ProcessMetrics()); - MetricService.getInstance().addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE)); - MetricService.getInstance().addMetricSet(new NetMetrics(IoTDBConstant.DN_ROLE)); - MetricService.getInstance().addMetricSet(ClientManagerMetrics.getInstance()); - initCpuMetrics(); - initSystemMetrics(); - MetricService.getInstance().addMetricSet(WritingMetrics.getInstance()); + MetricService metricService = MetricService.getInstance(); + metricService.addMetricSet(new UpTimeMetrics()); + metricService.addMetricSet(new JvmMetrics()); + metricService.addMetricSet(ThreadPoolMetrics.getInstance()); + metricService.addMetricSet(new LogbackMetrics()); + metricService.addMetricSet(FileMetrics.getInstance()); + metricService.addMetricSet(CompactionMetrics.getInstance()); + metricService.addMetricSet(new ProcessMetrics()); + metricService.addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE)); + metricService.addMetricSet(new NetMetrics(IoTDBConstant.DN_ROLE)); + metricService.addMetricSet(ClientManagerMetrics.getInstance()); + initCpuMetrics(metricService); + initSystemMetrics(metricService); + metricService.addMetricSet(WritingMetrics.getInstance()); // bind query related metrics - MetricService.getInstance().addMetricSet(QueryPlanCostMetricSet.getInstance()); - MetricService.getInstance().addMetricSet(SeriesScanCostMetricSet.getInstance()); - MetricService.getInstance().addMetricSet(QueryExecutionMetricSet.getInstance()); - MetricService.getInstance().addMetricSet(QueryResourceMetricSet.getInstance()); - MetricService.getInstance().addMetricSet(DataExchangeCostMetricSet.getInstance()); - MetricService.getInstance().addMetricSet(DataExchangeCountMetricSet.getInstance()); - MetricService.getInstance().addMetricSet(DriverSchedulerMetricSet.getInstance()); - MetricService.getInstance().addMetricSet(QueryRelatedResourceMetricSet.getInstance()); + metricService.addMetricSet(QueryPlanCostMetricSet.getInstance()); + metricService.addMetricSet(SeriesScanCostMetricSet.getInstance()); + metricService.addMetricSet(QueryExecutionMetricSet.getInstance()); + metricService.addMetricSet(QueryResourceMetricSet.getInstance()); + metricService.addMetricSet(DataExchangeCostMetricSet.getInstance()); + metricService.addMetricSet(DataExchangeCountMetricSet.getInstance()); + metricService.addMetricSet(DriverSchedulerMetricSet.getInstance()); + metricService.addMetricSet(QueryRelatedResourceMetricSet.getInstance()); // bind performance overview related metrics - MetricService.getInstance().addMetricSet(PerformanceOverviewMetrics.getInstance()); + metricService.addMetricSet(PerformanceOverviewMetrics.getInstance()); // bind gc metrics - MetricService.getInstance().addMetricSet(JvmGcMonitorMetrics.getInstance()); + metricService.addMetricSet(JvmGcMonitorMetrics.getInstance()); // bind pipe related metrics - MetricService.getInstance().addMetricSet(PipeDataNodeMetrics.getInstance()); + metricService.addMetricSet(PipeDataNodeMetrics.getInstance()); // bind load tsfile memory related metrics - MetricService.getInstance().addMetricSet(LoadTsFileMemMetricSet.getInstance()); + metricService.addMetricSet(LoadTsFileMemMetricSet.getInstance()); // bind subscription related metrics - MetricService.getInstance().addMetricSet(SubscriptionMetrics.getInstance()); + metricService.addMetricSet(SubscriptionMetrics.getInstance()); // bind load related metrics - MetricService.getInstance().addMetricSet(LoadTsFileCostMetricsSet.getInstance()); - MetricService.getInstance().addMetricSet(ActiveLoadingFilesNumberMetricsSet.getInstance()); - MetricService.getInstance().addMetricSet(ActiveLoadingFilesSizeMetricsSet.getInstance()); + metricService.addMetricSet(LoadTsFileCostMetricsSet.getInstance()); + metricService.addMetricSet(ActiveLoadingFilesNumberMetricsSet.getInstance()); + metricService.addMetricSet(ActiveLoadingFilesSizeMetricsSet.getInstance()); + + // bind memory related metrics + metricService.addMetricSet(ThresholdMemoryMetrics.getInstance()); } - private static void initSystemMetrics() { + private static void initSystemMetrics(MetricService metricService) { ArrayList diskDirs = new ArrayList<>(); diskDirs.add(IoTDBDescriptor.getInstance().getConfig().getSystemDir()); diskDirs.add(IoTDBDescriptor.getInstance().getConfig().getConsensusDir()); @@ -112,20 +117,19 @@ private static void initSystemMetrics() { diskDirs.add(CommonDescriptor.getInstance().getConfig().getSyncDir()); diskDirs.add(IoTDBDescriptor.getInstance().getConfig().getSortTmpDir()); SystemMetrics.getInstance().setDiskDirs(diskDirs); - MetricService.getInstance().addMetricSet(SystemMetrics.getInstance()); + metricService.addMetricSet(SystemMetrics.getInstance()); } - private static void initCpuMetrics() { + private static void initCpuMetrics(MetricService metricService) { List threadModules = new ArrayList<>(); Arrays.stream(ThreadModule.values()).forEach(x -> threadModules.add(x.toString())); List pools = new ArrayList<>(); Arrays.stream(ThreadName.values()).forEach(x -> pools.add(x.name())); - MetricService.getInstance() - .addMetricSet( - new CpuUsageMetrics( - threadModules, - pools, - x -> ThreadName.getModuleTheThreadBelongs(x).toString(), - x -> ThreadName.getThreadPoolTheThreadBelongs(x).name())); + metricService.addMetricSet( + new CpuUsageMetrics( + threadModules, + pools, + x -> ThreadName.getModuleTheThreadBelongs(x).toString(), + x -> ThreadName.getThreadPoolTheThreadBelongs(x).name())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java new file mode 100644 index 000000000000..bdb333a37952 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +public class ThresholdMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final SystemInfo systemInfo = SystemInfo.getInstance(); + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + "StorageEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0") + .set(config.getAllocateMemoryForStorageEngine()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + "QueryEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0") + .set(config.getAllocateMemoryForRead()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + "SchemaEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0") + .set(config.getAllocateMemoryForSchema()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + "Consensus", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0") + .set(config.getAllocateMemoryForConsensus()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + "StreamEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0") + .set(config.getAllocateMemoryForPipe()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + "DirectBuffer", + Tag.TYPE.toString(), + "OffHeap", + Tag.LEVEL.toString(), + "0") + .set(systemInfo.getTotalDirectBufferMemorySizeLimit()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + "StorageEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0"); + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + "QueryEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0"); + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + "SchemaEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0"); + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + "Consensus", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0"); + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + "StreamEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "0"); + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + "DirectBuffer", + Tag.TYPE.toString(), + "OffHeap", + Tag.LEVEL.toString(), + "0"); + } + + public static ThresholdMemoryMetrics getInstance() { + return ThresholdMemoryMetrics.ThresholdMemoryMetricsHolder.INSTANCE; + } + + private static class ThresholdMemoryMetricsHolder { + + private static final ThresholdMemoryMetrics INSTANCE = new ThresholdMemoryMetrics(); + + private ThresholdMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 6d104a643763..b70aae9057ff 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -187,7 +187,7 @@ public enum Metric { MEMTABLE_POINT_COUNT("memtable_point_count"), BINARY_ALLOCATOR("binary_allocator"), // memory related - MEMORY_SIZE("memory_size"), + THRESHOLD_MEMORY_SIZE("threshold_memory_size"), ; final String value; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java index 30ee06e8484c..5fecc6b31220 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java @@ -31,8 +31,8 @@ public enum Tag { INTERFACE("interface"), CREATION_TIME("creation_time"), INDEX("index"), - MODULE("module") - ; + MODULE("module"), + LEVEL("level"); final String value; From 05b0757a23f5d6b5046dcf531cc10ab959e7f838 Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 27 Dec 2024 10:49:25 +0800 Subject: [PATCH 04/33] add total --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 7 --- .../memory/ThresholdMemoryMetrics.java | 44 ++++++++++++++----- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 9b26ca3da239..3554610e20bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -2156,13 +2156,6 @@ public void setAllocateMemoryForPipe(long allocateMemoryForPipe) { this.allocateMemoryForPipe = allocateMemoryForPipe; } - public long getAllocateMemoryForFree() { - return Runtime.getRuntime().maxMemory() - - allocateMemoryForStorageEngine - - allocateMemoryForRead - - allocateMemoryForSchema; - } - public boolean isEnablePartialInsert() { return enablePartialInsert; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index bdb333a37952..bd31e32f4b9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -40,11 +40,22 @@ public void bindTo(AbstractMetricService metricService) { Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - "StorageEngine", + "Total", Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), "0") + .set(Runtime.getRuntime().maxMemory()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + "StorageEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "1") .set(config.getAllocateMemoryForStorageEngine()); metricService .getOrCreateGauge( @@ -55,7 +66,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), - "0") + "1") .set(config.getAllocateMemoryForRead()); metricService .getOrCreateGauge( @@ -66,7 +77,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), - "0") + "1") .set(config.getAllocateMemoryForSchema()); metricService .getOrCreateGauge( @@ -77,7 +88,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), - "0") + "1") .set(config.getAllocateMemoryForConsensus()); metricService .getOrCreateGauge( @@ -88,7 +99,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), - "0") + "1") .set(config.getAllocateMemoryForPipe()); metricService .getOrCreateGauge( @@ -99,7 +110,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), "OffHeap", Tag.LEVEL.toString(), - "0") + "1") .set(systemInfo.getTotalDirectBufferMemorySizeLimit()); } @@ -109,11 +120,20 @@ public void unbindFrom(AbstractMetricService metricService) { MetricType.GAUGE, Metric.THRESHOLD_MEMORY_SIZE.toString(), Tag.NAME.toString(), - "StorageEngine", + "Total", Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), "0"); + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + "StorageEngine", + Tag.TYPE.toString(), + "OnHeap", + Tag.LEVEL.toString(), + "1"); metricService.remove( MetricType.GAUGE, Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -122,7 +142,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), - "0"); + "1"); metricService.remove( MetricType.GAUGE, Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -131,7 +151,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), - "0"); + "1"); metricService.remove( MetricType.GAUGE, Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -140,7 +160,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), - "0"); + "1"); metricService.remove( MetricType.GAUGE, Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -149,7 +169,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.TYPE.toString(), "OnHeap", Tag.LEVEL.toString(), - "0"); + "1"); metricService.remove( MetricType.GAUGE, Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -158,7 +178,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.TYPE.toString(), "OffHeap", Tag.LEVEL.toString(), - "0"); + "1"); } public static ThresholdMemoryMetrics getInstance() { From 01ad4153056a12d6541b8d3ca475db284bbf25c2 Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 10:34:33 +0800 Subject: [PATCH 05/33] add part of write metrics --- .../memory/ThresholdMemoryMetrics.java | 282 ++++++++++++------ 1 file changed, 184 insertions(+), 98 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index bd31e32f4b9d..bca8ceeb373c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -25,160 +25,246 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.impl.DoNothingMetricManager; import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.type.Gauge; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.MetricType; +import java.util.Arrays; +import java.util.Collections; + public class ThresholdMemoryMetrics implements IMetricSet { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final SystemInfo systemInfo = SystemInfo.getInstance(); + private Gauge totalMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge storageEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge queryEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge schemaEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge consensusMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge streamEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge directBufferMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + + private static final String TOTAL = "Total"; + private static final String ON_HEAP = "OnHeap"; + private static final String OFF_HEAP = "OffHeap"; + private static final String STORAGE_ENGINE = "StorageEngine"; + private static final String STORAGE_ENGINE_WRITE = "StorageEngine-Write"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE = "StorageEngine-Write-Memtable"; + private static final String STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO = + "StorageEngine-Write-TimePartitionInfo"; + private static final String STORAGE_ENGINE_COMPACTION = "StorageEngine-Compaction"; + private static final String QUERY_ENGINE = "QueryEngine"; + private static final String SCHEMA_ENGINE = "SchemaEngine"; + private static final String CONSENSUS = "Consensus"; + private static final String STREAM_ENGINE = "StreamEngine"; + private static final String DIRECT_BUFFER = "DirectBuffer"; + @Override public void bindTo(AbstractMetricService metricService) { - metricService - .getOrCreateGauge( + totalMemorySize = + metricService.getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + TOTAL, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + "0"); + totalMemorySize.set(Runtime.getRuntime().maxMemory()); + bindStorageEngineRelatedMemoryMetrics(metricService); + queryEngineMemorySize = + metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - "Total", + QUERY_ENGINE, Tag.TYPE.toString(), - "OnHeap", + ON_HEAP, Tag.LEVEL.toString(), - "0") - .set(Runtime.getRuntime().maxMemory()); - metricService - .getOrCreateGauge( + "1"); + queryEngineMemorySize.set(config.getAllocateMemoryForRead()); + schemaEngineMemorySize = + metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - "StorageEngine", + SCHEMA_ENGINE, Tag.TYPE.toString(), - "OnHeap", + ON_HEAP, Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForStorageEngine()); - metricService - .getOrCreateGauge( + "1"); + schemaEngineMemorySize.set(config.getAllocateMemoryForSchema()); + consensusMemorySize = + metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - "QueryEngine", + CONSENSUS, Tag.TYPE.toString(), - "OnHeap", + ON_HEAP, Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForRead()); - metricService - .getOrCreateGauge( + "1"); + consensusMemorySize.set(config.getAllocateMemoryForConsensus()); + streamEngineMemorySize = + metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - "SchemaEngine", + SCHEMA_ENGINE, Tag.TYPE.toString(), - "OnHeap", + ON_HEAP, Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForSchema()); - metricService - .getOrCreateGauge( + "1"); + streamEngineMemorySize.set(config.getAllocateMemoryForPipe()); + directBufferMemorySize = + metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - "Consensus", + DIRECT_BUFFER, Tag.TYPE.toString(), - "OnHeap", + ON_HEAP, Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForConsensus()); - metricService - .getOrCreateGauge( + "1"); + directBufferMemorySize.set(systemInfo.getTotalDirectBufferMemorySizeLimit()); + } + + private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricService) { + long storageEngineSize = config.getAllocateMemoryForStorageEngine(); + long writeSize = + (long) + (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); + long memtableSize = + (long) + (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable()); + long timePartitionInfoSize = config.getAllocateMemoryForTimePartitionInfo(); + long compactionSize = storageEngineSize - writeSize; + storageEngineMemorySize = + metricService.getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + "1"); + schemaEngineMemorySize.set(storageEngineSize); + writeMemorySize = + metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - "StreamEngine", + STORAGE_ENGINE_WRITE, Tag.TYPE.toString(), - "OnHeap", + OFF_HEAP, Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForPipe()); - metricService - .getOrCreateGauge( + "2"); + writeMemorySize.set(writeSize); + memtableMemorySize = + metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - "DirectBuffer", + STORAGE_ENGINE_WRITE_MEMTABLE, Tag.TYPE.toString(), - "OffHeap", + OFF_HEAP, Tag.LEVEL.toString(), - "1") - .set(systemInfo.getTotalDirectBufferMemorySizeLimit()); + "2"); + memtableMemorySize.set(memtableSize); + timePartitionInfoMemorySize = + metricService.getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + "2"); + timePartitionInfoMemorySize.set(timePartitionInfoSize); + compactionMemorySize = + metricService.getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_COMPACTION, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + "2"); + compactionMemorySize.set(compactionSize); } @Override public void unbindFrom(AbstractMetricService metricService) { + totalMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; metricService.remove( MetricType.GAUGE, Metric.THRESHOLD_MEMORY_SIZE.toString(), Tag.NAME.toString(), - "Total", + TOTAL, Tag.TYPE.toString(), - "OnHeap", + ON_HEAP, Tag.LEVEL.toString(), "0"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "StorageEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "QueryEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "SchemaEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "Consensus", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "StreamEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "DirectBuffer", - Tag.TYPE.toString(), - "OffHeap", - Tag.LEVEL.toString(), - "1"); + storageEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + queryEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + schemaEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + consensusMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + streamEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + directBufferMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + Arrays.asList(STORAGE_ENGINE, QUERY_ENGINE, SCHEMA_ENGINE, CONSENSUS, STREAM_ENGINE) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + "1")); + Collections.singletonList(DIRECT_BUFFER) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + "1")); + writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + Arrays.asList( + STORAGE_ENGINE_WRITE, + STORAGE_ENGINE_WRITE_MEMTABLE, + STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, + STORAGE_ENGINE_COMPACTION) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + "2")); } public static ThresholdMemoryMetrics getInstance() { From d019c60c9aafb9aa3e4a8e234b5a26fbf777e4f1 Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 10:37:23 +0800 Subject: [PATCH 06/33] remove hard string code --- .../memory/ThresholdMemoryMetrics.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index bca8ceeb373c..9a85ab666399 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -64,6 +64,7 @@ public class ThresholdMemoryMetrics implements IMetricSet { private static final String CONSENSUS = "Consensus"; private static final String STREAM_ENGINE = "StreamEngine"; private static final String DIRECT_BUFFER = "DirectBuffer"; + private static final String[] LEVELS = {"0", "1", "2"}; @Override public void bindTo(AbstractMetricService metricService) { @@ -76,7 +77,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - "0"); + LEVELS[0]); totalMemorySize.set(Runtime.getRuntime().maxMemory()); bindStorageEngineRelatedMemoryMetrics(metricService); queryEngineMemorySize = @@ -88,7 +89,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - "1"); + LEVELS[1]); queryEngineMemorySize.set(config.getAllocateMemoryForRead()); schemaEngineMemorySize = metricService.getOrCreateGauge( @@ -99,7 +100,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - "1"); + LEVELS[1]); schemaEngineMemorySize.set(config.getAllocateMemoryForSchema()); consensusMemorySize = metricService.getOrCreateGauge( @@ -110,7 +111,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - "1"); + LEVELS[1]); consensusMemorySize.set(config.getAllocateMemoryForConsensus()); streamEngineMemorySize = metricService.getOrCreateGauge( @@ -121,7 +122,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - "1"); + LEVELS[1]); streamEngineMemorySize.set(config.getAllocateMemoryForPipe()); directBufferMemorySize = metricService.getOrCreateGauge( @@ -132,7 +133,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - "1"); + LEVELS[1]); directBufferMemorySize.set(systemInfo.getTotalDirectBufferMemorySizeLimit()); } @@ -155,7 +156,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - "1"); + LEVELS[1]); schemaEngineMemorySize.set(storageEngineSize); writeMemorySize = metricService.getOrCreateGauge( @@ -166,7 +167,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - "2"); + LEVELS[2]); writeMemorySize.set(writeSize); memtableMemorySize = metricService.getOrCreateGauge( @@ -177,7 +178,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - "2"); + LEVELS[2]); memtableMemorySize.set(memtableSize); timePartitionInfoMemorySize = metricService.getOrCreateGauge( @@ -188,7 +189,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - "2"); + LEVELS[2]); timePartitionInfoMemorySize.set(timePartitionInfoSize); compactionMemorySize = metricService.getOrCreateGauge( @@ -199,7 +200,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - "2"); + LEVELS[2]); compactionMemorySize.set(compactionSize); } @@ -232,7 +233,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - "1")); + LEVELS[1])); Collections.singletonList(DIRECT_BUFFER) .forEach( name -> @@ -244,7 +245,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - "1")); + LEVELS[1])); writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; @@ -264,7 +265,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - "2")); + LEVELS[2])); } public static ThresholdMemoryMetrics getInstance() { From b97b406e408c9471411d5392dd1000181832814e Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 10:38:11 +0800 Subject: [PATCH 07/33] fix error code(set storage engine size to schema engine --- .../iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index 9a85ab666399..e4926657a22a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -157,7 +157,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS OFF_HEAP, Tag.LEVEL.toString(), LEVELS[1]); - schemaEngineMemorySize.set(storageEngineSize); + storageEngineMemorySize.set(storageEngineSize); writeMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), From 57d529bc81511e06d7415ce52108b6ca3f14715c Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 10:39:57 +0800 Subject: [PATCH 08/33] degrade metric level --- .../db/service/metrics/WritingMetrics.java | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java index 8820be02fa86..d53047b7464d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java @@ -67,37 +67,37 @@ private void bindFlushMetrics(AbstractMetricService metricService) { flushStageSortTimer = metricService.getOrCreateTimer( Metric.FLUSH_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), FLUSH_STAGE_SORT); flushStageEncodingTimer = metricService.getOrCreateTimer( Metric.FLUSH_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), FLUSH_STAGE_ENCODING); flushStageIOTimer = metricService.getOrCreateTimer( Metric.FLUSH_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), FLUSH_STAGE_IO); writePlanIndicesTimer = metricService.getOrCreateTimer( Metric.FLUSH_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), WRITE_PLAN_INDICES); metricService.createAutoGauge( Metric.PENDING_FLUSH_TASK.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, FlushManager.getInstance(), FlushManager::getNumberOfPendingTasks, Tag.TYPE.toString(), PENDING_TASK_NUM); metricService.createAutoGauge( Metric.PENDING_FLUSH_TASK.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, FlushManager.getInstance(), FlushManager::getNumberOfPendingSubTasks, Tag.TYPE.toString(), @@ -139,19 +139,19 @@ private void bindFlushSubTaskMetrics(AbstractMetricService metricService) { sortTaskTimer = metricService.getOrCreateTimer( Metric.FLUSH_SUB_TASK_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), SORT_TASK); encodingTaskTimer = metricService.getOrCreateTimer( Metric.FLUSH_SUB_TASK_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), ENCODING_TASK); ioTaskTimer = metricService.getOrCreateTimer( Metric.FLUSH_SUB_TASK_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), IO_TASK); } @@ -200,73 +200,73 @@ private void unbindFlushSubTaskMetrics(AbstractMetricService metricService) { private void bindWALMetrics(AbstractMetricService metricService) { metricService.createAutoGauge( Metric.WAL_NODE_NUM.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, WAL_MANAGER, WALManager::getWALNodesNum, Tag.NAME.toString(), WAL_NODES_NUM); usedRatioHistogram = metricService.getOrCreateHistogram( - Metric.WAL_BUFFER.toString(), MetricLevel.IMPORTANT, Tag.NAME.toString(), USED_RATIO); + Metric.WAL_BUFFER.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), USED_RATIO); entriesCountHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), ENTRIES_COUNT); serializedWALBufferSizeHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), SERIALIZED_WAL_BUFFER_SIZE_BYTE); wroteWALBufferSizeHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), WROTE_WAL_BUFFER_SIZE_BYTE); walCompressCostHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), WAL_COMPRESS_COST_NS); walUncompressCostHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), WAL_UNCOMPRESS_COST_NS); readWALBufferSizeHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), READ_WAL_BUFFER_SIZE_BYTE); readWALBufferCostHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), READ_WAL_BUFFER_COST_NS); writeWALBufferCostHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), WRITE_WAL_BUFFER_COST_NS); walQueueMaxMemSizeGauge = metricService.getOrCreateGauge( Metric.WAL_QUEUE_MEM_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), WAL_QUEUE_MAX_MEM_COST); SystemInfo systemInfo = SystemInfo.getInstance(); metricService.createAutoGauge( Metric.WAL_QUEUE_MEM_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, systemInfo, SystemInfo::getCurrentWalQueueMemoryCost, Tag.NAME.toString(), @@ -324,7 +324,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { globalMemoryTableInfoTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), MAKE_CHECKPOINT, Tag.TYPE.toString(), @@ -332,7 +332,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { createMemoryTableTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), MAKE_CHECKPOINT, Tag.TYPE.toString(), @@ -340,7 +340,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { flushMemoryTableTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), MAKE_CHECKPOINT, Tag.TYPE.toString(), @@ -348,7 +348,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { serializeWalEntryTotalTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), SERIALIZE_WAL_ENTRY, Tag.TYPE.toString(), @@ -356,7 +356,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { syncTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), SYNC_WAL_BUFFER, Tag.TYPE.toString(), @@ -364,7 +364,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { fsyncTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.STAGE.toString(), SYNC_WAL_BUFFER, Tag.TYPE.toString(), @@ -465,20 +465,20 @@ public void bindDataRegionMetrics() { MetricService.getInstance() .getOrCreateGauge( Metric.MEMTABLE_THRESHOLD.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), FLUSH_THRESHOLD); rejectThreholdGauge = MetricService.getInstance() .getOrCreateGauge( Metric.MEMTABLE_THRESHOLD.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), REJECT_THRESHOLD); memtableLiveTimer = MetricService.getInstance() - .getOrCreateTimer(Metric.MEMTABLE_LIVE_DURATION.toString(), MetricLevel.IMPORTANT); + .getOrCreateTimer(Metric.MEMTABLE_LIVE_DURATION.toString(), MetricLevel.NORMAL); } public void unbindDataRegionMetrics() { @@ -515,7 +515,7 @@ public void createDataRegionMemoryCostMetrics(DataRegion dataRegion) { MetricService.getInstance() .createAutoGauge( Metric.DATA_REGION_MEM_COST.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, dataRegion, DataRegion::getMemCost, Tag.REGION.toString(), @@ -541,7 +541,7 @@ public void createWALNodeInfoMetrics(String walNodeId) { MetricService.getInstance() .getOrCreateHistogram( Metric.WAL_NODE_INFO.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), name, Tag.TYPE.toString(), @@ -578,7 +578,7 @@ public void createFlushingMemTableStatusMetrics(DataRegionId dataRegionId) { MetricService.getInstance() .getOrCreateHistogram( Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), name, Tag.REGION.toString(), @@ -589,7 +589,7 @@ public Counter createWalFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), WAL_FLUSH_MEMTABLE_COUNT); } @@ -598,7 +598,7 @@ public Counter createTimedFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), TIMED_FLUSH_MEMTABLE_COUNT); } @@ -607,7 +607,7 @@ public Counter createSeriesFullFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), SERIES_FULL_FLUSH_MEMTABLE); } @@ -616,7 +616,7 @@ public Counter createManualFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), MANUAL_FLUSH_MEMTABLE_COUNT); } @@ -625,7 +625,7 @@ public Counter createMemControlFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.TYPE.toString(), MEM_CONTROL_FLUSH_MEMTABLE_COUNT); } @@ -634,14 +634,14 @@ public void createActiveMemtableCounterMetrics(DataRegionId dataRegionId) { MetricService.getInstance() .getOrCreateCounter( Metric.ACTIVE_MEMTABLE_COUNT.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.REGION.toString(), dataRegionId.toString()); } public void createActiveTimePartitionCounterMetrics() { MetricService.getInstance() - .getOrCreateCounter(Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), MetricLevel.IMPORTANT); + .getOrCreateCounter(Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), MetricLevel.NORMAL); } public void removeSeriesFullFlushMemTableCounterMetrics() { @@ -728,7 +728,7 @@ public void recordWALNodeEffectiveInfoRatio(String walNodeId, double ratio) { .histogram( (long) (ratio * 100), Metric.WAL_NODE_INFO.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), EFFECTIVE_RATIO_INFO, Tag.TYPE.toString(), @@ -740,7 +740,7 @@ public void recordMemTableRamWhenCauseSnapshot(String walNodeId, long ram) { .histogram( ram, Metric.WAL_NODE_INFO.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), OLDEST_MEM_TABLE_RAM_WHEN_CAUSE_SNAPSHOT, Tag.TYPE.toString(), @@ -752,7 +752,7 @@ public void recordMemTableRamWhenCauseFlush(String walNodeId, long ram) { .histogram( ram, Metric.WAL_NODE_INFO.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), OLDEST_MEM_TABLE_RAM_WHEN_CAUSE_FLUSH, Tag.TYPE.toString(), @@ -765,7 +765,7 @@ public void recordTsFileCompressionRatioOfFlushingMemTable( .histogram( (long) (compressionRatio * 100), Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), COMPRESSION_RATIO, Tag.REGION.toString(), @@ -783,7 +783,7 @@ public void recordFlushingMemTableStatus( .histogram( memSize, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), MEM_TABLE_SIZE, Tag.REGION.toString(), @@ -792,7 +792,7 @@ public void recordFlushingMemTableStatus( .histogram( seriesNum, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), SERIES_NUM, Tag.REGION.toString(), @@ -801,7 +801,7 @@ public void recordFlushingMemTableStatus( .histogram( totalPointsNum, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), POINTS_NUM, Tag.REGION.toString(), @@ -810,7 +810,7 @@ public void recordFlushingMemTableStatus( .histogram( avgSeriesNum, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), AVG_SERIES_POINT_NUM, Tag.REGION.toString(), @@ -826,7 +826,7 @@ public void recordFlushTsFileSize(String storageGroup, long size) { .histogram( size, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.NAME.toString(), FLUSH_TSFILE_SIZE, Tag.REGION.toString(), @@ -978,14 +978,14 @@ public void recordActiveMemTableCount(String dataRegionId, int number) { .count( number, Metric.ACTIVE_MEMTABLE_COUNT.toString(), - MetricLevel.IMPORTANT, + MetricLevel.NORMAL, Tag.REGION.toString(), dataRegionId); } public void recordActiveTimePartitionCount(int number) { MetricService.getInstance() - .count(number, Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), MetricLevel.IMPORTANT); + .count(number, Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), MetricLevel.NORMAL); } // endregion From 1d235924f10191872f31ec03db90f91c7030cbd8 Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 10:54:22 +0800 Subject: [PATCH 09/33] fix error level --- .../db/service/metrics/WritingMetrics.java | 10 ++----- .../memory/ThresholdMemoryMetrics.java | 28 ++++++++++++------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java index d53047b7464d..d6b16e85daf7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java @@ -78,10 +78,7 @@ private void bindFlushMetrics(AbstractMetricService metricService) { FLUSH_STAGE_ENCODING); flushStageIOTimer = metricService.getOrCreateTimer( - Metric.FLUSH_COST.toString(), - MetricLevel.NORMAL, - Tag.STAGE.toString(), - FLUSH_STAGE_IO); + Metric.FLUSH_COST.toString(), MetricLevel.NORMAL, Tag.STAGE.toString(), FLUSH_STAGE_IO); writePlanIndicesTimer = metricService.getOrCreateTimer( Metric.FLUSH_COST.toString(), @@ -210,10 +207,7 @@ private void bindWALMetrics(AbstractMetricService metricService) { Metric.WAL_BUFFER.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), USED_RATIO); entriesCountHistogram = metricService.getOrCreateHistogram( - Metric.WAL_BUFFER.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - ENTRIES_COUNT); + Metric.WAL_BUFFER.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), ENTRIES_COUNT); serializedWALBufferSizeHistogram = metricService.getOrCreateHistogram( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index e4926657a22a..c96045a97887 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -64,7 +64,7 @@ public class ThresholdMemoryMetrics implements IMetricSet { private static final String CONSENSUS = "Consensus"; private static final String STREAM_ENGINE = "StreamEngine"; private static final String DIRECT_BUFFER = "DirectBuffer"; - private static final String[] LEVELS = {"0", "1", "2"}; + private static final String[] LEVELS = {"0", "1", "2", "3", "4"}; @Override public void bindTo(AbstractMetricService metricService) { @@ -178,7 +178,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - LEVELS[2]); + LEVELS[3]); memtableMemorySize.set(memtableSize); timePartitionInfoMemorySize = metricService.getOrCreateGauge( @@ -189,7 +189,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - LEVELS[2]); + LEVELS[3]); timePartitionInfoMemorySize.set(timePartitionInfoSize); compactionMemorySize = metricService.getOrCreateGauge( @@ -247,14 +247,8 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.LEVEL.toString(), LEVELS[1])); writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - Arrays.asList( - STORAGE_ENGINE_WRITE, - STORAGE_ENGINE_WRITE_MEMTABLE, - STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, - STORAGE_ENGINE_COMPACTION) + Arrays.asList(STORAGE_ENGINE_WRITE, STORAGE_ENGINE_COMPACTION) .forEach( name -> metricService.remove( @@ -266,6 +260,20 @@ public void unbindFrom(AbstractMetricService metricService) { OFF_HEAP, Tag.LEVEL.toString(), LEVELS[2])); + memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + LEVELS[3])); } public static ThresholdMemoryMetrics getInstance() { From 44adcdbceccdd87b8d5779ed87fa045013251881 Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 11:07:49 +0800 Subject: [PATCH 10/33] fix error type and add datanode device path cache --- .../memory/ThresholdMemoryMetrics.java | 83 +++++++++++++------ 1 file changed, 59 insertions(+), 24 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index c96045a97887..e88c501c8193 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -42,6 +42,7 @@ public class ThresholdMemoryMetrics implements IMetricSet { private Gauge storageEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge dataNodeDevicePathCacheMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge queryEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; @@ -56,6 +57,7 @@ public class ThresholdMemoryMetrics implements IMetricSet { private static final String STORAGE_ENGINE = "StorageEngine"; private static final String STORAGE_ENGINE_WRITE = "StorageEngine-Write"; private static final String STORAGE_ENGINE_WRITE_MEMTABLE = "StorageEngine-Write-Memtable"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE_CACHE = "StorageEngine-Write-Memtable-DevicePathCache"; private static final String STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO = "StorageEngine-Write-TimePartitionInfo"; private static final String STORAGE_ENGINE_COMPACTION = "StorageEngine-Compaction"; @@ -131,7 +133,7 @@ public void bindTo(AbstractMetricService metricService) { Tag.NAME.toString(), DIRECT_BUFFER, Tag.TYPE.toString(), - ON_HEAP, + OFF_HEAP, Tag.LEVEL.toString(), LEVELS[1]); directBufferMemorySize.set(systemInfo.getTotalDirectBufferMemorySizeLimit()); @@ -139,14 +141,7 @@ public void bindTo(AbstractMetricService metricService) { private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricService) { long storageEngineSize = config.getAllocateMemoryForStorageEngine(); - long writeSize = - (long) - (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); - long memtableSize = - (long) - (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable()); - long timePartitionInfoSize = config.getAllocateMemoryForTimePartitionInfo(); - long compactionSize = storageEngineSize - writeSize; + // Total memory size of storage engine storageEngineMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -154,10 +149,15 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.NAME.toString(), STORAGE_ENGINE, Tag.TYPE.toString(), - OFF_HEAP, + ON_HEAP, Tag.LEVEL.toString(), LEVELS[1]); storageEngineMemorySize.set(storageEngineSize); + // The memory of storage engine divided into Write and Compaction 2 part + long writeSize = + (long) + (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); + long compactionSize = storageEngineSize - writeSize; writeMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -165,10 +165,26 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.NAME.toString(), STORAGE_ENGINE_WRITE, Tag.TYPE.toString(), - OFF_HEAP, + ON_HEAP, Tag.LEVEL.toString(), LEVELS[2]); writeMemorySize.set(writeSize); + compactionMemorySize = + metricService.getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_COMPACTION, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]); + compactionMemorySize.set(compactionSize); + // The write memory of storage engine divided into MemTable and TimePartitionInfo 2 parts + long memtableSize = + (long) + (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable()); + long timePartitionInfoSize = config.getAllocateMemoryForTimePartitionInfo(); memtableMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -176,32 +192,38 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.NAME.toString(), STORAGE_ENGINE_WRITE_MEMTABLE, Tag.TYPE.toString(), - OFF_HEAP, + ON_HEAP, Tag.LEVEL.toString(), LEVELS[3]); memtableMemorySize.set(memtableSize); - timePartitionInfoMemorySize = + // The memtable memory of storage engine contain DataNodeDevicePathCache (NOTICE: This part of memory is not divided) + long dataNodeDevicePathCacheSize = + (long) + (config.getAllocateMemoryForStorageEngine() + * config.getWriteProportionForMemtable() + * config.getDevicePathCacheProportion()); + dataNodeDevicePathCacheMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, + STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, Tag.TYPE.toString(), - OFF_HEAP, + ON_HEAP, Tag.LEVEL.toString(), - LEVELS[3]); - timePartitionInfoMemorySize.set(timePartitionInfoSize); - compactionMemorySize = + LEVELS[4]); + dataNodeDevicePathCacheMemorySize.set(dataNodeDevicePathCacheSize); + timePartitionInfoMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - STORAGE_ENGINE_COMPACTION, + STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, Tag.TYPE.toString(), - OFF_HEAP, + ON_HEAP, Tag.LEVEL.toString(), - LEVELS[2]); - compactionMemorySize.set(compactionSize); + LEVELS[3]); + timePartitionInfoMemorySize.set(timePartitionInfoSize); } @Override @@ -257,7 +279,7 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.NAME.toString(), name, Tag.TYPE.toString(), - OFF_HEAP, + ON_HEAP, Tag.LEVEL.toString(), LEVELS[2])); memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; @@ -271,9 +293,22 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.NAME.toString(), name, Tag.TYPE.toString(), - OFF_HEAP, + ON_HEAP, Tag.LEVEL.toString(), LEVELS[3])); + dataNodeDevicePathCacheMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + Collections.singletonList(STORAGE_ENGINE_WRITE_MEMTABLE_CACHE) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[4])); } public static ThresholdMemoryMetrics getInstance() { From 713e39bc18c50dd7a15941f7cfd7c8bdea50203f Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 11:13:49 +0800 Subject: [PATCH 11/33] finish write part --- .../memory/ThresholdMemoryMetrics.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index e88c501c8193..b3ba20a9710a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -44,6 +44,7 @@ public class ThresholdMemoryMetrics implements IMetricSet { private Gauge memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge dataNodeDevicePathCacheMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge bufferedArraysMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge queryEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge schemaEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; @@ -60,6 +61,8 @@ public class ThresholdMemoryMetrics implements IMetricSet { private static final String STORAGE_ENGINE_WRITE_MEMTABLE_CACHE = "StorageEngine-Write-Memtable-DevicePathCache"; private static final String STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO = "StorageEngine-Write-TimePartitionInfo"; + private static final String STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS = + "StorageEngine-Write-BufferedArrays"; private static final String STORAGE_ENGINE_COMPACTION = "StorageEngine-Compaction"; private static final String QUERY_ENGINE = "QueryEngine"; private static final String SCHEMA_ENGINE = "SchemaEngine"; @@ -158,6 +161,9 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS (long) (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); long compactionSize = storageEngineSize - writeSize; + long bufferedArraySize = + (long) (config.getAllocateMemoryForStorageEngine() + * config.getBufferedArraysMemoryProportion()); writeMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -224,6 +230,17 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.LEVEL.toString(), LEVELS[3]); timePartitionInfoMemorySize.set(timePartitionInfoSize); + bufferedArraysMemorySize = + metricService.getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[3]); + bufferedArraysMemorySize.set(bufferedArraySize); } @Override @@ -284,7 +301,8 @@ public void unbindFrom(AbstractMetricService metricService) { LEVELS[2])); memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO) + bufferedArraysMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS) .forEach( name -> metricService.remove( From a093a1c9bcd2761cd04554a56903f27e7032dce9 Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 11:22:35 +0800 Subject: [PATCH 12/33] fix format --- .../metrics/memory/ThresholdMemoryMetrics.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index b3ba20a9710a..2c261727bd55 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -58,7 +58,8 @@ public class ThresholdMemoryMetrics implements IMetricSet { private static final String STORAGE_ENGINE = "StorageEngine"; private static final String STORAGE_ENGINE_WRITE = "StorageEngine-Write"; private static final String STORAGE_ENGINE_WRITE_MEMTABLE = "StorageEngine-Write-Memtable"; - private static final String STORAGE_ENGINE_WRITE_MEMTABLE_CACHE = "StorageEngine-Write-Memtable-DevicePathCache"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE_CACHE = + "StorageEngine-Write-Memtable-DevicePathCache"; private static final String STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO = "StorageEngine-Write-TimePartitionInfo"; private static final String STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS = @@ -162,8 +163,9 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); long compactionSize = storageEngineSize - writeSize; long bufferedArraySize = - (long) (config.getAllocateMemoryForStorageEngine() - * config.getBufferedArraysMemoryProportion()); + (long) + (config.getAllocateMemoryForStorageEngine() + * config.getBufferedArraysMemoryProportion()); writeMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -202,7 +204,8 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.LEVEL.toString(), LEVELS[3]); memtableMemorySize.set(memtableSize); - // The memtable memory of storage engine contain DataNodeDevicePathCache (NOTICE: This part of memory is not divided) + // The memtable memory of storage engine contain DataNodeDevicePathCache (NOTICE: This part of + // memory is not divided) long dataNodeDevicePathCacheSize = (long) (config.getAllocateMemoryForStorageEngine() @@ -302,7 +305,10 @@ public void unbindFrom(AbstractMetricService metricService) { memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; bufferedArraysMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS) + Arrays.asList( + STORAGE_ENGINE_WRITE_MEMTABLE, + STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, + STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS) .forEach( name -> metricService.remove( From 79e7a9e47afd0dc7e73bffac0a5e41a523af6f59 Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 13:28:22 +0800 Subject: [PATCH 13/33] fix name bug --- .../iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index 2c261727bd55..ce836d6c79ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -124,7 +124,7 @@ public void bindTo(AbstractMetricService metricService) { Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - SCHEMA_ENGINE, + STREAM_ENGINE, Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), From 2a4c27528314bed556ce9d73bf6c6d5421ff734d Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 30 Dec 2024 19:13:48 +0800 Subject: [PATCH 14/33] move buffer pool to memtable --- .../memory/ThresholdMemoryMetrics.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index ce836d6c79ff..580a56351ec0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -43,8 +43,8 @@ public class ThresholdMemoryMetrics implements IMetricSet { private Gauge writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge dataNodeDevicePathCacheMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge bufferedArraysMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; + private Gauge timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge queryEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; private Gauge schemaEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; @@ -60,10 +60,10 @@ public class ThresholdMemoryMetrics implements IMetricSet { private static final String STORAGE_ENGINE_WRITE_MEMTABLE = "StorageEngine-Write-Memtable"; private static final String STORAGE_ENGINE_WRITE_MEMTABLE_CACHE = "StorageEngine-Write-Memtable-DevicePathCache"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS = + "StorageEngine-Write-Memtable-BufferedArrays"; private static final String STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO = "StorageEngine-Write-TimePartitionInfo"; - private static final String STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS = - "StorageEngine-Write-BufferedArrays"; private static final String STORAGE_ENGINE_COMPACTION = "StorageEngine-Compaction"; private static final String QUERY_ENGINE = "QueryEngine"; private static final String SCHEMA_ENGINE = "SchemaEngine"; @@ -222,28 +222,28 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.LEVEL.toString(), LEVELS[4]); dataNodeDevicePathCacheMemorySize.set(dataNodeDevicePathCacheSize); - timePartitionInfoMemorySize = + bufferedArraysMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, + STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS, Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[3]); - timePartitionInfoMemorySize.set(timePartitionInfoSize); - bufferedArraysMemorySize = + LEVELS[4]); + bufferedArraysMemorySize.set(bufferedArraySize); + timePartitionInfoMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), - STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS, + STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), LEVELS[3]); - bufferedArraysMemorySize.set(bufferedArraySize); + timePartitionInfoMemorySize.set(timePartitionInfoSize); } @Override @@ -288,6 +288,10 @@ public void unbindFrom(AbstractMetricService metricService) { OFF_HEAP, Tag.LEVEL.toString(), LEVELS[1])); + unbindStorageEngineRelatedMemoryMetrics(metricService); + } + + private void unbindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricService) { writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; Arrays.asList(STORAGE_ENGINE_WRITE, STORAGE_ENGINE_COMPACTION) @@ -305,10 +309,7 @@ public void unbindFrom(AbstractMetricService metricService) { memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; bufferedArraysMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - Arrays.asList( - STORAGE_ENGINE_WRITE_MEMTABLE, - STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, - STORAGE_ENGINE_WRITE_BUFFERED_ARRAYS) + Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO) .forEach( name -> metricService.remove( @@ -321,7 +322,8 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.LEVEL.toString(), LEVELS[3])); dataNodeDevicePathCacheMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - Collections.singletonList(STORAGE_ENGINE_WRITE_MEMTABLE_CACHE) + Arrays.asList( + STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS) .forEach( name -> metricService.remove( From 8fc262d10177011af17afc19a9246195355c8217 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 2 Jan 2025 18:39:08 +0800 Subject: [PATCH 15/33] fix comment --- .../db/service/metrics/memory/ThresholdMemoryMetrics.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index 580a56351ec0..e8f67b13d989 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -157,7 +157,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.LEVEL.toString(), LEVELS[1]); storageEngineMemorySize.set(storageEngineSize); - // The memory of storage engine divided into Write and Compaction 2 part + // The memory of storage engine divided into Write and Compaction long writeSize = (long) (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); @@ -188,7 +188,7 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.LEVEL.toString(), LEVELS[2]); compactionMemorySize.set(compactionSize); - // The write memory of storage engine divided into MemTable and TimePartitionInfo 2 parts + // The write memory of storage engine divided into MemTable and TimePartitionInfo long memtableSize = (long) (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable()); From 2de1a03657306e38c6716e3cc82bedb72e347dc4 Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 3 Jan 2025 13:18:44 +0800 Subject: [PATCH 16/33] optimize the code --- .../memory/ThresholdMemoryMetrics.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index e8f67b13d989..1eac90e2e6d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -162,10 +162,6 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS (long) (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); long compactionSize = storageEngineSize - writeSize; - long bufferedArraySize = - (long) - (config.getAllocateMemoryForStorageEngine() - * config.getBufferedArraysMemoryProportion()); writeMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -204,6 +200,17 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.LEVEL.toString(), LEVELS[3]); memtableMemorySize.set(memtableSize); + timePartitionInfoMemorySize = + metricService.getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[3]); + timePartitionInfoMemorySize.set(timePartitionInfoSize); // The memtable memory of storage engine contain DataNodeDevicePathCache (NOTICE: This part of // memory is not divided) long dataNodeDevicePathCacheSize = @@ -211,6 +218,10 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable() * config.getDevicePathCacheProportion()); + long bufferedArraySize = + (long) + (config.getAllocateMemoryForStorageEngine() + * config.getBufferedArraysMemoryProportion()); dataNodeDevicePathCacheMemorySize = metricService.getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -233,17 +244,6 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.LEVEL.toString(), LEVELS[4]); bufferedArraysMemorySize.set(bufferedArraySize); - timePartitionInfoMemorySize = - metricService.getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[3]); - timePartitionInfoMemorySize.set(timePartitionInfoSize); } @Override From 4f1db4ece0a59ade8e9fec527902213d9c2e6365 Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 3 Jan 2025 14:01:31 +0800 Subject: [PATCH 17/33] add Query Memory Threshold(level=2) --- .../memory/ThresholdMemoryMetrics.java | 264 ++++++++++++------ 1 file changed, 175 insertions(+), 89 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index 1eac90e2e6d2..0bb15fc1ca17 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -25,9 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.metrics.AbstractMetricService; -import org.apache.iotdb.metrics.impl.DoNothingMetricManager; import org.apache.iotdb.metrics.metricsets.IMetricSet; -import org.apache.iotdb.metrics.type.Gauge; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.MetricType; @@ -38,20 +36,6 @@ public class ThresholdMemoryMetrics implements IMetricSet { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final SystemInfo systemInfo = SystemInfo.getInstance(); - private Gauge totalMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge storageEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge dataNodeDevicePathCacheMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge bufferedArraysMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge queryEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge schemaEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge consensusMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge streamEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private Gauge directBufferMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - private static final String TOTAL = "Total"; private static final String ON_HEAP = "OnHeap"; private static final String OFF_HEAP = "OffHeap"; @@ -66,6 +50,14 @@ public class ThresholdMemoryMetrics implements IMetricSet { "StorageEngine-Write-TimePartitionInfo"; private static final String STORAGE_ENGINE_COMPACTION = "StorageEngine-Compaction"; private static final String QUERY_ENGINE = "QueryEngine"; + private static final String QUERY_ENGINE_BLOOM_FILTER_CACHE = "QueryEngine-BloomFilterCache"; + private static final String QUERY_ENGINE_CHUNK_CACHE = "QueryEngine-ChunkCache"; + private static final String QUERY_ENGINE_TIME_SERIES_METADATA_CACHE = + "QueryEngine-TimeSeriesMetadataCache"; + private static final String QUERY_ENGINE_OPERATORS = "QueryEngine-Operators"; + private static final String QUERY_ENGINE_DATA_EXCHANGE = "QueryEngine-DataExchange"; + private static final String QUERY_ENGINE_TIME_INDEX = "QueryEngine-TimeIndex"; + private static final String QUERY_ENGINE_COORDINATOR = "QueryEngine-Coordinator"; private static final String SCHEMA_ENGINE = "SchemaEngine"; private static final String CONSENSUS = "Consensus"; private static final String STREAM_ENGINE = "StreamEngine"; @@ -74,8 +66,8 @@ public class ThresholdMemoryMetrics implements IMetricSet { @Override public void bindTo(AbstractMetricService metricService) { - totalMemorySize = - metricService.getOrCreateGauge( + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -83,22 +75,12 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[0]); - totalMemorySize.set(Runtime.getRuntime().maxMemory()); + LEVELS[0]) + .set(Runtime.getRuntime().maxMemory()); bindStorageEngineRelatedMemoryMetrics(metricService); - queryEngineMemorySize = - metricService.getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]); - queryEngineMemorySize.set(config.getAllocateMemoryForRead()); - schemaEngineMemorySize = - metricService.getOrCreateGauge( + bindQueryEngineMemoryMetrics(metricService); + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -106,10 +88,10 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[1]); - schemaEngineMemorySize.set(config.getAllocateMemoryForSchema()); - consensusMemorySize = - metricService.getOrCreateGauge( + LEVELS[1]) + .set(config.getAllocateMemoryForSchema()); + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -117,10 +99,10 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[1]); - consensusMemorySize.set(config.getAllocateMemoryForConsensus()); - streamEngineMemorySize = - metricService.getOrCreateGauge( + LEVELS[1]) + .set(config.getAllocateMemoryForConsensus()); + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -128,10 +110,10 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[1]); - streamEngineMemorySize.set(config.getAllocateMemoryForPipe()); - directBufferMemorySize = - metricService.getOrCreateGauge( + LEVELS[1]) + .set(config.getAllocateMemoryForPipe()); + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -139,15 +121,16 @@ public void bindTo(AbstractMetricService metricService) { Tag.TYPE.toString(), OFF_HEAP, Tag.LEVEL.toString(), - LEVELS[1]); - directBufferMemorySize.set(systemInfo.getTotalDirectBufferMemorySizeLimit()); + LEVELS[1]) + .set(systemInfo.getTotalDirectBufferMemorySizeLimit()); } + /** Bind the memory threshold metrics of storage engine */ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricService) { long storageEngineSize = config.getAllocateMemoryForStorageEngine(); // Total memory size of storage engine - storageEngineMemorySize = - metricService.getOrCreateGauge( + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -155,15 +138,15 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[1]); - storageEngineMemorySize.set(storageEngineSize); + LEVELS[1]) + .set(storageEngineSize); // The memory of storage engine divided into Write and Compaction long writeSize = (long) (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); long compactionSize = storageEngineSize - writeSize; - writeMemorySize = - metricService.getOrCreateGauge( + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -171,10 +154,10 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[2]); - writeMemorySize.set(writeSize); - compactionMemorySize = - metricService.getOrCreateGauge( + LEVELS[2]) + .set(writeSize); + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -182,15 +165,15 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[2]); - compactionMemorySize.set(compactionSize); + LEVELS[2]) + .set(compactionSize); // The write memory of storage engine divided into MemTable and TimePartitionInfo long memtableSize = (long) (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable()); long timePartitionInfoSize = config.getAllocateMemoryForTimePartitionInfo(); - memtableMemorySize = - metricService.getOrCreateGauge( + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -198,10 +181,10 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[3]); - memtableMemorySize.set(memtableSize); - timePartitionInfoMemorySize = - metricService.getOrCreateGauge( + LEVELS[3]) + .set(memtableSize); + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -209,8 +192,8 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[3]); - timePartitionInfoMemorySize.set(timePartitionInfoSize); + LEVELS[3]) + .set(timePartitionInfoSize); // The memtable memory of storage engine contain DataNodeDevicePathCache (NOTICE: This part of // memory is not divided) long dataNodeDevicePathCacheSize = @@ -222,8 +205,8 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS (long) (config.getAllocateMemoryForStorageEngine() * config.getBufferedArraysMemoryProportion()); - dataNodeDevicePathCacheMemorySize = - metricService.getOrCreateGauge( + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -231,10 +214,10 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[4]); - dataNodeDevicePathCacheMemorySize.set(dataNodeDevicePathCacheSize); - bufferedArraysMemorySize = - metricService.getOrCreateGauge( + LEVELS[4]) + .set(dataNodeDevicePathCacheSize); + metricService + .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), @@ -242,13 +225,104 @@ private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricS Tag.TYPE.toString(), ON_HEAP, Tag.LEVEL.toString(), - LEVELS[4]); - bufferedArraysMemorySize.set(bufferedArraySize); + LEVELS[4]) + .set(bufferedArraySize); + } + + /** Bind the memory threshold metrics of query engine */ + private void bindQueryEngineMemoryMetrics(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[1]) + .set(config.getAllocateMemoryForRead()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_BLOOM_FILTER_CACHE, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForBloomFilterCache()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_CHUNK_CACHE, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForChunkCache()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForTimeSeriesMetaDataCache()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_OPERATORS, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForOperators()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_DATA_EXCHANGE, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForDataExchange()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_TIME_INDEX, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForTimeIndex()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_COORDINATOR, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForCoordinator()); } @Override public void unbindFrom(AbstractMetricService metricService) { - totalMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; metricService.remove( MetricType.GAUGE, Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -258,12 +332,6 @@ public void unbindFrom(AbstractMetricService metricService) { ON_HEAP, Tag.LEVEL.toString(), "0"); - storageEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - queryEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - schemaEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - consensusMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - streamEngineMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - directBufferMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; Arrays.asList(STORAGE_ENGINE, QUERY_ENGINE, SCHEMA_ENGINE, CONSENSUS, STREAM_ENGINE) .forEach( name -> @@ -276,6 +344,7 @@ public void unbindFrom(AbstractMetricService metricService) { ON_HEAP, Tag.LEVEL.toString(), LEVELS[1])); + unbindStorageEngineRelatedMemoryMetrics(metricService); Collections.singletonList(DIRECT_BUFFER) .forEach( name -> @@ -288,12 +357,10 @@ public void unbindFrom(AbstractMetricService metricService) { OFF_HEAP, Tag.LEVEL.toString(), LEVELS[1])); - unbindStorageEngineRelatedMemoryMetrics(metricService); } + /** Unbind the memory threshold metrics of storage engine */ private void unbindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricService) { - writeMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - compactionMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; Arrays.asList(STORAGE_ENGINE_WRITE, STORAGE_ENGINE_COMPACTION) .forEach( name -> @@ -306,9 +373,6 @@ private void unbindStorageEngineRelatedMemoryMetrics(AbstractMetricService metri ON_HEAP, Tag.LEVEL.toString(), LEVELS[2])); - memtableMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - timePartitionInfoMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; - bufferedArraysMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO) .forEach( name -> @@ -321,7 +385,6 @@ private void unbindStorageEngineRelatedMemoryMetrics(AbstractMetricService metri ON_HEAP, Tag.LEVEL.toString(), LEVELS[3])); - dataNodeDevicePathCacheMemorySize = DoNothingMetricManager.DO_NOTHING_GAUGE; Arrays.asList( STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS) .forEach( @@ -337,6 +400,29 @@ private void unbindStorageEngineRelatedMemoryMetrics(AbstractMetricService metri LEVELS[4])); } + /** Unbind the memory threshold metrics of query engine */ + private void unbindQueryEngineMemoryMetrics(AbstractMetricService metricService) { + Arrays.asList( + QUERY_ENGINE_BLOOM_FILTER_CACHE, + QUERY_ENGINE_CHUNK_CACHE, + QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, + QUERY_ENGINE_OPERATORS, + QUERY_ENGINE_DATA_EXCHANGE, + QUERY_ENGINE_TIME_INDEX, + QUERY_ENGINE_COORDINATOR) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2])); + } + public static ThresholdMemoryMetrics getInstance() { return ThresholdMemoryMetrics.ThresholdMemoryMetricsHolder.INSTANCE; } From 3fa60883605d61ea0e9bb5ea938eaa55bfe357b4 Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 3 Jan 2025 14:34:50 +0800 Subject: [PATCH 18/33] add Schema Memory Threshold(level=2) --- .../memory/ThresholdMemoryMetrics.java | 80 ++++++++++++++++--- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index 0bb15fc1ca17..ed7a19bdb889 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -59,6 +59,9 @@ public class ThresholdMemoryMetrics implements IMetricSet { private static final String QUERY_ENGINE_TIME_INDEX = "QueryEngine-TimeIndex"; private static final String QUERY_ENGINE_COORDINATOR = "QueryEngine-Coordinator"; private static final String SCHEMA_ENGINE = "SchemaEngine"; + private static final String SCHEMA_ENGINE_SCHEMA_REGION = "SchemaEngine-SchemaRegion"; + private static final String SCHEMA_ENGINE_SCHEMA_CACHE = "SchemaEngine-SchemaCache"; + private static final String SCHEMA_ENGINE_PARTITION_CACHE = "SchemaEngine-PartitionCache"; private static final String CONSENSUS = "Consensus"; private static final String STREAM_ENGINE = "StreamEngine"; private static final String DIRECT_BUFFER = "DirectBuffer"; @@ -79,17 +82,7 @@ public void bindTo(AbstractMetricService metricService) { .set(Runtime.getRuntime().maxMemory()); bindStorageEngineRelatedMemoryMetrics(metricService); bindQueryEngineMemoryMetrics(metricService); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - SCHEMA_ENGINE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]) - .set(config.getAllocateMemoryForSchema()); + bindSchemaEngineMemoryMetrics(metricService); metricService .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -321,6 +314,53 @@ private void bindQueryEngineMemoryMetrics(AbstractMetricService metricService) { .set(config.getAllocateMemoryForCoordinator()); } + private void bindSchemaEngineMemoryMetrics(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[1]) + .set(config.getAllocateMemoryForSchema()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_SCHEMA_REGION, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForSchemaRegion()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_SCHEMA_CACHE, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForSchemaCache()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_PARTITION_CACHE, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2]) + .set(config.getAllocateMemoryForPartitionCache()); + } + @Override public void unbindFrom(AbstractMetricService metricService) { metricService.remove( @@ -345,6 +385,8 @@ public void unbindFrom(AbstractMetricService metricService) { Tag.LEVEL.toString(), LEVELS[1])); unbindStorageEngineRelatedMemoryMetrics(metricService); + unbindQueryEngineMemoryMetrics(metricService); + unbindSchemaEngineMemoryMetrics(metricService); Collections.singletonList(DIRECT_BUFFER) .forEach( name -> @@ -423,6 +465,22 @@ private void unbindQueryEngineMemoryMetrics(AbstractMetricService metricService) LEVELS[2])); } + private void unbindSchemaEngineMemoryMetrics(AbstractMetricService metricService) { + Arrays.asList( + SCHEMA_ENGINE_SCHEMA_REGION, SCHEMA_ENGINE_SCHEMA_CACHE, SCHEMA_ENGINE_PARTITION_CACHE) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[2])); + } + public static ThresholdMemoryMetrics getInstance() { return ThresholdMemoryMetrics.ThresholdMemoryMetricsHolder.INSTANCE; } From e0ce0af68245900e5557992bfd7c82ed45172afe Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 3 Jan 2025 14:51:14 +0800 Subject: [PATCH 19/33] add Schema Memory Threshold(update comment) --- .../iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index ed7a19bdb889..fdecdf4e6fd4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -314,6 +314,7 @@ private void bindQueryEngineMemoryMetrics(AbstractMetricService metricService) { .set(config.getAllocateMemoryForCoordinator()); } + /** Bind the memory threshold metrics of schema engine */ private void bindSchemaEngineMemoryMetrics(AbstractMetricService metricService) { metricService .getOrCreateGauge( From 468e1c9eb67b04b402e8c2df9fba16a37b29f798 Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 3 Jan 2025 15:57:31 +0800 Subject: [PATCH 20/33] move Consensus memory threshold --- .../memory/ThresholdMemoryMetrics.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index fdecdf4e6fd4..be3e96efc04d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -83,6 +83,7 @@ public void bindTo(AbstractMetricService metricService) { bindStorageEngineRelatedMemoryMetrics(metricService); bindQueryEngineMemoryMetrics(metricService); bindSchemaEngineMemoryMetrics(metricService); + bindConsensusMemoryMetrics(metricService); metricService .getOrCreateGauge( Metric.THRESHOLD_MEMORY_SIZE.toString(), @@ -362,6 +363,21 @@ private void bindSchemaEngineMemoryMetrics(AbstractMetricService metricService) .set(config.getAllocateMemoryForPartitionCache()); } + /** Bind the memory threshold metrics of consensus(NOTICE: iot consensus only) */ + private void bindConsensusMemoryMetrics(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + CONSENSUS, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[1]) + .set(config.getAllocateMemoryForConsensus()); + } + @Override public void unbindFrom(AbstractMetricService metricService) { metricService.remove( @@ -388,6 +404,7 @@ public void unbindFrom(AbstractMetricService metricService) { unbindStorageEngineRelatedMemoryMetrics(metricService); unbindQueryEngineMemoryMetrics(metricService); unbindSchemaEngineMemoryMetrics(metricService); + unbindConsensusMemoryMetrics(metricService); Collections.singletonList(DIRECT_BUFFER) .forEach( name -> @@ -482,6 +499,10 @@ private void unbindSchemaEngineMemoryMetrics(AbstractMetricService metricService LEVELS[2])); } + private void unbindConsensusMemoryMetrics(AbstractMetricService metricService) { + + } + public static ThresholdMemoryMetrics getInstance() { return ThresholdMemoryMetrics.ThresholdMemoryMetricsHolder.INSTANCE; } From d26cc05b3152f442f3825cf671ec160eec9af85a Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 3 Jan 2025 15:58:49 +0800 Subject: [PATCH 21/33] move Consensus memory threshold --- .../db/service/metrics/memory/ThresholdMemoryMetrics.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java index be3e96efc04d..b7e6156d93e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java @@ -499,10 +499,6 @@ private void unbindSchemaEngineMemoryMetrics(AbstractMetricService metricService LEVELS[2])); } - private void unbindConsensusMemoryMetrics(AbstractMetricService metricService) { - - } - public static ThresholdMemoryMetrics getInstance() { return ThresholdMemoryMetrics.ThresholdMemoryMetricsHolder.INSTANCE; } From 3273bffdbef544e09aa28891257b1b5325ee89d2 Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 3 Jan 2025 16:35:04 +0800 Subject: [PATCH 22/33] split class to optimize code --- .../metrics/DataNodeMetricsHelper.java | 4 +- .../memory/ConsensusMemoryMetrics.java | 73 +++ .../metrics/memory/GlobalMemoryMetrics.java | 116 ++++ .../memory/QueryEngineMemoryMetrics.java | 179 ++++++ .../memory/SchemaEngineMemoryMetrics.java | 124 +++++ .../memory/StorageEngineMemoryMetrics.java | 210 +++++++ .../memory/StreamEngineMemoryMetrics.java | 73 +++ .../memory/ThresholdMemoryMetrics.java | 512 ------------------ 8 files changed, 777 insertions(+), 514 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java index dda0a403bc52..8ac2fa1fde4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java @@ -39,7 +39,7 @@ import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; -import org.apache.iotdb.db.service.metrics.memory.ThresholdMemoryMetrics; +import org.apache.iotdb.db.service.metrics.memory.GlobalMemoryMetrics; import org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesNumberMetricsSet; import org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesSizeMetricsSet; import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet; @@ -105,7 +105,7 @@ public static void bind() { metricService.addMetricSet(ActiveLoadingFilesSizeMetricsSet.getInstance()); // bind memory related metrics - metricService.addMetricSet(ThresholdMemoryMetrics.getInstance()); + metricService.addMetricSet(GlobalMemoryMetrics.getInstance()); } private static void initSystemMetrics(MetricService metricService) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java new file mode 100644 index 000000000000..c4d331cd7160 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +public class ConsensusMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String CONSENSUS = "Consensus"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + CONSENSUS, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(config.getAllocateMemoryForConsensus()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + CONSENSUS, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + } + + public static ConsensusMemoryMetrics getInstance() { + return ConsensusMemoryMetricsHolder.INSTANCE; + } + + private static class ConsensusMemoryMetricsHolder { + + private static final ConsensusMemoryMetrics INSTANCE = new ConsensusMemoryMetrics(); + + private ConsensusMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java new file mode 100644 index 000000000000..714974ca16f7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Collections; + +public class GlobalMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final SystemInfo systemInfo = SystemInfo.getInstance(); + + private static final String TOTAL = "Total"; + public static final String ON_HEAP = "OnHeap"; + public static final String OFF_HEAP = "OffHeap"; + public static final String[] LEVELS = {"0", "1", "2", "3", "4"}; + + private static final String DIRECT_BUFFER = "DirectBuffer"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + TOTAL, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[0]) + .set(Runtime.getRuntime().maxMemory()); + StorageEngineMemoryMetrics.getInstance().bindTo(metricService); + QueryEngineMemoryMetrics.getInstance().bindTo(metricService); + SchemaEngineMemoryMetrics.getInstance().bindTo(metricService); + ConsensusMemoryMetrics.getInstance().bindTo(metricService); + StreamEngineMemoryMetrics.getInstance().bindTo(metricService); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + DIRECT_BUFFER, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + LEVELS[1]) + .set(systemInfo.getTotalDirectBufferMemorySizeLimit()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + TOTAL, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + "0"); + StorageEngineMemoryMetrics.getInstance().unbindFrom(metricService); + QueryEngineMemoryMetrics.getInstance().unbindFrom(metricService); + SchemaEngineMemoryMetrics.getInstance().unbindFrom(metricService); + ConsensusMemoryMetrics.getInstance().unbindFrom(metricService); + StreamEngineMemoryMetrics.getInstance().unbindFrom(metricService); + Collections.singletonList(DIRECT_BUFFER) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + LEVELS[1])); + } + + public static GlobalMemoryMetrics getInstance() { + return GlobalMemoryMetricsHolder.INSTANCE; + } + + private static class GlobalMemoryMetricsHolder { + + private static final GlobalMemoryMetrics INSTANCE = new GlobalMemoryMetrics(); + + private GlobalMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java new file mode 100644 index 000000000000..dedd9cbc1caf --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Arrays; + +public class QueryEngineMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String QUERY_ENGINE = "QueryEngine"; + private static final String QUERY_ENGINE_BLOOM_FILTER_CACHE = "QueryEngine-BloomFilterCache"; + private static final String QUERY_ENGINE_CHUNK_CACHE = "QueryEngine-ChunkCache"; + private static final String QUERY_ENGINE_TIME_SERIES_METADATA_CACHE = + "QueryEngine-TimeSeriesMetadataCache"; + private static final String QUERY_ENGINE_OPERATORS = "QueryEngine-Operators"; + private static final String QUERY_ENGINE_DATA_EXCHANGE = "QueryEngine-DataExchange"; + private static final String QUERY_ENGINE_TIME_INDEX = "QueryEngine-TimeIndex"; + private static final String QUERY_ENGINE_COORDINATOR = "QueryEngine-Coordinator"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(config.getAllocateMemoryForRead()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_BLOOM_FILTER_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForBloomFilterCache()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_CHUNK_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForChunkCache()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForTimeSeriesMetaDataCache()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_OPERATORS, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForOperators()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_DATA_EXCHANGE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForDataExchange()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_TIME_INDEX, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForTimeIndex()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_COORDINATOR, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForCoordinator()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + QUERY_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + Arrays.asList( + QUERY_ENGINE_BLOOM_FILTER_CACHE, + QUERY_ENGINE_CHUNK_CACHE, + QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, + QUERY_ENGINE_OPERATORS, + QUERY_ENGINE_DATA_EXCHANGE, + QUERY_ENGINE_TIME_INDEX, + QUERY_ENGINE_COORDINATOR) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2])); + } + + public static QueryEngineMemoryMetrics getInstance() { + return QueryEngineMemoryMetricsHolder.INSTANCE; + } + + private static class QueryEngineMemoryMetricsHolder { + + private static final QueryEngineMemoryMetrics INSTANCE = new QueryEngineMemoryMetrics(); + + private QueryEngineMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java new file mode 100644 index 000000000000..dc7e34cff44b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Arrays; + +public class SchemaEngineMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String SCHEMA_ENGINE = "SchemaEngine"; + private static final String SCHEMA_ENGINE_SCHEMA_REGION = "SchemaEngine-SchemaRegion"; + private static final String SCHEMA_ENGINE_SCHEMA_CACHE = "SchemaEngine-SchemaCache"; + private static final String SCHEMA_ENGINE_PARTITION_CACHE = "SchemaEngine-PartitionCache"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(config.getAllocateMemoryForSchema()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_SCHEMA_REGION, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForSchemaRegion()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_SCHEMA_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForSchemaCache()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_PARTITION_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForPartitionCache()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + SCHEMA_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + Arrays.asList( + SCHEMA_ENGINE_SCHEMA_REGION, SCHEMA_ENGINE_SCHEMA_CACHE, SCHEMA_ENGINE_PARTITION_CACHE) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2])); + } + + public static SchemaEngineMemoryMetrics getInstance() { + return SchemaEngineMemoryMetricsHolder.INSTANCE; + } + + private static class SchemaEngineMemoryMetricsHolder { + + private static final SchemaEngineMemoryMetrics INSTANCE = new SchemaEngineMemoryMetrics(); + + private SchemaEngineMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java new file mode 100644 index 000000000000..996cb4784017 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Arrays; + +public class StorageEngineMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String STORAGE_ENGINE = "StorageEngine"; + private static final String STORAGE_ENGINE_WRITE = "StorageEngine-Write"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE = "StorageEngine-Write-Memtable"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE_CACHE = + "StorageEngine-Write-Memtable-DevicePathCache"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS = + "StorageEngine-Write-Memtable-BufferedArrays"; + private static final String STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO = + "StorageEngine-Write-TimePartitionInfo"; + private static final String STORAGE_ENGINE_COMPACTION = "StorageEngine-Compaction"; + + @Override + public void bindTo(AbstractMetricService metricService) { + long storageEngineSize = config.getAllocateMemoryForStorageEngine(); + // Total memory size of storage engine + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(storageEngineSize); + // The memory of storage engine divided into Write and Compaction + long writeSize = + (long) + (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); + long compactionSize = storageEngineSize - writeSize; + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(writeSize); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_COMPACTION, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(compactionSize); + // The write memory of storage engine divided into MemTable and TimePartitionInfo + long memtableSize = + (long) + (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable()); + long timePartitionInfoSize = config.getAllocateMemoryForTimePartitionInfo(); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_MEMTABLE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[3]) + .set(memtableSize); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[3]) + .set(timePartitionInfoSize); + // The memtable memory of storage engine contain DataNodeDevicePathCache (NOTICE: This part of + // memory is not divided) + long dataNodeDevicePathCacheSize = + (long) + (config.getAllocateMemoryForStorageEngine() + * config.getWriteProportionForMemtable() + * config.getDevicePathCacheProportion()); + long bufferedArraySize = + (long) + (config.getAllocateMemoryForStorageEngine() + * config.getBufferedArraysMemoryProportion()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[4]) + .set(dataNodeDevicePathCacheSize); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[4]) + .set(bufferedArraySize); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + STORAGE_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + Arrays.asList(STORAGE_ENGINE_WRITE, STORAGE_ENGINE_COMPACTION) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2])); + Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[3])); + Arrays.asList( + STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[4])); + } + + public static StorageEngineMemoryMetrics getInstance() { + return StorageEngineMemoryMetricsHolder.INSTANCE; + } + + private static class StorageEngineMemoryMetricsHolder { + + private static final StorageEngineMemoryMetrics INSTANCE = new StorageEngineMemoryMetrics(); + + private StorageEngineMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java new file mode 100644 index 000000000000..7a2accbabca9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +public class StreamEngineMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String STREAM_ENGINE = "StreamEngine"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STREAM_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(config.getAllocateMemoryForPipe()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + STREAM_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + } + + public static StreamEngineMemoryMetrics getInstance() { + return StreamEngineMemoryMetricsHolder.INSTANCE; + } + + private static class StreamEngineMemoryMetricsHolder { + + private static final StreamEngineMemoryMetrics INSTANCE = new StreamEngineMemoryMetrics(); + + private StreamEngineMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java deleted file mode 100644 index b7e6156d93e2..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ /dev/null @@ -1,512 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.service.metrics.memory; - -import org.apache.iotdb.commons.service.metric.enums.Metric; -import org.apache.iotdb.commons.service.metric.enums.Tag; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; -import org.apache.iotdb.metrics.AbstractMetricService; -import org.apache.iotdb.metrics.metricsets.IMetricSet; -import org.apache.iotdb.metrics.utils.MetricLevel; -import org.apache.iotdb.metrics.utils.MetricType; - -import java.util.Arrays; -import java.util.Collections; - -public class ThresholdMemoryMetrics implements IMetricSet { - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private static final SystemInfo systemInfo = SystemInfo.getInstance(); - - private static final String TOTAL = "Total"; - private static final String ON_HEAP = "OnHeap"; - private static final String OFF_HEAP = "OffHeap"; - private static final String STORAGE_ENGINE = "StorageEngine"; - private static final String STORAGE_ENGINE_WRITE = "StorageEngine-Write"; - private static final String STORAGE_ENGINE_WRITE_MEMTABLE = "StorageEngine-Write-Memtable"; - private static final String STORAGE_ENGINE_WRITE_MEMTABLE_CACHE = - "StorageEngine-Write-Memtable-DevicePathCache"; - private static final String STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS = - "StorageEngine-Write-Memtable-BufferedArrays"; - private static final String STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO = - "StorageEngine-Write-TimePartitionInfo"; - private static final String STORAGE_ENGINE_COMPACTION = "StorageEngine-Compaction"; - private static final String QUERY_ENGINE = "QueryEngine"; - private static final String QUERY_ENGINE_BLOOM_FILTER_CACHE = "QueryEngine-BloomFilterCache"; - private static final String QUERY_ENGINE_CHUNK_CACHE = "QueryEngine-ChunkCache"; - private static final String QUERY_ENGINE_TIME_SERIES_METADATA_CACHE = - "QueryEngine-TimeSeriesMetadataCache"; - private static final String QUERY_ENGINE_OPERATORS = "QueryEngine-Operators"; - private static final String QUERY_ENGINE_DATA_EXCHANGE = "QueryEngine-DataExchange"; - private static final String QUERY_ENGINE_TIME_INDEX = "QueryEngine-TimeIndex"; - private static final String QUERY_ENGINE_COORDINATOR = "QueryEngine-Coordinator"; - private static final String SCHEMA_ENGINE = "SchemaEngine"; - private static final String SCHEMA_ENGINE_SCHEMA_REGION = "SchemaEngine-SchemaRegion"; - private static final String SCHEMA_ENGINE_SCHEMA_CACHE = "SchemaEngine-SchemaCache"; - private static final String SCHEMA_ENGINE_PARTITION_CACHE = "SchemaEngine-PartitionCache"; - private static final String CONSENSUS = "Consensus"; - private static final String STREAM_ENGINE = "StreamEngine"; - private static final String DIRECT_BUFFER = "DirectBuffer"; - private static final String[] LEVELS = {"0", "1", "2", "3", "4"}; - - @Override - public void bindTo(AbstractMetricService metricService) { - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - TOTAL, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[0]) - .set(Runtime.getRuntime().maxMemory()); - bindStorageEngineRelatedMemoryMetrics(metricService); - bindQueryEngineMemoryMetrics(metricService); - bindSchemaEngineMemoryMetrics(metricService); - bindConsensusMemoryMetrics(metricService); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - CONSENSUS, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]) - .set(config.getAllocateMemoryForConsensus()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STREAM_ENGINE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]) - .set(config.getAllocateMemoryForPipe()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - DIRECT_BUFFER, - Tag.TYPE.toString(), - OFF_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]) - .set(systemInfo.getTotalDirectBufferMemorySizeLimit()); - } - - /** Bind the memory threshold metrics of storage engine */ - private void bindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricService) { - long storageEngineSize = config.getAllocateMemoryForStorageEngine(); - // Total memory size of storage engine - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STORAGE_ENGINE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]) - .set(storageEngineSize); - // The memory of storage engine divided into Write and Compaction - long writeSize = - (long) - (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); - long compactionSize = storageEngineSize - writeSize; - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STORAGE_ENGINE_WRITE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(writeSize); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STORAGE_ENGINE_COMPACTION, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(compactionSize); - // The write memory of storage engine divided into MemTable and TimePartitionInfo - long memtableSize = - (long) - (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable()); - long timePartitionInfoSize = config.getAllocateMemoryForTimePartitionInfo(); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STORAGE_ENGINE_WRITE_MEMTABLE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[3]) - .set(memtableSize); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[3]) - .set(timePartitionInfoSize); - // The memtable memory of storage engine contain DataNodeDevicePathCache (NOTICE: This part of - // memory is not divided) - long dataNodeDevicePathCacheSize = - (long) - (config.getAllocateMemoryForStorageEngine() - * config.getWriteProportionForMemtable() - * config.getDevicePathCacheProportion()); - long bufferedArraySize = - (long) - (config.getAllocateMemoryForStorageEngine() - * config.getBufferedArraysMemoryProportion()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[4]) - .set(dataNodeDevicePathCacheSize); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[4]) - .set(bufferedArraySize); - } - - /** Bind the memory threshold metrics of query engine */ - private void bindQueryEngineMemoryMetrics(AbstractMetricService metricService) { - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]) - .set(config.getAllocateMemoryForRead()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE_BLOOM_FILTER_CACHE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForBloomFilterCache()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE_CHUNK_CACHE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForChunkCache()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForTimeSeriesMetaDataCache()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE_OPERATORS, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForOperators()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE_DATA_EXCHANGE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForDataExchange()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE_TIME_INDEX, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForTimeIndex()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - QUERY_ENGINE_COORDINATOR, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForCoordinator()); - } - - /** Bind the memory threshold metrics of schema engine */ - private void bindSchemaEngineMemoryMetrics(AbstractMetricService metricService) { - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - SCHEMA_ENGINE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]) - .set(config.getAllocateMemoryForSchema()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - SCHEMA_ENGINE_SCHEMA_REGION, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForSchemaRegion()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - SCHEMA_ENGINE_SCHEMA_CACHE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForSchemaCache()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - SCHEMA_ENGINE_PARTITION_CACHE, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2]) - .set(config.getAllocateMemoryForPartitionCache()); - } - - /** Bind the memory threshold metrics of consensus(NOTICE: iot consensus only) */ - private void bindConsensusMemoryMetrics(AbstractMetricService metricService) { - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - CONSENSUS, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1]) - .set(config.getAllocateMemoryForConsensus()); - } - - @Override - public void unbindFrom(AbstractMetricService metricService) { - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - TOTAL, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - "0"); - Arrays.asList(STORAGE_ENGINE, QUERY_ENGINE, SCHEMA_ENGINE, CONSENSUS, STREAM_ENGINE) - .forEach( - name -> - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - name, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[1])); - unbindStorageEngineRelatedMemoryMetrics(metricService); - unbindQueryEngineMemoryMetrics(metricService); - unbindSchemaEngineMemoryMetrics(metricService); - unbindConsensusMemoryMetrics(metricService); - Collections.singletonList(DIRECT_BUFFER) - .forEach( - name -> - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - name, - Tag.TYPE.toString(), - OFF_HEAP, - Tag.LEVEL.toString(), - LEVELS[1])); - } - - /** Unbind the memory threshold metrics of storage engine */ - private void unbindStorageEngineRelatedMemoryMetrics(AbstractMetricService metricService) { - Arrays.asList(STORAGE_ENGINE_WRITE, STORAGE_ENGINE_COMPACTION) - .forEach( - name -> - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - name, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2])); - Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO) - .forEach( - name -> - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - name, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[3])); - Arrays.asList( - STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS) - .forEach( - name -> - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - name, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[4])); - } - - /** Unbind the memory threshold metrics of query engine */ - private void unbindQueryEngineMemoryMetrics(AbstractMetricService metricService) { - Arrays.asList( - QUERY_ENGINE_BLOOM_FILTER_CACHE, - QUERY_ENGINE_CHUNK_CACHE, - QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, - QUERY_ENGINE_OPERATORS, - QUERY_ENGINE_DATA_EXCHANGE, - QUERY_ENGINE_TIME_INDEX, - QUERY_ENGINE_COORDINATOR) - .forEach( - name -> - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - name, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2])); - } - - private void unbindSchemaEngineMemoryMetrics(AbstractMetricService metricService) { - Arrays.asList( - SCHEMA_ENGINE_SCHEMA_REGION, SCHEMA_ENGINE_SCHEMA_CACHE, SCHEMA_ENGINE_PARTITION_CACHE) - .forEach( - name -> - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - name, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - LEVELS[2])); - } - - public static ThresholdMemoryMetrics getInstance() { - return ThresholdMemoryMetrics.ThresholdMemoryMetricsHolder.INSTANCE; - } - - private static class ThresholdMemoryMetricsHolder { - - private static final ThresholdMemoryMetrics INSTANCE = new ThresholdMemoryMetrics(); - - private ThresholdMemoryMetricsHolder() {} - } -} From 5d17be6f5836f28da5b7e85f540e32e27e26911b Mon Sep 17 00:00:00 2001 From: spricoder Date: Fri, 3 Jan 2025 17:27:50 +0800 Subject: [PATCH 23/33] Add Off Heap Memory Total --- .../metrics/memory/GlobalMemoryMetrics.java | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java index 714974ca16f7..0dbfe9290cd8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java @@ -29,6 +29,7 @@ import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.MetricType; +import java.util.Arrays; import java.util.Collections; public class GlobalMemoryMetrics implements IMetricSet { @@ -55,6 +56,17 @@ public void bindTo(AbstractMetricService metricService) { Tag.LEVEL.toString(), LEVELS[0]) .set(Runtime.getRuntime().maxMemory()); + metricService + .getOrCreateGauge( + Metric.THRESHOLD_MEMORY_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + TOTAL, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + LEVELS[0]) + .set(config.getMaxOffHeapMemoryBytes()); StorageEngineMemoryMetrics.getInstance().bindTo(metricService); QueryEngineMemoryMetrics.getInstance().bindTo(metricService); SchemaEngineMemoryMetrics.getInstance().bindTo(metricService); @@ -75,15 +87,19 @@ public void bindTo(AbstractMetricService metricService) { @Override public void unbindFrom(AbstractMetricService metricService) { - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - TOTAL, - Tag.TYPE.toString(), - ON_HEAP, - Tag.LEVEL.toString(), - "0"); + Arrays.asList(ON_HEAP, OFF_HEAP) + .forEach( + type -> { + metricService.remove( + MetricType.GAUGE, + Metric.THRESHOLD_MEMORY_SIZE.toString(), + Tag.NAME.toString(), + TOTAL, + Tag.TYPE.toString(), + type, + Tag.LEVEL.toString(), + LEVELS[0]); + }); StorageEngineMemoryMetrics.getInstance().unbindFrom(metricService); QueryEngineMemoryMetrics.getInstance().unbindFrom(metricService); SchemaEngineMemoryMetrics.getInstance().unbindFrom(metricService); From 9bf1bc84f9f762012396f8f9e5a7a4b2099966fc Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 6 Jan 2025 17:05:09 +0800 Subject: [PATCH 24/33] change metric name --- .../memory/ConsensusMemoryMetrics.java | 4 ++-- .../metrics/memory/GlobalMemoryMetrics.java | 10 ++++----- .../memory/QueryEngineMemoryMetrics.java | 20 ++++++++--------- .../memory/SchemaEngineMemoryMetrics.java | 12 +++++----- .../memory/StorageEngineMemoryMetrics.java | 22 +++++++++---------- .../memory/StreamEngineMemoryMetrics.java | 4 ++-- .../commons/service/metric/enums/Metric.java | 2 +- 7 files changed, 37 insertions(+), 37 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java index c4d331cd7160..b3f108892e71 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java @@ -36,7 +36,7 @@ public class ConsensusMemoryMetrics implements IMetricSet { public void bindTo(AbstractMetricService metricService) { metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), CONSENSUS, @@ -51,7 +51,7 @@ public void bindTo(AbstractMetricService metricService) { public void unbindFrom(AbstractMetricService metricService) { metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), CONSENSUS, Tag.TYPE.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java index 0dbfe9290cd8..cca3f46f1d88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java @@ -47,7 +47,7 @@ public class GlobalMemoryMetrics implements IMetricSet { public void bindTo(AbstractMetricService metricService) { metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), TOTAL, @@ -58,7 +58,7 @@ public void bindTo(AbstractMetricService metricService) { .set(Runtime.getRuntime().maxMemory()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), TOTAL, @@ -74,7 +74,7 @@ public void bindTo(AbstractMetricService metricService) { StreamEngineMemoryMetrics.getInstance().bindTo(metricService); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), DIRECT_BUFFER, @@ -92,7 +92,7 @@ public void unbindFrom(AbstractMetricService metricService) { type -> { metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), TOTAL, Tag.TYPE.toString(), @@ -110,7 +110,7 @@ public void unbindFrom(AbstractMetricService metricService) { name -> metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), name, Tag.TYPE.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java index dedd9cbc1caf..67538646525a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java @@ -46,7 +46,7 @@ public class QueryEngineMemoryMetrics implements IMetricSet { public void bindTo(AbstractMetricService metricService) { metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), QUERY_ENGINE, @@ -57,7 +57,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForRead()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), QUERY_ENGINE_BLOOM_FILTER_CACHE, @@ -68,7 +68,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForBloomFilterCache()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), QUERY_ENGINE_CHUNK_CACHE, @@ -79,7 +79,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForChunkCache()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, @@ -90,7 +90,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForTimeSeriesMetaDataCache()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), QUERY_ENGINE_OPERATORS, @@ -101,7 +101,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForOperators()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), QUERY_ENGINE_DATA_EXCHANGE, @@ -112,7 +112,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForDataExchange()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), QUERY_ENGINE_TIME_INDEX, @@ -123,7 +123,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForTimeIndex()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), QUERY_ENGINE_COORDINATOR, @@ -138,7 +138,7 @@ public void bindTo(AbstractMetricService metricService) { public void unbindFrom(AbstractMetricService metricService) { metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), QUERY_ENGINE, Tag.TYPE.toString(), @@ -157,7 +157,7 @@ public void unbindFrom(AbstractMetricService metricService) { name -> metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), name, Tag.TYPE.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java index dc7e34cff44b..5b0757263637 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java @@ -41,7 +41,7 @@ public class SchemaEngineMemoryMetrics implements IMetricSet { public void bindTo(AbstractMetricService metricService) { metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), SCHEMA_ENGINE, @@ -52,7 +52,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForSchema()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), SCHEMA_ENGINE_SCHEMA_REGION, @@ -63,7 +63,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForSchemaRegion()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), SCHEMA_ENGINE_SCHEMA_CACHE, @@ -74,7 +74,7 @@ public void bindTo(AbstractMetricService metricService) { .set(config.getAllocateMemoryForSchemaCache()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), SCHEMA_ENGINE_PARTITION_CACHE, @@ -89,7 +89,7 @@ public void bindTo(AbstractMetricService metricService) { public void unbindFrom(AbstractMetricService metricService) { metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), SCHEMA_ENGINE, Tag.TYPE.toString(), @@ -102,7 +102,7 @@ public void unbindFrom(AbstractMetricService metricService) { name -> metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), name, Tag.TYPE.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java index 996cb4784017..2fd726272d53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java @@ -49,7 +49,7 @@ public void bindTo(AbstractMetricService metricService) { // Total memory size of storage engine metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), STORAGE_ENGINE, @@ -65,7 +65,7 @@ public void bindTo(AbstractMetricService metricService) { long compactionSize = storageEngineSize - writeSize; metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), STORAGE_ENGINE_WRITE, @@ -76,7 +76,7 @@ public void bindTo(AbstractMetricService metricService) { .set(writeSize); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), STORAGE_ENGINE_COMPACTION, @@ -92,7 +92,7 @@ public void bindTo(AbstractMetricService metricService) { long timePartitionInfoSize = config.getAllocateMemoryForTimePartitionInfo(); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), STORAGE_ENGINE_WRITE_MEMTABLE, @@ -103,7 +103,7 @@ public void bindTo(AbstractMetricService metricService) { .set(memtableSize); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, @@ -125,7 +125,7 @@ public void bindTo(AbstractMetricService metricService) { * config.getBufferedArraysMemoryProportion()); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, @@ -136,7 +136,7 @@ public void bindTo(AbstractMetricService metricService) { .set(dataNodeDevicePathCacheSize); metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS, @@ -151,7 +151,7 @@ public void bindTo(AbstractMetricService metricService) { public void unbindFrom(AbstractMetricService metricService) { metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), STORAGE_ENGINE, Tag.TYPE.toString(), @@ -163,7 +163,7 @@ public void unbindFrom(AbstractMetricService metricService) { name -> metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), name, Tag.TYPE.toString(), @@ -175,7 +175,7 @@ public void unbindFrom(AbstractMetricService metricService) { name -> metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), name, Tag.TYPE.toString(), @@ -188,7 +188,7 @@ public void unbindFrom(AbstractMetricService metricService) { name -> metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), name, Tag.TYPE.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java index 7a2accbabca9..2fd8da49cf47 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java @@ -36,7 +36,7 @@ public class StreamEngineMemoryMetrics implements IMetricSet { public void bindTo(AbstractMetricService metricService) { metricService .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), STREAM_ENGINE, @@ -51,7 +51,7 @@ public void bindTo(AbstractMetricService metricService) { public void unbindFrom(AbstractMetricService metricService) { metricService.remove( MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), + Metric.MEMORY_THRESHOLD_SIZE.toString(), Tag.NAME.toString(), STREAM_ENGINE, Tag.TYPE.toString(), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 064a47f2b89a..919a91e2a519 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -188,7 +188,7 @@ public enum Metric { MEMTABLE_POINT_COUNT("memtable_point_count"), BINARY_ALLOCATOR("binary_allocator"), // memory related - THRESHOLD_MEMORY_SIZE("threshold_memory_size"), + MEMORY_THRESHOLD_SIZE("memory_threshold_size"), ; final String value; From ba88514f8aceb07da6d2cf035a333b43e43d33df Mon Sep 17 00:00:00 2001 From: spricoder Date: Mon, 6 Jan 2025 17:11:53 +0800 Subject: [PATCH 25/33] add Metric memory_actual_size --- .../org/apache/iotdb/commons/service/metric/enums/Metric.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 919a91e2a519..906c9f05968e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -189,6 +189,7 @@ public enum Metric { BINARY_ALLOCATOR("binary_allocator"), // memory related MEMORY_THRESHOLD_SIZE("memory_threshold_size"), + MEMORY_ACTUAL_SIZE("memory_actual_size"), ; final String value; From 7c4aeb9659a09a656b445e1ac42d8e99b7d25b7e Mon Sep 17 00:00:00 2001 From: spricoder Date: Wed, 8 Jan 2025 10:45:00 +0800 Subject: [PATCH 26/33] init first version of memory manager and memory block --- .../commons/memory/IoTDBMemoryBlock.java | 149 ++++++++++++++ .../commons/memory/IoTDBMemoryBlockType.java | 26 +++ .../commons/memory/IoTDBMemoryManager.java | 193 ++++++++++++++++++ .../IoTDBRuntimeOutOfMemoryException.java | 86 ++++++++ 4 files changed, 454 insertions(+) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java new file mode 100644 index 000000000000..af4ec206370d --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.memory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +public class IoTDBMemoryBlock implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBMemoryBlock.class); + private final IoTDBMemoryManager memoryManager; + private final IoTDBMemoryBlockType memoryBlockType; + private final ReentrantLock lock = new ReentrantLock(); + private long maxMemorySizeInByte = 0; + private final AtomicLong memoryUsageInBytes = new AtomicLong(0); + private volatile boolean isReleased = false; + + public IoTDBMemoryBlock(final IoTDBMemoryManager memoryManager, final long maxMemorySizeInByte) { + this.memoryManager = memoryManager; + this.maxMemorySizeInByte = maxMemorySizeInByte; + this.memoryBlockType = IoTDBMemoryBlockType.NONE; + } + + public IoTDBMemoryBlock( + final IoTDBMemoryManager memoryManager, + final long maxMemorySizeInByte, + final IoTDBMemoryBlockType memoryBlockType) { + this.memoryManager = memoryManager; + this.maxMemorySizeInByte = maxMemorySizeInByte; + this.memoryBlockType = memoryBlockType; + } + + public boolean useMemory(final long size) { + if (size <= 0) { + memoryUsageInBytes.addAndGet(-size); + return true; + } else { + + AtomicBoolean result = new AtomicBoolean(false); + memoryUsageInBytes.updateAndGet( + memorySize -> { + if (size > maxMemorySizeInByte - memorySize) { + LOGGER.debug( + "consensus memory limited. required: {}, used: {}, total: {}", + size, + memorySize, + maxMemorySizeInByte); + result.set(false); + return memorySize; + } else { + LOGGER.debug( + "{} add {} bytes, total memory size: {} bytes.", + Thread.currentThread().getName(), + size, + memorySize + size); + result.set(true); + return memorySize + size; + } + }); + return result.get(); + } + } + + public long getMaxMemorySizeInByte() { + return maxMemorySizeInByte; + } + + public long getMemoryUsageInBytes() { + return memoryUsageInBytes.get(); + } + + public void setMaxMemorySizeInByte(final long maxMemorySizeInByte) { + this.maxMemorySizeInByte = maxMemorySizeInByte; + } + + public boolean isReleased() { + return isReleased; + } + + public void markAsReleased() { + isReleased = true; + } + + @Override + public String toString() { + return "IoTDBMemoryBlock{" + + "memoryBlockType=" + + memoryBlockType + + "maxMemorySizeInByte=" + + maxMemorySizeInByte + + "memoryUsageInBytes=" + + memoryUsageInBytes + + ", isReleased=" + + isReleased + + '}'; + } + + @Override + public void close() throws Exception { + boolean isInterrupted = false; + + while (true) { + try { + if (lock.tryLock(50, TimeUnit.MICROSECONDS)) { + try { + memoryManager.release(this); + if (isInterrupted) { + LOGGER.warn("{} is released after thread interruption.", this); + } + break; + } finally { + lock.unlock(); + } + } + } catch (final InterruptedException e) { + // Each time the close task is run, it means that the interrupt status left by the previous + // tryLock does not need to be retained. Otherwise, it will lead to an infinite loop. + isInterrupted = true; + LOGGER.warn("Interrupted while waiting for the lock.", e); + } + } + + // Restore the interrupt status of the current thread + if (isInterrupted) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java new file mode 100644 index 000000000000..0b12b77c4002 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.memory; + +public enum IoTDBMemoryBlockType { + NONE, + FUNCTION, + PERFORMANCE, +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java new file mode 100644 index 000000000000..b7195355c0b9 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.memory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Set; +import java.util.function.LongUnaryOperator; + +public class IoTDBMemoryManager { + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBMemoryManager.class); + // TODO spricoder: make it configurable + private static final boolean ENABLED = false; + + /** max retry times for memory allocation */ + private static final int MEMORY_ALLOCATE_MAX_RETRIES = 3; + + /** retry interval for memory allocation */ + private static final long MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS = 1000; + + /** min memory size to allocate */ + private static final long MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES = 32; + + private long totalMemorySizeInBytes = 0L; + private long usedMemorySizeInBytes = 0L; + private IoTDBMemoryManager parentMemoryManager; + private final Set childrens = new HashSet<>(); + private final Set allocatedMemoryBlocks = new HashSet<>(); + + public IoTDBMemoryManager(IoTDBMemoryManager parentMemoryManager, long totalMemorySizeInBytes) { + this.parentMemoryManager = parentMemoryManager; + this.parentMemoryManager.addChildMemoryManager(this); + this.totalMemorySizeInBytes = totalMemorySizeInBytes; + } + + private IoTDBMemoryBlock forceAllocate(long sizeInBytes, IoTDBMemoryBlockType type) { + if (!ENABLED) { + return new IoTDBMemoryBlock(this, sizeInBytes, type); + } + for (int i = 0; i < MEMORY_ALLOCATE_MAX_RETRIES; i++) { + if (totalMemorySizeInBytes - usedMemorySizeInBytes >= sizeInBytes) { + return registerMemoryBlock(sizeInBytes); + } + + try { + // TODO spricoder: try to find more memory + Thread.sleep(MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.warn("forceAllocate: interrupted while waiting for available memory", e); + } + } + + throw new IoTDBRuntimeOutOfMemoryException( + String.format( + "forceAllocate: failed to allocate memory after %d retries, " + + "total memory size %d bytes, used memory size %d bytes, " + + "requested memory size %d bytes", + MEMORY_ALLOCATE_MAX_RETRIES, + totalMemorySizeInBytes, + usedMemorySizeInBytes, + sizeInBytes)); + } + + public synchronized IoTDBMemoryBlock forceAllocateIfSufficient( + long sizeInBytes, float usedThreshold) { + if (usedThreshold < 0.0f || usedThreshold > 1.0f) { + return null; + } + + if (!ENABLED) { + return new IoTDBMemoryBlock(this, sizeInBytes); + } + if (totalMemorySizeInBytes - usedMemorySizeInBytes >= sizeInBytes + && (float) usedMemorySizeInBytes / totalMemorySizeInBytes < usedThreshold) { + return forceAllocate(sizeInBytes, IoTDBMemoryBlockType.NONE); + } else { + // TODO spricoder: try to find more memory + LOGGER.debug( + "forceAllocateIfSufficient: failed to allocate memory, " + + "total memory size {} bytes, used memory size {} bytes, " + + "requested memory size {} bytes, used threshold {}", + totalMemorySizeInBytes, + usedMemorySizeInBytes, + sizeInBytes, + usedThreshold); + } + + return null; + } + + private IoTDBMemoryBlock registerMemoryBlock(long sizeInBytes) { + return registerMemoryBlock(sizeInBytes, IoTDBMemoryBlockType.NONE); + } + + private IoTDBMemoryBlock registerMemoryBlock(long sizeInBytes, IoTDBMemoryBlockType type) { + usedMemorySizeInBytes += sizeInBytes; + final IoTDBMemoryBlock memoryBlock = new IoTDBMemoryBlock(this, sizeInBytes, type); + allocatedMemoryBlocks.add(memoryBlock); + return memoryBlock; + } + + public synchronized IoTDBMemoryBlock tryAllocate( + long sizeInBytes, LongUnaryOperator customAllocateStrategy) { + if (!ENABLED) { + return new IoTDBMemoryBlock(this, sizeInBytes); + } + + if (totalMemorySizeInBytes - usedMemorySizeInBytes >= sizeInBytes) { + return registerMemoryBlock(sizeInBytes); + } + + long sizeToAllocateInBytes = sizeInBytes; + while (sizeToAllocateInBytes > MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES) { + if (totalMemorySizeInBytes - usedMemorySizeInBytes >= sizeToAllocateInBytes) { + LOGGER.info( + "tryAllocate: allocated memory, " + + "total memory size {} bytes, used memory size {} bytes, " + + "original requested memory size {} bytes, " + + "actual requested memory size {} bytes", + totalMemorySizeInBytes, + usedMemorySizeInBytes, + sizeInBytes, + sizeToAllocateInBytes); + return registerMemoryBlock(sizeToAllocateInBytes); + } + + sizeToAllocateInBytes = + Math.max( + customAllocateStrategy.applyAsLong(sizeToAllocateInBytes), + MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES); + } + + // TODO spricoder: try to shrink first + LOGGER.warn( + "tryAllocate: failed to allocate memory, " + + "total memory size {} bytes, used memory size {} bytes, " + + "requested memory size {} bytes", + totalMemorySizeInBytes, + usedMemorySizeInBytes, + sizeInBytes); + return registerMemoryBlock(0); + } + + public synchronized void release(IoTDBMemoryBlock block) { + if (!ENABLED || block == null || block.isReleased()) { + return; + } + + allocatedMemoryBlocks.remove(block); + usedMemorySizeInBytes -= block.getMemoryUsageInBytes(); + block.markAsReleased(); + + this.notifyAll(); + } + + public synchronized void addChildMemoryManager(IoTDBMemoryManager childMemoryManager) { + if (childMemoryManager != null) { + childrens.add(childMemoryManager); + } + } + + public long getUsedMemorySizeInBytes() { + return usedMemorySizeInBytes; + } + + public long getFreeMemorySizeInBytes() { + return totalMemorySizeInBytes - usedMemorySizeInBytes; + } + + public long getTotalMemorySizeInBytes() { + return totalMemorySizeInBytes; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java new file mode 100644 index 000000000000..c2c5777658c6 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.memory; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class IoTDBRuntimeOutOfMemoryException extends RuntimeException { + + private final long timeStamp; + private static final long VERSION = 1L; + + public IoTDBRuntimeOutOfMemoryException(final String message) { + super(message); + this.timeStamp = System.currentTimeMillis(); + } + + public IoTDBRuntimeOutOfMemoryException(final String message, final long timeStamp) { + super(message); + this.timeStamp = timeStamp; + } + + public IoTDBRuntimeOutOfMemoryException(final String message, final Throwable cause) { + super(message, cause); + this.timeStamp = System.currentTimeMillis(); + } + + public long getTimeStamp() { + return timeStamp; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof IoTDBRuntimeOutOfMemoryException + && Objects.equals(getMessage(), ((IoTDBRuntimeOutOfMemoryException) obj).getMessage()) + && Objects.equals(getTimeStamp(), ((IoTDBRuntimeOutOfMemoryException) obj).getTimeStamp()); + } + + @Override + public int hashCode() { + return Objects.hash(getMessage(), getTimeStamp()); + } + + public void serialize(final ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(VERSION, byteBuffer); + ReadWriteIOUtils.write(getMessage(), byteBuffer); + ReadWriteIOUtils.write(getTimeStamp(), byteBuffer); + } + + public void serialize(final OutputStream stream) throws IOException { + ReadWriteIOUtils.write(VERSION, stream); + ReadWriteIOUtils.write(getMessage(), stream); + ReadWriteIOUtils.write(getTimeStamp(), stream); + } + + @Override + public String toString() { + return "IoTDBRuntimeOutOfMemoryException{" + + "message='" + + getMessage() + + "', timeStamp=" + + getTimeStamp() + + "}"; + } +} From cc890215bc0e1cbdd5fd9ed95ccd81f715daec4e Mon Sep 17 00:00:00 2001 From: spricoder Date: Wed, 8 Jan 2025 10:52:56 +0800 Subject: [PATCH 27/33] add IIoTDBMemoryBlock --- .../commons/memory/IIoTDBMemoryBlock.java | 54 +++++++++++++++++++ .../commons/memory/IoTDBMemoryBlock.java | 31 +---------- 2 files changed, 56 insertions(+), 29 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java new file mode 100644 index 000000000000..48365f9afb77 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.memory; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +public abstract class IIoTDBMemoryBlock implements AutoCloseable{ + protected IoTDBMemoryManager memoryManager; + protected IoTDBMemoryBlockType memoryBlockType; + protected final ReentrantLock lock = new ReentrantLock(); + protected long maxMemorySizeInByte = 0; + protected final AtomicLong memoryUsageInBytes = new AtomicLong(0); + protected volatile boolean isReleased = false; + + public abstract boolean useMemory(final long size); + + public long getMaxMemorySizeInByte() { + return maxMemorySizeInByte; + } + + public long getMemoryUsageInBytes() { + return memoryUsageInBytes.get(); + } + + public void setMaxMemorySizeInByte(final long maxMemorySizeInByte) { + this.maxMemorySizeInByte = maxMemorySizeInByte; + } + + public boolean isReleased() { + return isReleased; + } + + public void markAsReleased() { + isReleased = true; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java index af4ec206370d..8c8e8b410bc2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java @@ -24,17 +24,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; -public class IoTDBMemoryBlock implements AutoCloseable { +public class IoTDBMemoryBlock extends IIoTDBMemoryBlock{ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBMemoryBlock.class); - private final IoTDBMemoryManager memoryManager; - private final IoTDBMemoryBlockType memoryBlockType; - private final ReentrantLock lock = new ReentrantLock(); - private long maxMemorySizeInByte = 0; - private final AtomicLong memoryUsageInBytes = new AtomicLong(0); - private volatile boolean isReleased = false; public IoTDBMemoryBlock(final IoTDBMemoryManager memoryManager, final long maxMemorySizeInByte) { this.memoryManager = memoryManager; @@ -51,6 +43,7 @@ public IoTDBMemoryBlock( this.memoryBlockType = memoryBlockType; } + @Override public boolean useMemory(final long size) { if (size <= 0) { memoryUsageInBytes.addAndGet(-size); @@ -82,26 +75,6 @@ public boolean useMemory(final long size) { } } - public long getMaxMemorySizeInByte() { - return maxMemorySizeInByte; - } - - public long getMemoryUsageInBytes() { - return memoryUsageInBytes.get(); - } - - public void setMaxMemorySizeInByte(final long maxMemorySizeInByte) { - this.maxMemorySizeInByte = maxMemorySizeInByte; - } - - public boolean isReleased() { - return isReleased; - } - - public void markAsReleased() { - isReleased = true; - } - @Override public String toString() { return "IoTDBMemoryBlock{" From e33d234b2feeab41685cf03cb5731cfa91f63077 Mon Sep 17 00:00:00 2001 From: spricoder Date: Wed, 8 Jan 2025 11:09:09 +0800 Subject: [PATCH 28/33] fix template --- .../java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java | 2 +- .../java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java index 48365f9afb77..cbccd7c0fca6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java @@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; -public abstract class IIoTDBMemoryBlock implements AutoCloseable{ +public abstract class IIoTDBMemoryBlock implements AutoCloseable { protected IoTDBMemoryManager memoryManager; protected IoTDBMemoryBlockType memoryBlockType; protected final ReentrantLock lock = new ReentrantLock(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java index 8c8e8b410bc2..76d2754ed315 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java @@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -public class IoTDBMemoryBlock extends IIoTDBMemoryBlock{ +public class IoTDBMemoryBlock extends IIoTDBMemoryBlock { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBMemoryBlock.class); public IoTDBMemoryBlock(final IoTDBMemoryManager memoryManager, final long maxMemorySizeInByte) { From dbbfc5fde30ec63417196903c90edbd890580d67 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 9 Jan 2025 20:02:00 +0800 Subject: [PATCH 29/33] rollback --- .../org/apache/iotdb/db/service/metrics/WritingMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java index d6b16e85daf7..15238d4afa36 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java @@ -509,7 +509,7 @@ public void createDataRegionMemoryCostMetrics(DataRegion dataRegion) { MetricService.getInstance() .createAutoGauge( Metric.DATA_REGION_MEM_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, dataRegion, DataRegion::getMemCost, Tag.REGION.toString(), From 98fb2fb75e5c93f73ce2577d5bf788587a1c1f27 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 9 Jan 2025 20:02:52 +0800 Subject: [PATCH 30/33] Revert "fix template" This reverts commit e33d234b2feeab41685cf03cb5731cfa91f63077. --- .../java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java | 2 +- .../java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java index cbccd7c0fca6..48365f9afb77 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java @@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; -public abstract class IIoTDBMemoryBlock implements AutoCloseable { +public abstract class IIoTDBMemoryBlock implements AutoCloseable{ protected IoTDBMemoryManager memoryManager; protected IoTDBMemoryBlockType memoryBlockType; protected final ReentrantLock lock = new ReentrantLock(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java index 76d2754ed315..8c8e8b410bc2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java @@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -public class IoTDBMemoryBlock extends IIoTDBMemoryBlock { +public class IoTDBMemoryBlock extends IIoTDBMemoryBlock{ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBMemoryBlock.class); public IoTDBMemoryBlock(final IoTDBMemoryManager memoryManager, final long maxMemorySizeInByte) { From 8d093d4a533b401ff1c3d2aa5ffe77541d795b10 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 9 Jan 2025 20:02:55 +0800 Subject: [PATCH 31/33] Revert "add IIoTDBMemoryBlock" This reverts commit cc890215bc0e1cbdd5fd9ed95ccd81f715daec4e. --- .../commons/memory/IIoTDBMemoryBlock.java | 54 ------------------- .../commons/memory/IoTDBMemoryBlock.java | 31 ++++++++++- 2 files changed, 29 insertions(+), 56 deletions(-) delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java deleted file mode 100644 index 48365f9afb77..000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IIoTDBMemoryBlock.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.memory; - -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; - -public abstract class IIoTDBMemoryBlock implements AutoCloseable{ - protected IoTDBMemoryManager memoryManager; - protected IoTDBMemoryBlockType memoryBlockType; - protected final ReentrantLock lock = new ReentrantLock(); - protected long maxMemorySizeInByte = 0; - protected final AtomicLong memoryUsageInBytes = new AtomicLong(0); - protected volatile boolean isReleased = false; - - public abstract boolean useMemory(final long size); - - public long getMaxMemorySizeInByte() { - return maxMemorySizeInByte; - } - - public long getMemoryUsageInBytes() { - return memoryUsageInBytes.get(); - } - - public void setMaxMemorySizeInByte(final long maxMemorySizeInByte) { - this.maxMemorySizeInByte = maxMemorySizeInByte; - } - - public boolean isReleased() { - return isReleased; - } - - public void markAsReleased() { - isReleased = true; - } -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java index 8c8e8b410bc2..af4ec206370d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java @@ -24,9 +24,17 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; -public class IoTDBMemoryBlock extends IIoTDBMemoryBlock{ +public class IoTDBMemoryBlock implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBMemoryBlock.class); + private final IoTDBMemoryManager memoryManager; + private final IoTDBMemoryBlockType memoryBlockType; + private final ReentrantLock lock = new ReentrantLock(); + private long maxMemorySizeInByte = 0; + private final AtomicLong memoryUsageInBytes = new AtomicLong(0); + private volatile boolean isReleased = false; public IoTDBMemoryBlock(final IoTDBMemoryManager memoryManager, final long maxMemorySizeInByte) { this.memoryManager = memoryManager; @@ -43,7 +51,6 @@ public IoTDBMemoryBlock( this.memoryBlockType = memoryBlockType; } - @Override public boolean useMemory(final long size) { if (size <= 0) { memoryUsageInBytes.addAndGet(-size); @@ -75,6 +82,26 @@ public boolean useMemory(final long size) { } } + public long getMaxMemorySizeInByte() { + return maxMemorySizeInByte; + } + + public long getMemoryUsageInBytes() { + return memoryUsageInBytes.get(); + } + + public void setMaxMemorySizeInByte(final long maxMemorySizeInByte) { + this.maxMemorySizeInByte = maxMemorySizeInByte; + } + + public boolean isReleased() { + return isReleased; + } + + public void markAsReleased() { + isReleased = true; + } + @Override public String toString() { return "IoTDBMemoryBlock{" From 4974193ebf935de11bfaac50e5069027235b0a7b Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 9 Jan 2025 20:02:58 +0800 Subject: [PATCH 32/33] Revert "init first version of memory manager and memory block" This reverts commit 7c4aeb9659a09a656b445e1ac42d8e99b7d25b7e. --- .../commons/memory/IoTDBMemoryBlock.java | 149 -------------- .../commons/memory/IoTDBMemoryBlockType.java | 26 --- .../commons/memory/IoTDBMemoryManager.java | 193 ------------------ .../IoTDBRuntimeOutOfMemoryException.java | 86 -------- 4 files changed, 454 deletions(-) delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java deleted file mode 100644 index af4ec206370d..000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlock.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.memory; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; - -public class IoTDBMemoryBlock implements AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBMemoryBlock.class); - private final IoTDBMemoryManager memoryManager; - private final IoTDBMemoryBlockType memoryBlockType; - private final ReentrantLock lock = new ReentrantLock(); - private long maxMemorySizeInByte = 0; - private final AtomicLong memoryUsageInBytes = new AtomicLong(0); - private volatile boolean isReleased = false; - - public IoTDBMemoryBlock(final IoTDBMemoryManager memoryManager, final long maxMemorySizeInByte) { - this.memoryManager = memoryManager; - this.maxMemorySizeInByte = maxMemorySizeInByte; - this.memoryBlockType = IoTDBMemoryBlockType.NONE; - } - - public IoTDBMemoryBlock( - final IoTDBMemoryManager memoryManager, - final long maxMemorySizeInByte, - final IoTDBMemoryBlockType memoryBlockType) { - this.memoryManager = memoryManager; - this.maxMemorySizeInByte = maxMemorySizeInByte; - this.memoryBlockType = memoryBlockType; - } - - public boolean useMemory(final long size) { - if (size <= 0) { - memoryUsageInBytes.addAndGet(-size); - return true; - } else { - - AtomicBoolean result = new AtomicBoolean(false); - memoryUsageInBytes.updateAndGet( - memorySize -> { - if (size > maxMemorySizeInByte - memorySize) { - LOGGER.debug( - "consensus memory limited. required: {}, used: {}, total: {}", - size, - memorySize, - maxMemorySizeInByte); - result.set(false); - return memorySize; - } else { - LOGGER.debug( - "{} add {} bytes, total memory size: {} bytes.", - Thread.currentThread().getName(), - size, - memorySize + size); - result.set(true); - return memorySize + size; - } - }); - return result.get(); - } - } - - public long getMaxMemorySizeInByte() { - return maxMemorySizeInByte; - } - - public long getMemoryUsageInBytes() { - return memoryUsageInBytes.get(); - } - - public void setMaxMemorySizeInByte(final long maxMemorySizeInByte) { - this.maxMemorySizeInByte = maxMemorySizeInByte; - } - - public boolean isReleased() { - return isReleased; - } - - public void markAsReleased() { - isReleased = true; - } - - @Override - public String toString() { - return "IoTDBMemoryBlock{" - + "memoryBlockType=" - + memoryBlockType - + "maxMemorySizeInByte=" - + maxMemorySizeInByte - + "memoryUsageInBytes=" - + memoryUsageInBytes - + ", isReleased=" - + isReleased - + '}'; - } - - @Override - public void close() throws Exception { - boolean isInterrupted = false; - - while (true) { - try { - if (lock.tryLock(50, TimeUnit.MICROSECONDS)) { - try { - memoryManager.release(this); - if (isInterrupted) { - LOGGER.warn("{} is released after thread interruption.", this); - } - break; - } finally { - lock.unlock(); - } - } - } catch (final InterruptedException e) { - // Each time the close task is run, it means that the interrupt status left by the previous - // tryLock does not need to be retained. Otherwise, it will lead to an infinite loop. - isInterrupted = true; - LOGGER.warn("Interrupted while waiting for the lock.", e); - } - } - - // Restore the interrupt status of the current thread - if (isInterrupted) { - Thread.currentThread().interrupt(); - } - } -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java deleted file mode 100644 index 0b12b77c4002..000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryBlockType.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.memory; - -public enum IoTDBMemoryBlockType { - NONE, - FUNCTION, - PERFORMANCE, -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java deleted file mode 100644 index b7195355c0b9..000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBMemoryManager.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.memory; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.Set; -import java.util.function.LongUnaryOperator; - -public class IoTDBMemoryManager { - private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBMemoryManager.class); - // TODO spricoder: make it configurable - private static final boolean ENABLED = false; - - /** max retry times for memory allocation */ - private static final int MEMORY_ALLOCATE_MAX_RETRIES = 3; - - /** retry interval for memory allocation */ - private static final long MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS = 1000; - - /** min memory size to allocate */ - private static final long MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES = 32; - - private long totalMemorySizeInBytes = 0L; - private long usedMemorySizeInBytes = 0L; - private IoTDBMemoryManager parentMemoryManager; - private final Set childrens = new HashSet<>(); - private final Set allocatedMemoryBlocks = new HashSet<>(); - - public IoTDBMemoryManager(IoTDBMemoryManager parentMemoryManager, long totalMemorySizeInBytes) { - this.parentMemoryManager = parentMemoryManager; - this.parentMemoryManager.addChildMemoryManager(this); - this.totalMemorySizeInBytes = totalMemorySizeInBytes; - } - - private IoTDBMemoryBlock forceAllocate(long sizeInBytes, IoTDBMemoryBlockType type) { - if (!ENABLED) { - return new IoTDBMemoryBlock(this, sizeInBytes, type); - } - for (int i = 0; i < MEMORY_ALLOCATE_MAX_RETRIES; i++) { - if (totalMemorySizeInBytes - usedMemorySizeInBytes >= sizeInBytes) { - return registerMemoryBlock(sizeInBytes); - } - - try { - // TODO spricoder: try to find more memory - Thread.sleep(MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOGGER.warn("forceAllocate: interrupted while waiting for available memory", e); - } - } - - throw new IoTDBRuntimeOutOfMemoryException( - String.format( - "forceAllocate: failed to allocate memory after %d retries, " - + "total memory size %d bytes, used memory size %d bytes, " - + "requested memory size %d bytes", - MEMORY_ALLOCATE_MAX_RETRIES, - totalMemorySizeInBytes, - usedMemorySizeInBytes, - sizeInBytes)); - } - - public synchronized IoTDBMemoryBlock forceAllocateIfSufficient( - long sizeInBytes, float usedThreshold) { - if (usedThreshold < 0.0f || usedThreshold > 1.0f) { - return null; - } - - if (!ENABLED) { - return new IoTDBMemoryBlock(this, sizeInBytes); - } - if (totalMemorySizeInBytes - usedMemorySizeInBytes >= sizeInBytes - && (float) usedMemorySizeInBytes / totalMemorySizeInBytes < usedThreshold) { - return forceAllocate(sizeInBytes, IoTDBMemoryBlockType.NONE); - } else { - // TODO spricoder: try to find more memory - LOGGER.debug( - "forceAllocateIfSufficient: failed to allocate memory, " - + "total memory size {} bytes, used memory size {} bytes, " - + "requested memory size {} bytes, used threshold {}", - totalMemorySizeInBytes, - usedMemorySizeInBytes, - sizeInBytes, - usedThreshold); - } - - return null; - } - - private IoTDBMemoryBlock registerMemoryBlock(long sizeInBytes) { - return registerMemoryBlock(sizeInBytes, IoTDBMemoryBlockType.NONE); - } - - private IoTDBMemoryBlock registerMemoryBlock(long sizeInBytes, IoTDBMemoryBlockType type) { - usedMemorySizeInBytes += sizeInBytes; - final IoTDBMemoryBlock memoryBlock = new IoTDBMemoryBlock(this, sizeInBytes, type); - allocatedMemoryBlocks.add(memoryBlock); - return memoryBlock; - } - - public synchronized IoTDBMemoryBlock tryAllocate( - long sizeInBytes, LongUnaryOperator customAllocateStrategy) { - if (!ENABLED) { - return new IoTDBMemoryBlock(this, sizeInBytes); - } - - if (totalMemorySizeInBytes - usedMemorySizeInBytes >= sizeInBytes) { - return registerMemoryBlock(sizeInBytes); - } - - long sizeToAllocateInBytes = sizeInBytes; - while (sizeToAllocateInBytes > MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES) { - if (totalMemorySizeInBytes - usedMemorySizeInBytes >= sizeToAllocateInBytes) { - LOGGER.info( - "tryAllocate: allocated memory, " - + "total memory size {} bytes, used memory size {} bytes, " - + "original requested memory size {} bytes, " - + "actual requested memory size {} bytes", - totalMemorySizeInBytes, - usedMemorySizeInBytes, - sizeInBytes, - sizeToAllocateInBytes); - return registerMemoryBlock(sizeToAllocateInBytes); - } - - sizeToAllocateInBytes = - Math.max( - customAllocateStrategy.applyAsLong(sizeToAllocateInBytes), - MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES); - } - - // TODO spricoder: try to shrink first - LOGGER.warn( - "tryAllocate: failed to allocate memory, " - + "total memory size {} bytes, used memory size {} bytes, " - + "requested memory size {} bytes", - totalMemorySizeInBytes, - usedMemorySizeInBytes, - sizeInBytes); - return registerMemoryBlock(0); - } - - public synchronized void release(IoTDBMemoryBlock block) { - if (!ENABLED || block == null || block.isReleased()) { - return; - } - - allocatedMemoryBlocks.remove(block); - usedMemorySizeInBytes -= block.getMemoryUsageInBytes(); - block.markAsReleased(); - - this.notifyAll(); - } - - public synchronized void addChildMemoryManager(IoTDBMemoryManager childMemoryManager) { - if (childMemoryManager != null) { - childrens.add(childMemoryManager); - } - } - - public long getUsedMemorySizeInBytes() { - return usedMemorySizeInBytes; - } - - public long getFreeMemorySizeInBytes() { - return totalMemorySizeInBytes - usedMemorySizeInBytes; - } - - public long getTotalMemorySizeInBytes() { - return totalMemorySizeInBytes; - } -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java deleted file mode 100644 index c2c5777658c6..000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/IoTDBRuntimeOutOfMemoryException.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.memory; - -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Objects; - -public class IoTDBRuntimeOutOfMemoryException extends RuntimeException { - - private final long timeStamp; - private static final long VERSION = 1L; - - public IoTDBRuntimeOutOfMemoryException(final String message) { - super(message); - this.timeStamp = System.currentTimeMillis(); - } - - public IoTDBRuntimeOutOfMemoryException(final String message, final long timeStamp) { - super(message); - this.timeStamp = timeStamp; - } - - public IoTDBRuntimeOutOfMemoryException(final String message, final Throwable cause) { - super(message, cause); - this.timeStamp = System.currentTimeMillis(); - } - - public long getTimeStamp() { - return timeStamp; - } - - @Override - public boolean equals(Object obj) { - return obj instanceof IoTDBRuntimeOutOfMemoryException - && Objects.equals(getMessage(), ((IoTDBRuntimeOutOfMemoryException) obj).getMessage()) - && Objects.equals(getTimeStamp(), ((IoTDBRuntimeOutOfMemoryException) obj).getTimeStamp()); - } - - @Override - public int hashCode() { - return Objects.hash(getMessage(), getTimeStamp()); - } - - public void serialize(final ByteBuffer byteBuffer) { - ReadWriteIOUtils.write(VERSION, byteBuffer); - ReadWriteIOUtils.write(getMessage(), byteBuffer); - ReadWriteIOUtils.write(getTimeStamp(), byteBuffer); - } - - public void serialize(final OutputStream stream) throws IOException { - ReadWriteIOUtils.write(VERSION, stream); - ReadWriteIOUtils.write(getMessage(), stream); - ReadWriteIOUtils.write(getTimeStamp(), stream); - } - - @Override - public String toString() { - return "IoTDBRuntimeOutOfMemoryException{" - + "message='" - + getMessage() - + "', timeStamp=" - + getTimeStamp() - + "}"; - } -} From 37840351b28a0c031df109f0ae53568510dfdd71 Mon Sep 17 00:00:00 2001 From: spricoder Date: Thu, 9 Jan 2025 20:16:18 +0800 Subject: [PATCH 33/33] rollback 2 --- .../db/service/metrics/WritingMetrics.java | 106 +++++++++--------- 1 file changed, 56 insertions(+), 50 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java index 15238d4afa36..8820be02fa86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java @@ -67,34 +67,37 @@ private void bindFlushMetrics(AbstractMetricService metricService) { flushStageSortTimer = metricService.getOrCreateTimer( Metric.FLUSH_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), FLUSH_STAGE_SORT); flushStageEncodingTimer = metricService.getOrCreateTimer( Metric.FLUSH_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), FLUSH_STAGE_ENCODING); flushStageIOTimer = metricService.getOrCreateTimer( - Metric.FLUSH_COST.toString(), MetricLevel.NORMAL, Tag.STAGE.toString(), FLUSH_STAGE_IO); + Metric.FLUSH_COST.toString(), + MetricLevel.IMPORTANT, + Tag.STAGE.toString(), + FLUSH_STAGE_IO); writePlanIndicesTimer = metricService.getOrCreateTimer( Metric.FLUSH_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), WRITE_PLAN_INDICES); metricService.createAutoGauge( Metric.PENDING_FLUSH_TASK.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, FlushManager.getInstance(), FlushManager::getNumberOfPendingTasks, Tag.TYPE.toString(), PENDING_TASK_NUM); metricService.createAutoGauge( Metric.PENDING_FLUSH_TASK.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, FlushManager.getInstance(), FlushManager::getNumberOfPendingSubTasks, Tag.TYPE.toString(), @@ -136,19 +139,19 @@ private void bindFlushSubTaskMetrics(AbstractMetricService metricService) { sortTaskTimer = metricService.getOrCreateTimer( Metric.FLUSH_SUB_TASK_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), SORT_TASK); encodingTaskTimer = metricService.getOrCreateTimer( Metric.FLUSH_SUB_TASK_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), ENCODING_TASK); ioTaskTimer = metricService.getOrCreateTimer( Metric.FLUSH_SUB_TASK_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), IO_TASK); } @@ -197,70 +200,73 @@ private void unbindFlushSubTaskMetrics(AbstractMetricService metricService) { private void bindWALMetrics(AbstractMetricService metricService) { metricService.createAutoGauge( Metric.WAL_NODE_NUM.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, WAL_MANAGER, WALManager::getWALNodesNum, Tag.NAME.toString(), WAL_NODES_NUM); usedRatioHistogram = metricService.getOrCreateHistogram( - Metric.WAL_BUFFER.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), USED_RATIO); + Metric.WAL_BUFFER.toString(), MetricLevel.IMPORTANT, Tag.NAME.toString(), USED_RATIO); entriesCountHistogram = metricService.getOrCreateHistogram( - Metric.WAL_BUFFER.toString(), MetricLevel.NORMAL, Tag.NAME.toString(), ENTRIES_COUNT); + Metric.WAL_BUFFER.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + ENTRIES_COUNT); serializedWALBufferSizeHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), SERIALIZED_WAL_BUFFER_SIZE_BYTE); wroteWALBufferSizeHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), WROTE_WAL_BUFFER_SIZE_BYTE); walCompressCostHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), WAL_COMPRESS_COST_NS); walUncompressCostHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), WAL_UNCOMPRESS_COST_NS); readWALBufferSizeHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), READ_WAL_BUFFER_SIZE_BYTE); readWALBufferCostHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), READ_WAL_BUFFER_COST_NS); writeWALBufferCostHistogram = metricService.getOrCreateHistogram( Metric.WAL_BUFFER.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), WRITE_WAL_BUFFER_COST_NS); walQueueMaxMemSizeGauge = metricService.getOrCreateGauge( Metric.WAL_QUEUE_MEM_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), WAL_QUEUE_MAX_MEM_COST); SystemInfo systemInfo = SystemInfo.getInstance(); metricService.createAutoGauge( Metric.WAL_QUEUE_MEM_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, systemInfo, SystemInfo::getCurrentWalQueueMemoryCost, Tag.NAME.toString(), @@ -318,7 +324,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { globalMemoryTableInfoTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), MAKE_CHECKPOINT, Tag.TYPE.toString(), @@ -326,7 +332,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { createMemoryTableTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), MAKE_CHECKPOINT, Tag.TYPE.toString(), @@ -334,7 +340,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { flushMemoryTableTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), MAKE_CHECKPOINT, Tag.TYPE.toString(), @@ -342,7 +348,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { serializeWalEntryTotalTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), SERIALIZE_WAL_ENTRY, Tag.TYPE.toString(), @@ -350,7 +356,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { syncTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), SYNC_WAL_BUFFER, Tag.TYPE.toString(), @@ -358,7 +364,7 @@ private void bindWALCostMetrics(AbstractMetricService metricService) { fsyncTimer = metricService.getOrCreateTimer( Metric.WAL_COST.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.STAGE.toString(), SYNC_WAL_BUFFER, Tag.TYPE.toString(), @@ -459,20 +465,20 @@ public void bindDataRegionMetrics() { MetricService.getInstance() .getOrCreateGauge( Metric.MEMTABLE_THRESHOLD.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), FLUSH_THRESHOLD); rejectThreholdGauge = MetricService.getInstance() .getOrCreateGauge( Metric.MEMTABLE_THRESHOLD.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), REJECT_THRESHOLD); memtableLiveTimer = MetricService.getInstance() - .getOrCreateTimer(Metric.MEMTABLE_LIVE_DURATION.toString(), MetricLevel.NORMAL); + .getOrCreateTimer(Metric.MEMTABLE_LIVE_DURATION.toString(), MetricLevel.IMPORTANT); } public void unbindDataRegionMetrics() { @@ -535,7 +541,7 @@ public void createWALNodeInfoMetrics(String walNodeId) { MetricService.getInstance() .getOrCreateHistogram( Metric.WAL_NODE_INFO.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), name, Tag.TYPE.toString(), @@ -572,7 +578,7 @@ public void createFlushingMemTableStatusMetrics(DataRegionId dataRegionId) { MetricService.getInstance() .getOrCreateHistogram( Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), name, Tag.REGION.toString(), @@ -583,7 +589,7 @@ public Counter createWalFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), WAL_FLUSH_MEMTABLE_COUNT); } @@ -592,7 +598,7 @@ public Counter createTimedFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), TIMED_FLUSH_MEMTABLE_COUNT); } @@ -601,7 +607,7 @@ public Counter createSeriesFullFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), SERIES_FULL_FLUSH_MEMTABLE); } @@ -610,7 +616,7 @@ public Counter createManualFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), MANUAL_FLUSH_MEMTABLE_COUNT); } @@ -619,7 +625,7 @@ public Counter createMemControlFlushMemTableCounterMetrics() { return MetricService.getInstance() .getOrCreateCounter( Metric.FLUSH_MEMTABLE_COUNT.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.TYPE.toString(), MEM_CONTROL_FLUSH_MEMTABLE_COUNT); } @@ -628,14 +634,14 @@ public void createActiveMemtableCounterMetrics(DataRegionId dataRegionId) { MetricService.getInstance() .getOrCreateCounter( Metric.ACTIVE_MEMTABLE_COUNT.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.REGION.toString(), dataRegionId.toString()); } public void createActiveTimePartitionCounterMetrics() { MetricService.getInstance() - .getOrCreateCounter(Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), MetricLevel.NORMAL); + .getOrCreateCounter(Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), MetricLevel.IMPORTANT); } public void removeSeriesFullFlushMemTableCounterMetrics() { @@ -722,7 +728,7 @@ public void recordWALNodeEffectiveInfoRatio(String walNodeId, double ratio) { .histogram( (long) (ratio * 100), Metric.WAL_NODE_INFO.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), EFFECTIVE_RATIO_INFO, Tag.TYPE.toString(), @@ -734,7 +740,7 @@ public void recordMemTableRamWhenCauseSnapshot(String walNodeId, long ram) { .histogram( ram, Metric.WAL_NODE_INFO.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), OLDEST_MEM_TABLE_RAM_WHEN_CAUSE_SNAPSHOT, Tag.TYPE.toString(), @@ -746,7 +752,7 @@ public void recordMemTableRamWhenCauseFlush(String walNodeId, long ram) { .histogram( ram, Metric.WAL_NODE_INFO.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), OLDEST_MEM_TABLE_RAM_WHEN_CAUSE_FLUSH, Tag.TYPE.toString(), @@ -759,7 +765,7 @@ public void recordTsFileCompressionRatioOfFlushingMemTable( .histogram( (long) (compressionRatio * 100), Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), COMPRESSION_RATIO, Tag.REGION.toString(), @@ -777,7 +783,7 @@ public void recordFlushingMemTableStatus( .histogram( memSize, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), MEM_TABLE_SIZE, Tag.REGION.toString(), @@ -786,7 +792,7 @@ public void recordFlushingMemTableStatus( .histogram( seriesNum, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), SERIES_NUM, Tag.REGION.toString(), @@ -795,7 +801,7 @@ public void recordFlushingMemTableStatus( .histogram( totalPointsNum, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), POINTS_NUM, Tag.REGION.toString(), @@ -804,7 +810,7 @@ public void recordFlushingMemTableStatus( .histogram( avgSeriesNum, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), AVG_SERIES_POINT_NUM, Tag.REGION.toString(), @@ -820,7 +826,7 @@ public void recordFlushTsFileSize(String storageGroup, long size) { .histogram( size, Metric.FLUSHING_MEM_TABLE_STATUS.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.NAME.toString(), FLUSH_TSFILE_SIZE, Tag.REGION.toString(), @@ -972,14 +978,14 @@ public void recordActiveMemTableCount(String dataRegionId, int number) { .count( number, Metric.ACTIVE_MEMTABLE_COUNT.toString(), - MetricLevel.NORMAL, + MetricLevel.IMPORTANT, Tag.REGION.toString(), dataRegionId); } public void recordActiveTimePartitionCount(int number) { MetricService.getInstance() - .count(number, Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), MetricLevel.NORMAL); + .count(number, Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), MetricLevel.IMPORTANT); } // endregion