diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java index dda0a403bc52..8ac2fa1fde4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java @@ -39,7 +39,7 @@ import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; -import org.apache.iotdb.db.service.metrics.memory.ThresholdMemoryMetrics; +import org.apache.iotdb.db.service.metrics.memory.GlobalMemoryMetrics; import org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesNumberMetricsSet; import org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesSizeMetricsSet; import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet; @@ -105,7 +105,7 @@ public static void bind() { metricService.addMetricSet(ActiveLoadingFilesSizeMetricsSet.getInstance()); // bind memory related metrics - metricService.addMetricSet(ThresholdMemoryMetrics.getInstance()); + metricService.addMetricSet(GlobalMemoryMetrics.getInstance()); } private static void initSystemMetrics(MetricService metricService) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java new file mode 100644 index 000000000000..b3f108892e71 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ConsensusMemoryMetrics.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +public class ConsensusMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String CONSENSUS = "Consensus"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + CONSENSUS, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(config.getAllocateMemoryForConsensus()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + CONSENSUS, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + } + + public static ConsensusMemoryMetrics getInstance() { + return ConsensusMemoryMetricsHolder.INSTANCE; + } + + private static class ConsensusMemoryMetricsHolder { + + private static final ConsensusMemoryMetrics INSTANCE = new ConsensusMemoryMetrics(); + + private ConsensusMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java new file mode 100644 index 000000000000..cca3f46f1d88 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/GlobalMemoryMetrics.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Arrays; +import java.util.Collections; + +public class GlobalMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final SystemInfo systemInfo = SystemInfo.getInstance(); + + private static final String TOTAL = "Total"; + public static final String ON_HEAP = "OnHeap"; + public static final String OFF_HEAP = "OffHeap"; + public static final String[] LEVELS = {"0", "1", "2", "3", "4"}; + + private static final String DIRECT_BUFFER = "DirectBuffer"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + TOTAL, + Tag.TYPE.toString(), + ON_HEAP, + Tag.LEVEL.toString(), + LEVELS[0]) + .set(Runtime.getRuntime().maxMemory()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + TOTAL, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + LEVELS[0]) + .set(config.getMaxOffHeapMemoryBytes()); + StorageEngineMemoryMetrics.getInstance().bindTo(metricService); + QueryEngineMemoryMetrics.getInstance().bindTo(metricService); + SchemaEngineMemoryMetrics.getInstance().bindTo(metricService); + ConsensusMemoryMetrics.getInstance().bindTo(metricService); + StreamEngineMemoryMetrics.getInstance().bindTo(metricService); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + DIRECT_BUFFER, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + LEVELS[1]) + .set(systemInfo.getTotalDirectBufferMemorySizeLimit()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + Arrays.asList(ON_HEAP, OFF_HEAP) + .forEach( + type -> { + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + TOTAL, + Tag.TYPE.toString(), + type, + Tag.LEVEL.toString(), + LEVELS[0]); + }); + StorageEngineMemoryMetrics.getInstance().unbindFrom(metricService); + QueryEngineMemoryMetrics.getInstance().unbindFrom(metricService); + SchemaEngineMemoryMetrics.getInstance().unbindFrom(metricService); + ConsensusMemoryMetrics.getInstance().unbindFrom(metricService); + StreamEngineMemoryMetrics.getInstance().unbindFrom(metricService); + Collections.singletonList(DIRECT_BUFFER) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + OFF_HEAP, + Tag.LEVEL.toString(), + LEVELS[1])); + } + + public static GlobalMemoryMetrics getInstance() { + return GlobalMemoryMetricsHolder.INSTANCE; + } + + private static class GlobalMemoryMetricsHolder { + + private static final GlobalMemoryMetrics INSTANCE = new GlobalMemoryMetrics(); + + private GlobalMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java new file mode 100644 index 000000000000..67538646525a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/QueryEngineMemoryMetrics.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Arrays; + +public class QueryEngineMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String QUERY_ENGINE = "QueryEngine"; + private static final String QUERY_ENGINE_BLOOM_FILTER_CACHE = "QueryEngine-BloomFilterCache"; + private static final String QUERY_ENGINE_CHUNK_CACHE = "QueryEngine-ChunkCache"; + private static final String QUERY_ENGINE_TIME_SERIES_METADATA_CACHE = + "QueryEngine-TimeSeriesMetadataCache"; + private static final String QUERY_ENGINE_OPERATORS = "QueryEngine-Operators"; + private static final String QUERY_ENGINE_DATA_EXCHANGE = "QueryEngine-DataExchange"; + private static final String QUERY_ENGINE_TIME_INDEX = "QueryEngine-TimeIndex"; + private static final String QUERY_ENGINE_COORDINATOR = "QueryEngine-Coordinator"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(config.getAllocateMemoryForRead()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_BLOOM_FILTER_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForBloomFilterCache()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_CHUNK_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForChunkCache()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForTimeSeriesMetaDataCache()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_OPERATORS, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForOperators()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_DATA_EXCHANGE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForDataExchange()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_TIME_INDEX, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForTimeIndex()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + QUERY_ENGINE_COORDINATOR, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForCoordinator()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + QUERY_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + Arrays.asList( + QUERY_ENGINE_BLOOM_FILTER_CACHE, + QUERY_ENGINE_CHUNK_CACHE, + QUERY_ENGINE_TIME_SERIES_METADATA_CACHE, + QUERY_ENGINE_OPERATORS, + QUERY_ENGINE_DATA_EXCHANGE, + QUERY_ENGINE_TIME_INDEX, + QUERY_ENGINE_COORDINATOR) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2])); + } + + public static QueryEngineMemoryMetrics getInstance() { + return QueryEngineMemoryMetricsHolder.INSTANCE; + } + + private static class QueryEngineMemoryMetricsHolder { + + private static final QueryEngineMemoryMetrics INSTANCE = new QueryEngineMemoryMetrics(); + + private QueryEngineMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java new file mode 100644 index 000000000000..5b0757263637 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/SchemaEngineMemoryMetrics.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Arrays; + +public class SchemaEngineMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String SCHEMA_ENGINE = "SchemaEngine"; + private static final String SCHEMA_ENGINE_SCHEMA_REGION = "SchemaEngine-SchemaRegion"; + private static final String SCHEMA_ENGINE_SCHEMA_CACHE = "SchemaEngine-SchemaCache"; + private static final String SCHEMA_ENGINE_PARTITION_CACHE = "SchemaEngine-PartitionCache"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(config.getAllocateMemoryForSchema()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_SCHEMA_REGION, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForSchemaRegion()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_SCHEMA_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForSchemaCache()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + SCHEMA_ENGINE_PARTITION_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(config.getAllocateMemoryForPartitionCache()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + SCHEMA_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + Arrays.asList( + SCHEMA_ENGINE_SCHEMA_REGION, SCHEMA_ENGINE_SCHEMA_CACHE, SCHEMA_ENGINE_PARTITION_CACHE) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2])); + } + + public static SchemaEngineMemoryMetrics getInstance() { + return SchemaEngineMemoryMetricsHolder.INSTANCE; + } + + private static class SchemaEngineMemoryMetricsHolder { + + private static final SchemaEngineMemoryMetrics INSTANCE = new SchemaEngineMemoryMetrics(); + + private SchemaEngineMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java new file mode 100644 index 000000000000..2fd726272d53 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StorageEngineMemoryMetrics.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.Arrays; + +public class StorageEngineMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String STORAGE_ENGINE = "StorageEngine"; + private static final String STORAGE_ENGINE_WRITE = "StorageEngine-Write"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE = "StorageEngine-Write-Memtable"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE_CACHE = + "StorageEngine-Write-Memtable-DevicePathCache"; + private static final String STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS = + "StorageEngine-Write-Memtable-BufferedArrays"; + private static final String STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO = + "StorageEngine-Write-TimePartitionInfo"; + private static final String STORAGE_ENGINE_COMPACTION = "StorageEngine-Compaction"; + + @Override + public void bindTo(AbstractMetricService metricService) { + long storageEngineSize = config.getAllocateMemoryForStorageEngine(); + // Total memory size of storage engine + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(storageEngineSize); + // The memory of storage engine divided into Write and Compaction + long writeSize = + (long) + (config.getAllocateMemoryForStorageEngine() * (1 - config.getCompactionProportion())); + long compactionSize = storageEngineSize - writeSize; + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(writeSize); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_COMPACTION, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2]) + .set(compactionSize); + // The write memory of storage engine divided into MemTable and TimePartitionInfo + long memtableSize = + (long) + (config.getAllocateMemoryForStorageEngine() * config.getWriteProportionForMemtable()); + long timePartitionInfoSize = config.getAllocateMemoryForTimePartitionInfo(); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_MEMTABLE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[3]) + .set(memtableSize); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[3]) + .set(timePartitionInfoSize); + // The memtable memory of storage engine contain DataNodeDevicePathCache (NOTICE: This part of + // memory is not divided) + long dataNodeDevicePathCacheSize = + (long) + (config.getAllocateMemoryForStorageEngine() + * config.getWriteProportionForMemtable() + * config.getDevicePathCacheProportion()); + long bufferedArraySize = + (long) + (config.getAllocateMemoryForStorageEngine() + * config.getBufferedArraysMemoryProportion()); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[4]) + .set(dataNodeDevicePathCacheSize); + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[4]) + .set(bufferedArraySize); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + STORAGE_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + Arrays.asList(STORAGE_ENGINE_WRITE, STORAGE_ENGINE_COMPACTION) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[2])); + Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[3])); + Arrays.asList( + STORAGE_ENGINE_WRITE_MEMTABLE_CACHE, STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS) + .forEach( + name -> + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + name, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[4])); + } + + public static StorageEngineMemoryMetrics getInstance() { + return StorageEngineMemoryMetricsHolder.INSTANCE; + } + + private static class StorageEngineMemoryMetricsHolder { + + private static final StorageEngineMemoryMetrics INSTANCE = new StorageEngineMemoryMetrics(); + + private StorageEngineMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java new file mode 100644 index 000000000000..2fd8da49cf47 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/StreamEngineMemoryMetrics.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.memory; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +public class StreamEngineMemoryMetrics implements IMetricSet { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String STREAM_ENGINE = "StreamEngine"; + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.MEMORY_THRESHOLD_SIZE.toString(), + MetricLevel.NORMAL, + Tag.NAME.toString(), + STREAM_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]) + .set(config.getAllocateMemoryForPipe()); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.MEMORY_THRESHOLD_SIZE.toString(), + Tag.NAME.toString(), + STREAM_ENGINE, + Tag.TYPE.toString(), + GlobalMemoryMetrics.ON_HEAP, + Tag.LEVEL.toString(), + GlobalMemoryMetrics.LEVELS[1]); + } + + public static StreamEngineMemoryMetrics getInstance() { + return StreamEngineMemoryMetricsHolder.INSTANCE; + } + + private static class StreamEngineMemoryMetricsHolder { + + private static final StreamEngineMemoryMetrics INSTANCE = new StreamEngineMemoryMetrics(); + + private StreamEngineMemoryMetricsHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java deleted file mode 100644 index bd31e32f4b9d..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.service.metrics.memory; - -import org.apache.iotdb.commons.service.metric.enums.Metric; -import org.apache.iotdb.commons.service.metric.enums.Tag; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; -import org.apache.iotdb.metrics.AbstractMetricService; -import org.apache.iotdb.metrics.metricsets.IMetricSet; -import org.apache.iotdb.metrics.utils.MetricLevel; -import org.apache.iotdb.metrics.utils.MetricType; - -public class ThresholdMemoryMetrics implements IMetricSet { - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private static final SystemInfo systemInfo = SystemInfo.getInstance(); - - @Override - public void bindTo(AbstractMetricService metricService) { - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - "Total", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "0") - .set(Runtime.getRuntime().maxMemory()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - "StorageEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForStorageEngine()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - "QueryEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForRead()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - "SchemaEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForSchema()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - "Consensus", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForConsensus()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - "StreamEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1") - .set(config.getAllocateMemoryForPipe()); - metricService - .getOrCreateGauge( - Metric.THRESHOLD_MEMORY_SIZE.toString(), - MetricLevel.NORMAL, - Tag.NAME.toString(), - "DirectBuffer", - Tag.TYPE.toString(), - "OffHeap", - Tag.LEVEL.toString(), - "1") - .set(systemInfo.getTotalDirectBufferMemorySizeLimit()); - } - - @Override - public void unbindFrom(AbstractMetricService metricService) { - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "Total", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "0"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "StorageEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "QueryEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "SchemaEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "Consensus", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "StreamEngine", - Tag.TYPE.toString(), - "OnHeap", - Tag.LEVEL.toString(), - "1"); - metricService.remove( - MetricType.GAUGE, - Metric.THRESHOLD_MEMORY_SIZE.toString(), - Tag.NAME.toString(), - "DirectBuffer", - Tag.TYPE.toString(), - "OffHeap", - Tag.LEVEL.toString(), - "1"); - } - - public static ThresholdMemoryMetrics getInstance() { - return ThresholdMemoryMetrics.ThresholdMemoryMetricsHolder.INSTANCE; - } - - private static class ThresholdMemoryMetricsHolder { - - private static final ThresholdMemoryMetrics INSTANCE = new ThresholdMemoryMetrics(); - - private ThresholdMemoryMetricsHolder() {} - } -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 064a47f2b89a..906c9f05968e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -188,7 +188,8 @@ public enum Metric { MEMTABLE_POINT_COUNT("memtable_point_count"), BINARY_ALLOCATOR("binary_allocator"), // memory related - THRESHOLD_MEMORY_SIZE("threshold_memory_size"), + MEMORY_THRESHOLD_SIZE("memory_threshold_size"), + MEMORY_ACTUAL_SIZE("memory_actual_size"), ; final String value;