[feat]curvefs/metaserver: add delete inode monitor

Signed-off-by: Cyber-SiKu <[email protected]>
opencurve · Oct 19, 2023 · 6b1964a · 6b1964a
1 parent 712fa17
commit 6b1964a
Show file tree

Hide file tree

Showing 13 changed files with 1,609 additions and 860 deletions.
diff --git a/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json b/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json
diff --git a/curvefs/src/client/metric/client_metric.cpp b/curvefs/src/client/metric/client_metric.cpp
@@ -23,11 +23,14 @@
 #include "curvefs/src/client/metric/client_metric.h"
 
 #include <memory>
+#include "src/client/client_metric.h"
 
 namespace curvefs {
 namespace client {
 namespace metric {
 
+using curve::client::CollectMetrics;
+
 const std::string MDSClientMetric::prefix = "curvefs_mds_client";  // NOLINT
 const std::string MetaServerClientMetric::prefix = "curvefs_metaserver_client";  // NOLINT
 const std::string ClientOpMetric::prefix = "curvefs_client";  // NOLINT
@@ -39,12 +42,6 @@ const std::string KVClientMetric::prefix = "curvefs_kvclient";  // NOLINT
 const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info";  // NOLINT
 const std::string WarmupManagerS3Metric::prefix = "curvefs_warmup";   // NOLINT
 
-void CollectMetrics(InterfaceMetric* interface, int count, uint64_t u_elapsed) {
-    interface->bps.count << count;
-    interface->qps.count << 1;
-    interface->latency << u_elapsed;
-}
-
 void AsyncContextCollectMetrics(
     std::shared_ptr<S3Metric> s3Metric,
     const std::shared_ptr<curve::common::PutObjectAsyncContext>& context) {

diff --git a/curvefs/src/client/metric/client_metric.h b/curvefs/src/client/metric/client_metric.h
@@ -332,8 +332,6 @@ struct WarmupManagerS3Metric {
           warmupS3CacheSize(prefix, "s3_cache_size") {}
 };
 
-void CollectMetrics(InterfaceMetric* interface, int count, uint64_t start);
-
 void AsyncContextCollectMetrics(
     std::shared_ptr<S3Metric> s3Metric,
     const std::shared_ptr<curve::common::PutObjectAsyncContext>& context);

diff --git a/curvefs/src/client/s3/client_s3_adaptor.cpp b/curvefs/src/client/s3/client_s3_adaptor.cpp
@@ -145,7 +145,7 @@ int S3ClientAdaptorImpl::Write(uint64_t inodeId, uint64_t offset,
     int ret = fileCacheManager->Write(offset, length, buf);
     fsCacheManager_->DataCacheByteDec(length);
     if (s3Metric_ != nullptr) {
-        metric::CollectMetrics(&s3Metric_->adaptorWrite, ret,
+        curve::client::CollectMetrics(&s3Metric_->adaptorWrite, ret,
                                butil::cpuwide_time_us() - start);
         s3Metric_->writeSize.set_value(length);
     }
@@ -167,8 +167,8 @@ int S3ClientAdaptorImpl::Read(uint64_t inodeId, uint64_t offset,
         return ret;
     }
     if (s3Metric_.get() != nullptr) {
-        metric::CollectMetrics(&s3Metric_->adaptorRead, ret,
-                               butil::cpuwide_time_us() - start);
+        curve::client::CollectMetrics(&s3Metric_->adaptorRead, ret,
+                                         butil::cpuwide_time_us() - start);
         s3Metric_->readSize.set_value(length);
     }
     VLOG(6) << "read end offset:" << offset << ", len:" << length

diff --git a/curvefs/src/client/s3/client_s3_cache_manager.cpp b/curvefs/src/client/s3/client_s3_cache_manager.cpp
@@ -496,10 +496,12 @@ bool FileCacheManager::ReadKVRequestFromLocalCache(const std::string &name,
     }
 
     if (s3ClientAdaptor_->s3Metric_) {
-        metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->adaptorReadS3, len,
-                               butil::cpuwide_time_us() - start);
-        metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->readFromDiskCache,
-                               len, butil::cpuwide_time_us() - start);
+        curve::client::CollectMetrics(
+            &s3ClientAdaptor_->s3Metric_->adaptorReadS3, len,
+            butil::cpuwide_time_us() - start);
+        curve::client::CollectMetrics(
+            &s3ClientAdaptor_->s3Metric_->readFromDiskCache, len,
+            butil::cpuwide_time_us() - start);
     }
     return true;
 }
@@ -515,7 +517,7 @@ bool FileCacheManager::ReadKVRequestFromRemoteCache(const std::string &name,
     CountDownEvent event(1);
     GetKVCacheDone cb = [&](const std::shared_ptr<GetKVCacheTask>& task) {
         if (task->res && s3ClientAdaptor_->s3Metric_ != nullptr) {
-            metric::CollectMetrics(
+            curve::client::CollectMetrics(
                 &s3ClientAdaptor_->s3Metric_->readFromKVCache, task->length,
                 task->timer.u_elapsed());
         }
@@ -543,10 +545,11 @@ bool FileCacheManager::ReadKVRequestFromS3(const std::string &name,
     }
 
     if (s3ClientAdaptor_->s3Metric_) {
-        metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->adaptorReadS3,
-                               length, butil::cpuwide_time_us() - start);
-        metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->readFromS3, length,
-                               butil::cpuwide_time_us() - start);
+        curve::client::CollectMetrics(
+            &s3ClientAdaptor_->s3Metric_->adaptorReadS3, length,
+            butil::cpuwide_time_us() - start);
+        curve::client::CollectMetrics(&s3ClientAdaptor_->s3Metric_->readFromS3,
+                                      length, butil::cpuwide_time_us() - start);
     }
 
     return true;
@@ -736,9 +739,9 @@ class AsyncPrefetchCallback {
         }
         if (s3Client_->s3Metric_ != nullptr) {
             // prefetch to disk
-            metric::CollectMetrics(&s3Client_->s3Metric_->writeToDiskCache,
-                                   context->actualLen,
-                                   butil::cpuwide_time_us() - start);
+            curve::client::CollectMetrics(
+                &s3Client_->s3Metric_->writeToDiskCache, context->actualLen,
+                butil::cpuwide_time_us() - start);
         }
         {
             curve::common::LockGuard lg(fileCache->downloadMtx_);
@@ -2369,7 +2372,7 @@ CURVEFS_ERROR DataCache::PrepareFlushTasks(
                 [this](const std::shared_ptr<SetKVCacheTask>& setTask) {
                     if (setTask->res &&
                         s3ClientAdaptor_->s3Metric_ != nullptr) {
-                        metric::CollectMetrics(
+                        curve::client::CollectMetrics(
                             &s3ClientAdaptor_->s3Metric_->writeToKVCache,
                             setTask->length, setTask->timer.u_elapsed());
                     }
@@ -2448,8 +2451,9 @@ void DataCache::FlushTaskExecute(
     SetKVCacheDone kvdone = [&](const std::shared_ptr<SetKVCacheTask> &task) {
         kvTaskEvent.Signal();
         if (task->res && s3ClientAdaptor_->s3Metric_ != nullptr) {
-            metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->writeToKVCache,
-                                   task->length, task->timer.u_elapsed());
+            curve::client::CollectMetrics(
+                &s3ClientAdaptor_->s3Metric_->writeToKVCache, task->length,
+                task->timer.u_elapsed());
         }
         return;
     };

diff --git a/curvefs/src/client/s3/disk_cache_manager.cpp b/curvefs/src/client/s3/disk_cache_manager.cpp
@@ -475,7 +475,7 @@ void DiskCacheManager::TrimCache() {
                         << "error is: " << errno;
                     continue;
                 }
-                metric::CollectMetrics(&metric_->trim_, statReadFile.st_size,
+                curve::client::CollectMetrics(&metric_->trim_, statReadFile.st_size,
                                        butil::cpuwide_time_us() - start);
                 UpdateDiskUsedBytes(-statReadFile.st_size);
                 VLOG(6) << "remove disk file success, file is: " << cacheKey;

diff --git a/curvefs/src/client/s3/disk_cache_manager_impl.cpp b/curvefs/src/client/s3/disk_cache_manager_impl.cpp
@@ -183,8 +183,9 @@ int DiskCacheManagerImpl::Read(const std::string name, char *buf,
             LOG(ERROR) << "download object fail. object name = " << name;
             return ret;
         }
-        metric::CollectMetrics(&diskCacheManager_->GetS3Metric()->readFromS3,
-                               length, butil::cpuwide_time_us() - start);
+        curve::client::CollectMetrics(
+            &diskCacheManager_->GetS3Metric()->readFromS3, length,
+            butil::cpuwide_time_us() - start);
     }
     VLOG(9) << "read success, read name = " << name;
     return ret;

diff --git a/curvefs/src/client/s3/disk_cache_write.cpp b/curvefs/src/client/s3/disk_cache_write.cpp
@@ -143,7 +143,7 @@ int DiskCacheWrite::UploadFile(const std::string &name,
          name](const std::shared_ptr<PutObjectAsyncContext>& context) {
             if (context->retCode >= 0) {
                 if (metric_ != nullptr) {
-                    metric::CollectMetrics(&metric_->writeS3,
+                    curve::client::CollectMetrics(&metric_->writeS3,
                                            context->bufferSize,
                                            context->timer.u_elapsed());
                 }
@@ -403,7 +403,7 @@ int DiskCacheWrite::UploadAllCacheWriteFile() {
             [&, buffer](const std::shared_ptr<PutObjectAsyncContext>& context) {
                 if (context->retCode >= 0) {
                     if (s3Metric_ != nullptr) {
-                        metric::CollectMetrics(&metric_->writeS3,
+                        curve::client::CollectMetrics(&metric_->writeS3,
                                                context->bufferSize,
                                                context->timer.u_elapsed());
                         metric::AsyncContextCollectMetrics(s3Metric_, context);

diff --git a/curvefs/src/client/warmup/warmup_manager.cpp b/curvefs/src/client/warmup/warmup_manager.cpp
@@ -536,9 +536,9 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
             if (context->retCode >= 0) {
                 VLOG(9) << "Get Object success: " << context->key;
                 PutObjectToCache(key, context);
-                metric::CollectMetrics(&warmupS3Metric_.warmupS3Cached,
-                                       context->len,
-                                       butil::cpuwide_time_us() - start);
+                curve::client::CollectMetrics(&warmupS3Metric_.warmupS3Cached,
+                                              context->len,
+                                              butil::cpuwide_time_us() - start);
                 warmupS3Metric_.warmupS3CacheSize << context->len;
                 if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
                     VLOG(6) << "pendingReq is over";

diff --git a/curvefs/src/metaserver/s3/metaserver_s3_adaptor.cpp b/curvefs/src/metaserver/s3/metaserver_s3_adaptor.cpp
@@ -21,12 +21,21 @@
  */
 
 #include "curvefs/src/metaserver/s3/metaserver_s3_adaptor.h"
-#include <list>
+
 #include <algorithm>
+#include <cstdint>
+#include <list>
+#include <string>
+
 #include "curvefs/src/common/s3util.h"
+#include "src/client/client_metric.h"
 
 namespace curvefs {
 namespace metaserver {
+
+const std::string S3ClientAdaptorMetric::prefix =  // NOLINT
+    "curvefs_metaserver_s3_client_adaptor";        // NOLINT
+
 void S3ClientAdaptorImpl::Init(const S3ClientAdaptorOption &option,
                                S3Client *client) {
     blockSize_ = option.blockSize;
@@ -48,12 +57,21 @@ void S3ClientAdaptorImpl::Reinit(const S3ClientAdaptorOption& option,
     client_->Reinit(ak, sk, endpoint, bucketName);
 }
 
-int S3ClientAdaptorImpl::Delete(const Inode &inode) {
+int S3ClientAdaptorImpl::Delete(const Inode& inode) {
+    int ret = 0;
+    uint64_t start = butil::cpuwide_time_us();
     if (enableBatchDelete_) {
-        return DeleteInodeByDeleteBatchChunk(inode);
+        ret =  DeleteInodeByDeleteBatchChunk(inode);
     } else {
-        return DeleteInodeByDeleteSingleChunk(inode);
+        ret = DeleteInodeByDeleteSingleChunk(inode);
     }
+    if (ret == 0) {
+        curve::client::CollectMetrics(&metric_->deleteInode, inode.length(),
+                                      butil::cpuwide_time_us() - start);
+    } else {
+        metric_->deleteInode.eps.count << 1;
+    }
+    return ret;
 }
 
 int S3ClientAdaptorImpl::DeleteInodeByDeleteSingleChunk(const Inode &inode) {

diff --git a/curvefs/src/metaserver/s3/metaserver_s3_adaptor.h b/curvefs/src/metaserver/s3/metaserver_s3_adaptor.h
@@ -23,10 +23,14 @@
 #ifndef CURVEFS_SRC_METASERVER_S3_METASERVER_S3_ADAPTOR_H_
 #define CURVEFS_SRC_METASERVER_S3_METASERVER_S3_ADAPTOR_H_
 
-#include <string>
 #include <list>
+#include <memory>
+#include <string>
+
+#include "absl/memory/memory.h"
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/metaserver/s3/metaserver_s3.h"
+#include "src/client/client_metric.h"
 
 namespace curvefs {
 namespace metaserver {
@@ -78,9 +82,20 @@ class S3ClientAdaptor {
     virtual void GetS3ClientAdaptorOption(S3ClientAdaptorOption *option) = 0;
 };
 
+struct S3ClientAdaptorMetric {
+    static const std::string prefix;
+
+    curve::client::InterfaceMetric deleteInode;
+
+    S3ClientAdaptorMetric() : deleteInode(prefix, "_delete_inode") {}
+};
+
 class S3ClientAdaptorImpl : public S3ClientAdaptor {
  public:
-    S3ClientAdaptorImpl() {}
+    S3ClientAdaptorImpl() {
+        metric_ = absl::make_unique<S3ClientAdaptorMetric>();
+    }
+
     ~S3ClientAdaptorImpl() {
         if (client_ != nullptr) {
             delete client_;
@@ -151,6 +166,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
     uint64_t batchSize_;
     uint32_t objectPrefix_;
     bool enableBatchDelete_;
+    std::unique_ptr<S3ClientAdaptorMetric> metric_;
 };
 }  // namespace metaserver
 }  // namespace curvefs

diff --git a/src/client/client_metric.cpp b/src/client/client_metric.cpp
@@ -0,0 +1,34 @@
+/*
+*  Copyright (c) 2023 NetEase Inc.
+*
+*  Licensed under the Apache License, Version 2.0 (the "License");
+*  you may not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*/
+/*
+ * Project: curve
+ * Created Date: 2023-10-19
+ * Author: chengyi01
+ */
+
+#include "src/client/client_metric.h"
+
+namespace curve {
+namespace client {
+
+void CollectMetrics(InterfaceMetric* interface, int count, uint64_t u_elapsed) {
+    interface->bps.count << count;
+    interface->qps.count << 1;
+    interface->latency << u_elapsed;
+}
+
+}  // namespace client
+}  // namespace curve
diff --git a/src/client/client_metric.h b/src/client/client_metric.h
@@ -87,6 +87,8 @@ struct InterfaceMetric {
           latency(prefix, name + "_lat") {}
 };
 
+void CollectMetrics(InterfaceMetric* interface, int count, uint64_t u_elapsed);
+
 struct DiscardMetric {
     explicit DiscardMetric(const std::string& prefix)
         : totalSuccess(prefix, "discard_total_success"),