Add top stat info to mongodb plugin (influxdata#8861)

* Add top stat info * fixes after review * fix README.md
SCP-2000 · Mar 16, 2021 · 8a47d6f · 8a47d6f
1 parent c4a1260
commit 8a47d6f
Show file tree

Hide file tree

Showing 7 changed files with 268 additions and 46 deletions.
diff --git a/plugins/inputs/mongodb/README.md b/plugins/inputs/mongodb/README.md
@@ -21,6 +21,10 @@
 
   ## When true, collect per collection stats
   # gather_col_stats = false
+
+  ## When true, collect usage statistics for each collection
+  ## (insert, update, queries, remove, getmore, commands etc...).
+  # gather_top_stat = false
 
   ## List of db where collections stats are collected
   ## If empty, all db are concerned
@@ -263,6 +267,29 @@ by running Telegraf with the `--debug` argument.
     - available (integer)
     - created (integer)
     - refreshing (integer)
+
+- mongodb_top_stats
+  - tags:
+    - collection
+  - fields:
+    - total_time (integer)
+    - total_count (integer)
+    - read_lock_time (integer)
+    - read_lock_count (integer)
+    - write_lock_time (integer)
+    - write_lock_count (integer)
+    - queries_time (integer)
+    - queries_count (integer)
+    - get_more_time (integer)
+    - get_more_count (integer)
+    - insert_time (integer)
+    - insert_count (integer)
+    - update_time (integer)
+    - update_count (integer)
+    - remove_time (integer)
+    - remove_count (integer)
+    - commands_time (integer)
+    - commands_count (integer)
 
 ### Example Output:
 ```
@@ -272,4 +299,5 @@ mongodb_db_stats,db_name=admin,hostname=127.0.0.1:27017 avg_obj_size=241,collect
 mongodb_db_stats,db_name=local,hostname=127.0.0.1:27017 avg_obj_size=813.9705882352941,collections=6i,data_size=55350i,index_size=102400i,indexes=5i,num_extents=0i,objects=68i,ok=1i,storage_size=204800i,type="db_stat" 1547159491000000000
 mongodb_col_stats,collection=foo,db_name=local,hostname=127.0.0.1:27017 size=375005928i,avg_obj_size=5494,type="col_stat",storage_size=249307136i,total_index_size=2138112i,ok=1i,count=68251i 1547159491000000000
 mongodb_shard_stats,hostname=127.0.0.1:27017,in_use=3i,available=3i,created=4i,refreshing=0i 1522799074000000000
+mongodb_top_stats,collection=foo,total_time=1471,total_count=158,read_lock_time=49614,read_lock_count=657,write_lock_time=49125456,write_lock_count=9841,queries_time=174,queries_count=495,get_more_time=498,get_more_count=46,insert_time=2651,insert_count=1265,update_time=0,update_count=0,remove_time=0,remove_count=0,commands_time=498611,commands_count=4615
 ```
diff --git a/plugins/inputs/mongodb/mongodb.go b/plugins/inputs/mongodb/mongodb.go
@@ -23,6 +23,7 @@ type MongoDB struct {
 	GatherClusterStatus bool
 	GatherPerdbStats    bool
 	GatherColStats      bool
+	GatherTopStat       bool
 	ColStatsDbs         []string
 	tlsint.ClientConfig
 
@@ -53,6 +54,10 @@ var sampleConfig = `
   ## When true, collect per collection stats
   # gather_col_stats = false
 
+  ## When true, collect usage statistics for each collection
+  ## (insert, update, queries, remove, getmore, commands etc...).
+  # gather_top_stat = false
+
   ## List of db where collections stats are collected
   ## If empty, all db are concerned
   # col_stats_dbs = ["local"]
@@ -183,7 +188,7 @@ func (m *MongoDB) gatherServer(server *Server, acc telegraf.Accumulator) error {
 		}
 		server.Session = sess
 	}
-	return server.gatherData(acc, m.GatherClusterStatus, m.GatherPerdbStats, m.GatherColStats, m.ColStatsDbs)
+	return server.gatherData(acc, m.GatherClusterStatus, m.GatherPerdbStats, m.GatherColStats, m.GatherTopStat, m.ColStatsDbs)
 }
 
 func init() {
@@ -193,6 +198,7 @@ func init() {
 			GatherClusterStatus: true,
 			GatherPerdbStats:    false,
 			GatherColStats:      false,
+			GatherTopStat:       false,
 			ColStatsDbs:         []string{"local"},
 		}
 	})

diff --git a/plugins/inputs/mongodb/mongodb_data.go b/plugins/inputs/mongodb/mongodb_data.go
@@ -15,6 +15,7 @@ type MongodbData struct {
 	DbData        []DbData
 	ColData       []ColData
 	ShardHostData []DbData
+	TopStatsData  []DbData
 }
 
 type DbData struct {
@@ -37,7 +38,7 @@ func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData {
 	}
 }
 
-var DefaultStats = map[string]string{
+var defaultStats = map[string]string{
 	"uptime_ns":                 "UptimeNanos",
 	"inserts":                   "InsertCnt",
 	"inserts_per_sec":           "Insert",
@@ -94,15 +95,15 @@ var DefaultStats = map[string]string{
 	"total_docs_scanned":        "TotalObjectsScanned",
 }
 
-var DefaultAssertsStats = map[string]string{
+var defaultAssertsStats = map[string]string{
 	"assert_regular":   "Regular",
 	"assert_warning":   "Warning",
 	"assert_msg":       "Msg",
 	"assert_user":      "User",
 	"assert_rollovers": "Rollovers",
 }
 
-var DefaultCommandsStats = map[string]string{
+var defaultCommandsStats = map[string]string{
 	"aggregate_command_total":        "AggregateCommandTotal",
 	"aggregate_command_failed":       "AggregateCommandFailed",
 	"count_command_total":            "CountCommandTotal",
@@ -123,7 +124,7 @@ var DefaultCommandsStats = map[string]string{
 	"update_command_failed":          "UpdateCommandFailed",
 }
 
-var DefaultLatencyStats = map[string]string{
+var defaultLatencyStats = map[string]string{
 	"latency_writes_count":   "WriteOpsCnt",
 	"latency_writes":         "WriteLatency",
 	"latency_reads_count":    "ReadOpsCnt",
@@ -132,7 +133,7 @@ var DefaultLatencyStats = map[string]string{
 	"latency_commands":       "CommandLatency",
 }
 
-var DefaultReplStats = map[string]string{
+var defaultReplStats = map[string]string{
 	"repl_inserts":                             "InsertRCnt",
 	"repl_inserts_per_sec":                     "InsertR",
 	"repl_queries":                             "QueryRCnt",
@@ -164,37 +165,37 @@ var DefaultReplStats = map[string]string{
 	"repl_executor_unsignaled_events":          "ReplExecutorUnsignaledEvents",
 }
 
-var DefaultClusterStats = map[string]string{
+var defaultClusterStats = map[string]string{
 	"jumbo_chunks": "JumboChunksCount",
 }
 
-var DefaultShardStats = map[string]string{
+var defaultShardStats = map[string]string{
 	"total_in_use":     "TotalInUse",
 	"total_available":  "TotalAvailable",
 	"total_created":    "TotalCreated",
 	"total_refreshing": "TotalRefreshing",
 }
 
-var ShardHostStats = map[string]string{
+var shardHostStats = map[string]string{
 	"in_use":     "InUse",
 	"available":  "Available",
 	"created":    "Created",
 	"refreshing": "Refreshing",
 }
 
-var MmapStats = map[string]string{
+var mmapStats = map[string]string{
 	"mapped_megabytes":     "Mapped",
 	"non-mapped_megabytes": "NonMapped",
 	"page_faults":          "FaultsCnt",
 	"page_faults_per_sec":  "Faults",
 }
 
-var WiredTigerStats = map[string]string{
+var wiredTigerStats = map[string]string{
 	"percent_cache_dirty": "CacheDirtyPercent",
 	"percent_cache_used":  "CacheUsedPercent",
 }
 
-var WiredTigerExtStats = map[string]string{
+var wiredTigerExtStats = map[string]string{
 	"wtcache_tracked_dirty_bytes":          "TrackedDirtyBytes",
 	"wtcache_current_bytes":                "CurrentCachedBytes",
 	"wtcache_max_bytes_configured":         "MaxBytesConfigured",
@@ -215,7 +216,7 @@ var WiredTigerExtStats = map[string]string{
 	"wtcache_unmodified_pages_evicted":     "UnmodifiedPagesEvicted",
 }
 
-var DefaultTCMallocStats = map[string]string{
+var defaultTCMallocStats = map[string]string{
 	"tcmalloc_current_allocated_bytes":          "TCMallocCurrentAllocatedBytes",
 	"tcmalloc_heap_size":                        "TCMallocHeapSize",
 	"tcmalloc_central_cache_free_bytes":         "TCMallocCentralCacheFreeBytes",
@@ -237,13 +238,13 @@ var DefaultTCMallocStats = map[string]string{
 	"tcmalloc_pageheap_total_reserve_bytes":     "TCMallocPageheapTotalReserveBytes",
 }
 
-var DefaultStorageStats = map[string]string{
+var defaultStorageStats = map[string]string{
 	"storage_freelist_search_bucket_exhausted": "StorageFreelistSearchBucketExhausted",
 	"storage_freelist_search_requests":         "StorageFreelistSearchRequests",
 	"storage_freelist_search_scanned":          "StorageFreelistSearchScanned",
 }
 
-var DbDataStats = map[string]string{
+var dbDataStats = map[string]string{
 	"collections":  "Collections",
 	"objects":      "Objects",
 	"avg_obj_size": "AvgObjSize",
@@ -255,7 +256,7 @@ var DbDataStats = map[string]string{
 	"ok":           "Ok",
 }
 
-var ColDataStats = map[string]string{
+var colDataStats = map[string]string{
 	"count":            "Count",
 	"size":             "Size",
 	"avg_obj_size":     "AvgObjSize",
@@ -264,6 +265,27 @@ var ColDataStats = map[string]string{
 	"ok":               "Ok",
 }
 
+var topDataStats = map[string]string{
+	"total_time":       "TotalTime",
+	"total_count":      "TotalCount",
+	"read_lock_time":   "ReadLockTime",
+	"read_lock_count":  "ReadLockCount",
+	"write_lock_time":  "WriteLockTime",
+	"write_lock_count": "WriteLockCount",
+	"queries_time":     "QueriesTime",
+	"queries_count":    "QueriesCount",
+	"get_more_time":    "GetMoreTime",
+	"get_more_count":   "GetMoreCount",
+	"insert_time":      "InsertTime",
+	"insert_count":     "InsertCount",
+	"update_time":      "UpdateTime",
+	"update_count":     "UpdateCount",
+	"remove_time":      "RemoveTime",
+	"remove_count":     "RemoveCount",
+	"commands_time":    "CommandsTime",
+	"commands_count":   "CommandsCount",
+}
+
 func (d *MongodbData) AddDbStats() {
 	for _, dbstat := range d.StatLine.DbStatsLines {
 		dbStatLine := reflect.ValueOf(&dbstat).Elem()
@@ -272,7 +294,7 @@ func (d *MongodbData) AddDbStats() {
 			Fields: make(map[string]interface{}),
 		}
 		newDbData.Fields["type"] = "db_stat"
-		for key, value := range DbDataStats {
+		for key, value := range dbDataStats {
 			val := dbStatLine.FieldByName(value).Interface()
 			newDbData.Fields[key] = val
 		}
@@ -289,7 +311,7 @@ func (d *MongodbData) AddColStats() {
 			Fields: make(map[string]interface{}),
 		}
 		newColData.Fields["type"] = "col_stat"
-		for key, value := range ColDataStats {
+		for key, value := range colDataStats {
 			val := colStatLine.FieldByName(value).Interface()
 			newColData.Fields[key] = val
 		}
@@ -305,24 +327,40 @@ func (d *MongodbData) AddShardHostStats() {
 			Fields: make(map[string]interface{}),
 		}
 		newDbData.Fields["type"] = "shard_host_stat"
-		for k, v := range ShardHostStats {
+		for k, v := range shardHostStats {
 			val := hostStatLine.FieldByName(v).Interface()
 			newDbData.Fields[k] = val
 		}
 		d.ShardHostData = append(d.ShardHostData, *newDbData)
 	}
 }
 
+func (d *MongodbData) AddTopStats() {
+	for _, topStat := range d.StatLine.TopStatLines {
+		topStatLine := reflect.ValueOf(&topStat).Elem()
+		newTopStatData := &DbData{
+			Name:   topStat.CollectionName,
+			Fields: make(map[string]interface{}),
+		}
+		newTopStatData.Fields["type"] = "top_stat"
+		for key, value := range topDataStats {
+			val := topStatLine.FieldByName(value).Interface()
+			newTopStatData.Fields[key] = val
+		}
+		d.TopStatsData = append(d.TopStatsData, *newTopStatData)
+	}
+}
+
 func (d *MongodbData) AddDefaultStats() {
 	statLine := reflect.ValueOf(d.StatLine).Elem()
-	d.addStat(statLine, DefaultStats)
+	d.addStat(statLine, defaultStats)
 	if d.StatLine.NodeType != "" {
-		d.addStat(statLine, DefaultReplStats)
+		d.addStat(statLine, defaultReplStats)
 		d.Tags["node_type"] = d.StatLine.NodeType
 	}
 
 	if d.StatLine.ReadLatency > 0 {
-		d.addStat(statLine, DefaultLatencyStats)
+		d.addStat(statLine, defaultLatencyStats)
 	}
 
 	if d.StatLine.ReplSetName != "" {
@@ -337,23 +375,23 @@ func (d *MongodbData) AddDefaultStats() {
 		d.add("version", d.StatLine.Version)
 	}
 
-	d.addStat(statLine, DefaultAssertsStats)
-	d.addStat(statLine, DefaultClusterStats)
-	d.addStat(statLine, DefaultCommandsStats)
-	d.addStat(statLine, DefaultShardStats)
-	d.addStat(statLine, DefaultStorageStats)
-	d.addStat(statLine, DefaultTCMallocStats)
+	d.addStat(statLine, defaultAssertsStats)
+	d.addStat(statLine, defaultClusterStats)
+	d.addStat(statLine, defaultCommandsStats)
+	d.addStat(statLine, defaultShardStats)
+	d.addStat(statLine, defaultStorageStats)
+	d.addStat(statLine, defaultTCMallocStats)
 
 	if d.StatLine.StorageEngine == "mmapv1" || d.StatLine.StorageEngine == "rocksdb" {
-		d.addStat(statLine, MmapStats)
+		d.addStat(statLine, mmapStats)
 	} else if d.StatLine.StorageEngine == "wiredTiger" {
-		for key, value := range WiredTigerStats {
+		for key, value := range wiredTigerStats {
 			val := statLine.FieldByName(value).Interface()
 			percentVal := fmt.Sprintf("%.1f", val.(float64)*100)
 			floatVal, _ := strconv.ParseFloat(percentVal, 64)
 			d.add(key, floatVal)
 		}
-		d.addStat(statLine, WiredTigerExtStats)
+		d.addStat(statLine, wiredTigerExtStats)
 		d.add("page_faults", d.StatLine.FaultsCnt)
 	}
 }
@@ -409,4 +447,14 @@ func (d *MongodbData) flush(acc telegraf.Accumulator) {
 		)
 		host.Fields = make(map[string]interface{})
 	}
+	for _, col := range d.TopStatsData {
+		d.Tags["collection"] = col.Name
+		acc.AddFields(
+			"mongodb_top_stats",
+			col.Fields,
+			d.Tags,
+			d.StatLine.Time,
+		)
+		col.Fields = make(map[string]interface{})
+	}
 }