Merge branch '2404-iss-redisbigkey' into 'dev'

Resolve "redis bigkey 采集兼容性问题"

See merge request cloudcare-tools/datakit!3215

internal/plugins/inputs/redis/input.go

@@ -79,6 +79,7 @@ type Input struct {
 	LatencyPercentiles bool          `toml:"latency_percentiles"`
 	Slowlog            bool          `toml:"slow_log"`
 	AllSlowLog         bool          `toml:"all_slow_log"`
+	RedisCliPath       string        `toml:"redis_cli_path"`
 	Hotkey             bool          `toml:"hotkey"`
 	BigKey             bool          `toml:"bigkey"`
 	KeyInterval        time.Duration `toml:"key_interval"`
@@ -156,6 +157,10 @@ func (ipt *Input) initCfg() error {
 		ipt.InsecureSkipVerify,
 	)
 
+	if ipt.RedisCliPath == "" {
+		ipt.RedisCliPath = "redis-cli"
+	}
+
 	tlsCfg, err := ipt.TLSClientConfig.TLSConfigWithBase64()
 	if err != nil {
 		return err

internal/plugins/inputs/redis/metric_bigkey.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	"fmt"
 	"os/exec"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -215,7 +216,7 @@ func (ipt *Input) getBigData(ctxKey context.Context, db int) (string, error) {
 	// Official docs be wrong: https://redis.io/docs/connect/cli/
 	// Right example: redis-cli --hotkeys -i 0.1 -u redis://127.0.0.1:6379/0 --user username --pass password
 	u := "redis://" + ipt.Host + ":" + fmt.Sprint(ipt.Port) + "/" + fmt.Sprint(db)
-	args := []string{"redis-cli", "--bigkeys", "-i", ipt.KeyScanSleep, "-u", u}
+	args := []string{ipt.RedisCliPath, "--bigkeys", "-i", ipt.KeyScanSleep, "-u", u}
 	if ipt.Username != "" && ipt.Password != "" {
 		args = append(args, "--user", ipt.Username, "--pass", ipt.Password)
 	}
@@ -279,8 +280,8 @@ func (ipt *Input) parseBigData(data string, db int) ([]*point.Point, error) {
 		}
 
 		if strings.HasPrefix(line, "Biggest ") &&
-			strings.Contains(line, " found '\"") &&
-			strings.Contains(line, "\"' has ") {
+			strings.Contains(line, " found ") &&
+			strings.Contains(line, " has ") {
 			kv = getBigKey(line)
 			if v, ok := kv["key"]; ok {
 				message += " key: " + fmt.Sprint(v)
@@ -318,23 +319,18 @@ func getBigKey(line string) map[string]interface{} {
 	// Example: Biggest string found '"keySlice2999"' has 47984 bytes
 	// Example: Biggest   zset found '"keyZSet2999"' has 3001 members
 	kv := map[string]interface{}{}
-
-	line = strings.TrimPrefix(line, "Biggest ")
-	line = strings.ReplaceAll(line, "\"' has ", " found '\"")
-	parts := strings.Split(line, " found '\"")
-	if len(parts) != 3 {
-		return kv
-	}
-
-	bigKeyCounterStrs := strings.Split(parts[2], " ")
-	valueLength, err := strconv.Atoi(bigKeyCounterStrs[0])
-	if err != nil {
-		return kv
+	pattern := `Biggest\s+(\w+)\s+found\s+['"]*(.+?)['"]*\s+has\s+(\d+)\s+\w+`
+	re := regexp.MustCompile(pattern)
+	matches := re.FindStringSubmatch(line)
+	if len(matches) == 4 {
+		kv["key_type"] = matches[1]
+		kv["key"] = matches[2]
+		valueLength, err := strconv.Atoi(matches[3])
+		if err != nil {
+			return kv
+		}
+		kv["value_length"] = valueLength
 	}
 
-	kv["value_length"] = valueLength
-	kv["key"] = strings.Trim(parts[1], " ")
-	kv["key_type"] = strings.Trim(parts[0], " ")
-
 	return kv
 }

internal/plugins/inputs/redis/metric_bigkey_test.go

@@ -46,6 +46,23 @@ func TestInput_parseBigData(t *testing.T) {
 			wantErr: false,
 		},
 
+		{
+			name: "v5.0.7",
+			fields: fields{
+				mergedTags: map[string]string{"for": "bar"},
+			},
+			args: args{
+				data: mockBigData5_0_7,
+				db:   0,
+			},
+			want: []string{
+				"redis_bigkey,db_name=0,for=bar key=\"keyname\",key_type=\"hash\",message=\"big key  key: keyname key_type: hash value_length: 2\",status=\"unknown\",value_length=2i",
+				"redis_bigkey,db_name=0,for=bar key=\"mykey\",key_type=\"string\",message=\"big key  key: mykey key_type: string value_length: 13\",status=\"unknown\",value_length=13i",
+				"redis_bigkey,db_name=0,for=bar keys_sampled=2i,message=\"big key  keys_sampled: 2\",status=\"unknown\"",
+			},
+			wantErr: false,
+		},
+
 		{
 			name: "v6.0.8",
 			fields: fields{
@@ -142,6 +159,30 @@ Biggest   zset found '"keyZSet2999"' has 3001 members
 3 zsets with 3033 members (00.10% of keys, avg size 1011.00)
 `
 
+var mockBigData5_0_7 = `
+# Scanning the entire keyspace to find biggest keys as well as
+# average sizes per key type.  You can use -i 0.1 to sleep 0.1 sec
+# per 100 SCAN commands (not usually needed).
+
+[00.00%] Biggest hash   found so far 'keyname' with 2 fields
+[00.00%] Biggest string found so far 'mykey' with 13 bytes
+
+-------- summary -------
+
+Sampled 2 keys in the keyspace!
+Total key length in bytes is 12 (avg len 6.00)
+
+Biggest   hash found 'keyname' has 2 fields
+Biggest string found 'mykey' has 13 bytes
+
+0 lists with 0 items (00.00% of keys, avg size 0.00)
+1 hashs with 2 fields (50.00% of keys, avg size 2.00)
+1 strings with 13 bytes (50.00% of keys, avg size 13.00)
+0 streams with 0 entries (00.00% of keys, avg size 0.00)
+0 sets with 0 members (00.00% of keys, avg size 0.00)
+0 zsets with 0 members (00.00% of keys, avg size 0.00)
+`
+
 var mockBigData6_0_8 = `
 Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
 

internal/plugins/inputs/redis/metric_hotkey.go

@@ -107,7 +107,7 @@ func (ipt *Input) getHotData(ctxKey context.Context, db int) (string, error) {
 	// Official docs be wrong: https://redis.io/docs/connect/cli/
 	// Right example: redis-cli --hotkeys -i 0.1 -u redis://127.0.0.1:6379/0 --user username --pass password
 	u := "redis://" + ipt.Host + ":" + fmt.Sprint(ipt.Port) + "/" + fmt.Sprint(db)
-	args := []string{"redis-cli", "--hotkeys", "-i", ipt.KeyScanSleep, "-u", u}
+	args := []string{ipt.RedisCliPath, "--hotkeys", "-i", ipt.KeyScanSleep, "-u", u}
 	if ipt.Username != "" && ipt.Password != "" {
 		args = append(args, "--user", ipt.Username, "--pass", ipt.Password)
 	}

internal/plugins/inputs/redis/metric_redis_info.go

@@ -294,13 +294,20 @@ func (m *infoMeasurement) Info() *inputs.MeasurementInfo {
 			"client_longest_output_list": &inputs.FieldInfo{DataType: inputs.Float, Type: inputs.Gauge, Unit: inputs.NCount, Desc: "Longest output list among current client connections"},
 		},
 		Tags: map[string]interface{}{
-			"host":          &inputs.TagInfo{Desc: "Hostname."},
-			"redis_version": &inputs.TagInfo{Desc: "Version of the Redis server."},
-			"server":        &inputs.TagInfo{Desc: "Server addr."},
-			"service_name":  &inputs.TagInfo{Desc: "Service name."},
-			"command_type":  &inputs.TagInfo{Desc: "Command type."},
-			"error_type":    &inputs.TagInfo{Desc: "Error type."},
-			"quantile":      &inputs.TagInfo{Desc: "Histogram `quantile`."},
+			"host":             &inputs.TagInfo{Desc: "Hostname."},
+			"redis_version":    &inputs.TagInfo{Desc: "Version of the Redis server."},
+			"server":           &inputs.TagInfo{Desc: "Server addr."},
+			"service_name":     &inputs.TagInfo{Desc: "Service name."},
+			"command_type":     &inputs.TagInfo{Desc: "Command type."},
+			"error_type":       &inputs.TagInfo{Desc: "Error type."},
+			"quantile":         &inputs.TagInfo{Desc: "Histogram `quantile`."},
+			"role":             &inputs.TagInfo{Desc: "Value is `master` if the instance is replica of no one, or `slave` if the instance is a replica of some master instance."},
+			"redis_build_id":   &inputs.TagInfo{Desc: "Build ID of the Redis server."},
+			"redis_mode":       &inputs.TagInfo{Desc: "Mode of the Redis server."},
+			"os":               &inputs.TagInfo{Desc: "Operating system of the Redis server."},
+			"maxmemory_policy": &inputs.TagInfo{Desc: "The value of the maxmemory-policy configuration directive."},
+			"run_id":           &inputs.TagInfo{Desc: "Random value identifying the Redis server (to be used by Sentinel and Cluster)."},
+			"process_id":       &inputs.TagInfo{Desc: "Process ID of the Redis server."},
 		},
 	}
 }

internal/plugins/inputs/redis/sample.go

@@ -40,6 +40,10 @@ const (
   ## @param interval - number - optional - default: 15
   interval = "15s"
 
+  ## @param redis_cli_path - string - optional - default: "redis-cli"
+  ## If you want to use a custom redis-cli path for bigkey or hotkey, set this to the path of the redis-cli binary.
+  # redis_cli_path = "/usr/bin/redis-cli"
+
   ## @param hotkey - boolean - optional - default: false
   ## If you collet hotkey, set this to true
   # hotkey = false

scripts/glossary.txt

@@ -170,6 +170,7 @@ mitm
 mnodes
 mnodes
 mysqld
+maxmemory
 netflow
 netlog
 nginx