export profiling metrics

GuanceCloud · Sep 25, 2024 · aec3e26 · aec3e26
1 parent 0f0e626
commit aec3e26
Show file tree

Hide file tree

Showing 112 changed files with 22,026 additions and 581 deletions.
diff --git a/.gitignore b/.gitignore
@@ -109,3 +109,4 @@ internal/plugins/externals/ebpf/demo/
 internal/plugins/externals/ebpf/internal/testuitls/mysqlins/mysqlins
 internal/export/doc/zh/inputs/imgs/tracing.png
 /git
+/build
diff --git a/go.mod b/go.mod
@@ -355,6 +355,7 @@ require (
 	github.com/cilium/ebpf v0.11.0
 	github.com/gin-contrib/size v0.0.0-20231230013409-e0f46cc9c1db
 	github.com/google/gopacket v0.0.0-00010101000000-000000000000
+	github.com/grafana/jfr-parser v0.0.1
 	github.com/grafana/pyroscope/ebpf v0.2.1
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/ibmdb/go_ibm_db v0.4.4
@@ -372,6 +373,7 @@ require (
 )
 
 require (
+	github.com/GuanceCloud/zipstream v0.1.0 // indirect
 	github.com/VictoriaMetrics/easyproto v0.1.4 // indirect
 	github.com/avast/retry-go/v4 v4.1.0 // indirect
 	github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270 // indirect
@@ -389,6 +391,7 @@ require (
 replace (
 	github.com/c-bata/go-prompt => github.com/coanor/go-prompt v0.2.6
 	github.com/google/gopacket => github.com/GuanceCloud/gopacket v0.0.1
+	github.com/grafana/jfr-parser => github.com/GuanceCloud/jfr-parser v0.8.6
 	github.com/influxdata/influxdb1-client => github.com/GuanceCloud/influxdb1-client v0.1.8
 	github.com/iovisor/gobpf => github.com/DataDog/gobpf v0.0.0-20210322155958-9866ef4cd22c
 	github.com/kardianos/service => github.com/GuanceCloud/service v1.2.4

diff --git a/go.sum b/go.sum
@@ -143,8 +143,6 @@ github.com/DataDog/sketches-go v1.4.1 h1:j5G6as+9FASM2qC36lvpvQAj9qsv/jUs3FtO8Cw
 github.com/DataDog/sketches-go v1.4.1/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk=
 github.com/GuanceCloud/client_model v0.0.0-20230418154757-93bd4e878a5e h1:i34dA4kiRTfG+KdvkIXCLPDduarVeFlQhGDD3TefgS4=
 github.com/GuanceCloud/client_model v0.0.0-20230418154757-93bd4e878a5e/go.mod h1:PMnE48aPzuRu83FmWZugC0O3d54ZupJd/MmiaYxz8sM=
-github.com/GuanceCloud/cliutils v1.1.21-0.20240904042137-2a87297900d6 h1:hXvV/9i3aWkVVnpnHLngbpjZwTU+ut36YxOZOS2J4MM=
-github.com/GuanceCloud/cliutils v1.1.21-0.20240904042137-2a87297900d6/go.mod h1:Qbeedf/Ji3immd8Ka01NDQG6SP6j8JBnbZwsHTtxyqs=
 github.com/GuanceCloud/cliutils v1.1.21 h1:UkENug9Kg4GVTq1ITWIz2KmIPIvpNrZxKKUmRxWWFfA=
 github.com/GuanceCloud/cliutils v1.1.21/go.mod h1:5bIAZ9yA6l7W8MMUKw0+SIZJRpmEwxM6ZYLy4vweTgU=
 github.com/GuanceCloud/confd v0.1.101 h1:yjHgfl6YzAlTbFOFMTE4ERpFJzIyovOW7ZFc2/ZssL0=
@@ -159,6 +157,8 @@ github.com/GuanceCloud/grok v1.1.4 h1:+w/U5a54cgY0O+dvfcKc2qD3JuhmaS8Hi29BM4QMYt
 github.com/GuanceCloud/grok v1.1.4/go.mod h1:AHkJZYf7Qbo1FTZT6htdyScpICpgnkQ5+Hc0EmA88vM=
 github.com/GuanceCloud/influxdb1-client v0.1.8 h1:7XNICWcW+NxAHFkzQ8mkOCKA/8U2WNH5m+Hm9g0vz4k=
 github.com/GuanceCloud/influxdb1-client v0.1.8/go.mod h1:4HC4b/O653/ezBiHMPBnHYnHCCfsNT2LvCr7wNLngw4=
+github.com/GuanceCloud/jfr-parser v0.8.6 h1:kyiVxH5LcxNc1Xc3R9uSJz8f8RmBDhy9ytJrXCL6pn8=
+github.com/GuanceCloud/jfr-parser v0.8.6/go.mod h1:mngmZuDZbFhqGn2F+fK7tyxq+EmwvNZqWnQQ+heWmE4=
 github.com/GuanceCloud/kubernetes v0.0.0-20230801080916-ca299820872b h1:9pkl38Cro+7xCCruRvPh9z1L6DwX8xo2N4RDgHGUmtg=
 github.com/GuanceCloud/kubernetes v0.0.0-20230801080916-ca299820872b/go.mod h1:Acv+3eRHxCb4Qvs1YQcZ17X/D0H7DArQrew+WJtsLiE=
 github.com/GuanceCloud/mdcheck v0.0.0-20230718065937-44c6728c995f h1:0+A0eeT48LSlnDpVOQ/sqoW/lbYmerKKF7NVNBlgnww=
@@ -177,6 +177,8 @@ github.com/GuanceCloud/toml v1.2.5 h1:jBWfqFSVortEY0C4RYqFPvhDKcGxIosKzcQqTPtZMf
 github.com/GuanceCloud/toml v1.2.5/go.mod h1:D7S1XowYqOvMQdtsp2+lg2rKmO6RVuyekXJL+MzkD5Y=
 github.com/GuanceCloud/tracing-protos v0.0.0-20230619071516-54c8cff1b6b3 h1:+b+MkQrj/eJcODklzCSObp19TBycmfuooqCBD+89qmU=
 github.com/GuanceCloud/tracing-protos v0.0.0-20230619071516-54c8cff1b6b3/go.mod h1:5nclDehqFMaV8YMZzt1FuXz9/JRVKq0LYhmV2Djc1GU=
+github.com/GuanceCloud/zipstream v0.1.0 h1:RToNErercYk7y/nmyvshjN0Zt12lFNg2BpLh3YXXSNY=
+github.com/GuanceCloud/zipstream v0.1.0/go.mod h1:d5rjEl0N0ucmRRvrfX1+9JtsZZMYt5sWg9AR6pyTkCM=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/IBM/sarama v1.41.2 h1:ZDBZfGPHAD4uuAtSv4U22fRZBgst0eEwGFzLj0fb85c=
@@ -2079,8 +2081,6 @@ github.com/rosedblabs/wal v1.3.6/go.mod h1:wdq54KJUyVTOv1uddMc6Cdh2d/YCIo8yjcwJA
 github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
@@ -2959,8 +2959,6 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
 golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -3256,8 +3254,6 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
-google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/CodapeWild/dd-trace-go.v1 v1.35.18 h1:RvveUgTvm5M1oOJGIySvMJnKcXbCQ+dTjcTBuQTKbWY=

diff --git a/internal/export/doc/zh/datakit-operator.md b/internal/export/doc/zh/datakit-operator.md
@@ -469,12 +469,15 @@ spec:
       labels:
         app: movies-java
       annotations:
-        admission.datakit/java-profiler.version: "latest"
+        admission.datakit/java-profiler.version: "0.4.4"
     spec:
       containers:
         - name: movies-java
           image: zhangyicloud/movies-java:latest
           imagePullPolicy: IfNotPresent
+          securityContext:
+            seccompProfile:
+              type: Unconfined
           env:
             - name: JAVA_OPTS
               value: ""

diff --git a/internal/export/doc/zh/inputs/profile-go.md b/internal/export/doc/zh/inputs/profile-go.md
@@ -96,6 +96,44 @@ func demo() {
 
 运行该程序后，DDTrace 会定期（默认 1 分钟一次）将数据推送给 DataKit。
 
+### 生成性能指标 {#metrics}
+
+Datakit 自 [1.39.0](changelog.md#cl-1.39.0) 开始支持从 `dd-trace-go` 的输出中抽取一组 Go 运行时的相关指标，下面列举其中部分指标加以说明：
+
+| 指标名称                              | 说明                                                     | 单位         |
+|-----------------------------------|--------------------------------------------------------|------------|
+| prof_go_cpu_cores                 | 消耗 CPU 核心数                                             | core       |
+| prof_go_cpu_cores_gc_overhead     | 执行 GC 使用的 CPU 核心数                                      | core       |
+| prof_go_alloc_bytes_per_sec       | 每秒分配内存字节数大小                                            | byte       |
+| prof_go_frees_per_sec             | 每秒 GC 回收对象数                                            | count      |
+| prof_go_heap_growth_bytes_per_sec | 每秒堆内存增长大小                                              | byte       |
+| prof_go_allocs_per_sec            | 每秒执行内存分配次数                                             | count      |
+| prof_go_alloc_bytes_total         | 单次 profiling 持续期间（dd-trace 默认以 60 秒为一个采集周期，下同）分配的总内存大小 | byte       |
+| prof_go_blocked_time              | 单次 profiling 持续期间协程阻塞的总时长                              | nanosecond |
+| prof_go_mutex_delay_time          | 单次 profiling 持续期间用于等待锁所消耗的总时间                          | nanosecond |
+| prof_go_gcs_per_sec               | 每秒运行 GC 次数                                             | count      |
+| prof_go_max_gc_pause_time         | 单次 profiling 持续期间由于执行 GC 导致的程序中断的单次最长时长                | nanosecond |
+| prof_go_gc_pause_time             | 单次 profiling 持续期间由于执行 GC 导致的程序中断的总时长                   | nanosecond |
+| prof_go_num_goroutine             | 当前协程总数                                                 | count      |
+| prof_go_lifetime_heap_bytes       | 当前堆内存中存活对象占用的内存总大小                                     | byte       |
+| prof_go_lifetime_heap_objects     | 当前堆内存中存活的对象总数                                          | count      |
+
+
+<!-- markdownlint-disable MD046 -->
+???+ tips
+
+    该功能默认开启，如果不需要可以通过修改采集器的配置文件 `<DATAKIT_INSTALL_DIR\>/datakit/conf.d/profile/profile.conf` 把其中的配置项 `generate_metrics` 置为 false 并重启 Datakit.
+
+    ```toml
+    [[inputs.profile]]
+
+    ...
+
+    ## set false to stop generating apm metrics from ddtrace output.
+    generate_metrics = false
+    ```
+<!-- markdownlint-enable -->
+
 ## Pull 方式 {#pull-mode}
 
 ### Go 应用开启 Profiling {#app-config}
-Original file line number
+Diff line change
@@ Expand Up / @@ -109,3 +109,4 @@ internal/plugins/externals/ebpf/demo/ @@
     internal/plugins/externals/ebpf/internal/testuitls/mysqlins/mysqlins
     internal/export/doc/zh/inputs/imgs/tracing.png
     /git
+    /build