【腾讯犀牛鸟开源课题实战】prometheus插件专项建设

CMakeLists.txt

@@ -158,6 +158,7 @@ include(snappy)
 include(lz4)
 include(toml11)
 include(flatbuffers)
+include(jwt_cpp)
 
 #---------------------------------------------------------------------------------------
 # Set complie options and include other libs if options are ON

LICENSE

@@ -454,6 +454,21 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 As an exception, if, as a result of your compiling your source code, portions of this Software are embedded into a machine-executable object form of such source code, you may redistribute such embedded portions in such object form without including the above copyright and permission notices.
 
 
+Open Source Software Licensed under the MIT with exceptions:
+--------------------------------------------------------------------
+1. jwt-cpp
+Copyright (c) 2018 Dominik Thalhammer
+
+
+Terms of the MIT with exceptions:
+--------------------------------------------------------------------
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 Open Source Software Licensed under the MIT:
 --------------------------------------------------------------------
 1. murmurhash3

cmake/config/trpc_config.cmake

@@ -55,7 +55,8 @@ set(INCLUDE_PATHS   ${TRPC_ROOT_PATH}
                     ${TRPC_ROOT_PATH}/cmake_third_party/nghttp2/lib/includes
                     ${TRPC_ROOT_PATH}/cmake_third_party/picohttpparser
                     ${TRPC_ROOT_PATH}/cmake_third_party/snappy
-                    ${TRPC_ROOT_PATH}/cmake_third_party/lz4)
+                    ${TRPC_ROOT_PATH}/cmake_third_party/lz4
+                    ${TRPC_ROOT_PATH}/cmake_third_party/jwt_cpp/include)
 
 # When use tRPC as a third-party library, selectively inject the header files at including any-lib.cmake.
 set(TARGET_INCLUDE_PATHS  ${TRPC_ROOT_PATH})
@@ -117,4 +118,5 @@ set(LIB_SSL ssl crypto)
 
 set(LIB_METRICS_PROMETHEUS prometheus-cpp-core
                            prometheus-cpp-pull
-                           prometheus-cpp-push)
+                           prometheus-cpp-push
+                           jwt-cpp)

cmake/jwt_cpp.cmake

@@ -0,0 +1,38 @@
+#
+#
+# Tencent is pleased to support the open source community by making tRPC available.
+#
+# Copyright (C) 2023 THL A29 Limited, a Tencent company.
+# All rights reserved.
+#
+# If you have downloaded a copy of the tRPC source code from Tencent,
+# please note that tRPC source code is licensed under the  Apache 2.0 License,
+# A copy of the Apache 2.0 License is included in this file.
+#
+#
+
+include(FetchContent)
+
+if(NOT DEFINED JWT_VET)
+  set(JWT_VET 0.7.0)
+endif()
+
+set(JWT_URL https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v${JWT_VET}.tar.gz)
+
+FetchContent_Declare(
+    jwt_cpp
+    URL             ${JWT_URL}
+    SOURCE_DIR      ${TRPC_ROOT_PATH}/cmake_third_party/jwt_cpp
+)
+
+FetchContent_GetProperties(jwt_cpp)
+
+if(NOT jwt_cpp_POPULATED)
+    FetchContent_Populate(jwt_cpp)
+    add_subdirectory(${TRPC_ROOT_PATH}/cmake_third_party/jwt_cpp)
+    add_library(trpc_jwt_cpp ALIAS jwt-cpp)
+    set(TARGET_INCLUDE_PATHS    ${TARGET_INCLUDE_PATHS}
+                                ${TRPC_ROOT_PATH}/cmake_third_party/jwt_cpp)
+    set(TARGET_LINK_LIBS ${TARGET_LINK_LIBS} trpc_jwt_cpp)
+endif()
+

cmake/prometheus.cmake

@@ -51,6 +51,9 @@ ${TRPC_BUILD_BINARY_PATH}/cmake_third_party/prometheus/core/include>"
                                 ${prometheus_SOURCE_DIR}/pull/include
                                 ${TRPC_BUILD_BINARY_PATH}/cmake_third_party/prometheus/pull/include
                                 ${CMAKE_SOURCE_DIR}/build/_deps/prometheus-build/pull/include
+                                ${prometheus_SOURCE_DIR}/push/include
+                                ${TRPC_BUILD_BINARY_PATH}/cmake_third_party/prometheus/push/include
+                                ${CMAKE_SOURCE_DIR}/build/_deps/prometheus-build/push/include
                                 ${prometheus_SOURCE_DIR}/core/include
                                 ${TRPC_BUILD_BINARY_PATH}/cmake_third_party/prometheus/core/include
                                 ${CMAKE_SOURCE_DIR}/build/_deps/prometheus-build/core/include)

docs/en/prometheus_metrics.md

@@ -19,6 +19,14 @@ By default, tRPC-Cpp framework does not compile related code with prometheus. To
 
 ## Bazel
 
+Before compile, please add prometheus deps in your WORKSPACE file.
+
+```python
+load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cpp_repositories")
+
+prometheus_cpp_repositories()
+```
+
 Add the `"trpc_include_prometheus"` compilation option during Bazel compilation.
 
 For example, add it in `.bazelrc` file.
@@ -66,6 +74,20 @@ The description of the configuration item are as follow.
 | histogram_module_cfg | Sequences | No, the default value is [1, 10, 100, 1000] | Statistical interval for latency distribution in ModuleReport, measured in milliseconds. |
 | const_labels | Mappings | No, the default value is empty. | Default labels attached to each RPC statistical data. |
 
+By default, we use pull mode to report. If you want to use push mode, please add below configuration to your yaml file.
+
+```yaml
+plugins:
+  metrics:
+    prometheus:
+      push_mode:
+        enable: true
+        gateway_host: 127.0.0.1
+        gateway_port: 9091
+        job_name: trpc_prometheus_push_metrics
+        interval_ms: 10000 # 默认上报间隔是10s
+```
+
 ## ModuleReport
 
 **ModuleReport refers to reporting metrics data of RPC inter-module calls, including caller reporting (for tracking client-side invocation information) and callee reporting (for tracking server-side invocation information).**

docs/zh/prometheus_metrics.md

@@ -19,6 +19,14 @@ tRPC-Cpp 框架默认不会编译 Prometheus 相关的代码。若要开启，
 
 ## Bazel 启用方式
 
+编译前，需要增加prometheus的依赖在WORKSPACE文件里。
+
+```python
+load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cpp_repositories")
+
+prometheus_cpp_repositories()
+```
+
 bazel 编译时加上`“trpc_include_prometheus”`编译选项。
 
 例如在 `.bazelrc` 中加上：
@@ -59,6 +67,20 @@ plugins:
         key2: value2
 ```
 
+默认采用pull模式，如果需要启用push模式，则需要增加类似如下的配置：
+
+```yaml
+plugins:
+  metrics:
+    prometheus:
+      push_mode:
+        enable: true
+        gateway_host: 127.0.0.1
+        gateway_port: 9091
+        job_name: trpc_prometheus_push_metrics
+        interval_ms: 10000 # 默认上报间隔是10s
+```
+
 配置项说明：
 
 | 参数 | 类型 | 是否必须配置 | 说明 |
@@ -398,3 +420,60 @@ std::vector<::prometheus::MetricFamily> Collect();
 ## 通过 admin 获取
 
 如果服务开启了 [admin 功能](./admin_service.md)，则可以通过访问 `http://admin_ip:admin_port/metrics` 获取序列化为字符串后的 Prometheus 数据。
+
+# 鉴权
+
+Prometheus插件鉴权分为两种模式：pull模式 和 push模式，不同模式下的配置方式有所区别。
+
+## pull模式
+
+在pull模式下，使用Json Web Token（JWT）方式来鉴权。需要同时配置**trpc的Prometheus插件**和**Prometheus服务器**。
+
+### 插件配置
+
+插件配置样例如下：
+
+```yaml
+plugins:
+  metrics:
+    prometheus:
+      auth_cfg:
+        iss: admin # issuer 签发人
+        sub: prometheus-pull # subject 主题
+        aud: trpc-server # audience 受众
+        secret: test # 密钥
+```
+
+需要配置**bearer_token**字段，该token可以通过[JWT官方工具](https://jwt.io/)生成。在payload中填写相应的iss，sub和aud字段，verify signature中填写secret字段，加密算法使用默认的 HS256。
+
+## push模式
+
+在push模式下，为了和pushgateway兼容，鉴权使用**username**和**password**的形式。
+
+### 插件配置
+
+插件配置样例如下：
+
+```yaml
+plugins:
+  metrics:
+    prometheus:
+      auth_cfg:
+        username: admin
+        password: test
+```
+
+### Pushgateway服务器配置
+
+需要在Pushgateway服务器启动时，通过带有通过**bcrypt**加密的密文的配置文件启动。Pushgateway启动的配置文件如下：
+
+```yaml
+basic_auth_users:
+  admin: $2y$05$5uq4H5p8JyfQm.e16o3xduW6tkI2bTRpArTK4MF4dEuvncpz/bqy.
+```
+
+密码的密文可以通过htpasswd工具生成：
+```shell
+> htpasswd -nbB admin test
+admin:$2y$05$5uq4H5p8JyfQm.e16o3xduW6tkI2bTRpArTK4MF4dEuvncpz/bqy.
+```

examples/features/prometheus/README.md

@@ -32,6 +32,14 @@ examples/features/prometheus/
 
 * Compilation
 
+Before compile, please add prometheus deps in your WORKSPACE.
+
+```python
+load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cpp_repositories")
+
+prometheus_cpp_repositories()
+```
+
 We can run the following command to compile the demo.
 
 ```shell

examples/features/prometheus/proxy/trpc_cpp_fiber.yaml

@@ -44,6 +44,12 @@ plugins:
       const_labels:
         const_key1: const_value1
         const_key2: const_value2
+      # push_mode:
+      #   enable: true
+      #   gateway_host: 127.0.0.1
+      #   gateway_port: 9091
+      #   job_name: trpc_prometheus_push_metrics
+      #   interval_ms: 1000
   log:
     default:
       - name: default

examples/features/prometheus/run.sh

@@ -1,12 +1,25 @@
 bazel build //examples/helloworld/...
+
+cat > WORKSPACE <<EOF
+workspace(name = "trpc_cpp")
+
+load("//trpc:workspace.bzl", "trpc_workspace")
+
+trpc_workspace()
+
+load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cpp_repositories")
+
+prometheus_cpp_repositories()
+EOF
+
 bazel build //examples/features/prometheus/... --define trpc_include_prometheus=true
 
 echo "begin"
 ./bazel-bin/examples/helloworld/helloworld_svr --config=examples/helloworld/conf/trpc_cpp_fiber.yaml &
 sleep 1
 ./bazel-bin/examples/features/prometheus/proxy/forward_server --config=examples/features/prometheus/proxy/trpc_cpp_fiber.yaml &
 sleep 1
-./bazel-bin/examples/features/prometheus/client/client_config --config=examples/features/prometheus/client/trpc_cpp_fiber.yaml
+./bazel-bin/examples/features/prometheus/client/client --client_config=examples/features/prometheus/client/trpc_cpp_fiber.yaml
 
 killall helloworld_svr
 if [ $? -ne 0 ]; then

third_party/com_github_thalhammer_jwt_cpp/jwt_cpp.BUILD

@@ -0,0 +1,13 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "jwt-cpp",
+    hdrs = glob(["include/jwt-cpp/**/*.h", "include/picojson/*.h"]),
+    includes = ["include"],
+    deps = [
+      "@com_github_openssl_openssl//:libcrypto",
+      "@com_github_openssl_openssl//:libssl",
+    ],
+)

trpc/admin/BUILD

@@ -354,13 +354,23 @@ cc_library(
         ":admin_handler",
         ":base_funcs",
         "//trpc/util:prometheus",
+        "//trpc/common/config:trpc_config",
+        "//trpc/log:trpc_log",
+        "//trpc/util/http:base64",
+        "//trpc/util/string:string_helper",
     ] + select({
         "//conditions:default": [],
         "//trpc:trpc_include_prometheus": [
             "@com_github_jupp0r_prometheus_cpp//pull",
+            "@com_github_thalhammer_jwt_cpp//:jwt-cpp",
+            "//trpc/metrics/prometheus:prometheus_metrics",
+            "//trpc/metrics/prometheus:prometheus_conf_parser",
         ],
         "//trpc:include_metrics_prometheus": [
             "@com_github_jupp0r_prometheus_cpp//pull",
+            "@com_github_thalhammer_jwt_cpp//:jwt-cpp",
+            "//trpc/metrics/prometheus:prometheus_metrics",
+            "//trpc/metrics/prometheus:prometheus_conf_parser",
         ],
     }),
 )

trpc/admin/admin_service.cc

@@ -117,7 +117,9 @@ AdminService::AdminService() {
 
 #ifdef TRPC_BUILD_INCLUDE_PROMETHEUS
   // Prometheus metrics.
-  RegisterCmd(http::OperationType::GET, "/metrics", std::make_shared<admin::PrometheusHandler>());
+  auto prometheus_handle_ptr = std::make_shared<admin::PrometheusHandler>();
+  prometheus_handle_ptr->Init();
+  RegisterCmd(http::OperationType::GET, "/metrics", prometheus_handle_ptr);
 #endif
 
   RegisterCmd(http::OperationType::POST, "/client_detach", std::make_shared<admin::ClientDetachHandler>());