Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(bklogin): monitoring #1397

Merged
merged 1 commit into from
Nov 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/bk-login/bklogin/common/error_codes.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ class ErrorCodeCategoryEnum(str, StructuredEnum):
class ErrorCodes:
# 通用
INVALID_ARGUMENT = ErrorCode(_("参数非法"))
UNAUTHENTICATED = ErrorCode(
_("未认证"),
code_category=ErrorCodeCategoryEnum.UNAUTHENTICATED,
status_code=HTTPStatus.UNAUTHORIZED,
)
NO_PERMISSION = ErrorCode(
_("无权限"),
code_category=ErrorCodeCategoryEnum.NO_PERMISSION,
Expand Down
10 changes: 10 additions & 0 deletions src/bk-login/bklogin/monitoring/healthz/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
22 changes: 22 additions & 0 deletions src/bk-login/bklogin/monitoring/healthz/probes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from blue_krill.monitoring.probe.mysql import MySQLProbe, transfer_django_db_settings
from django.conf import settings
from django.utils.module_loading import import_string


def get_default_probes():
return [import_string(p) for p in settings.HEALTHZ_PROBES]


class MysqlProbe(MySQLProbe):
name = "bklogin-mysql"
config = transfer_django_db_settings(settings.DATABASES["default"])
18 changes: 18 additions & 0 deletions src/bk-login/bklogin/monitoring/healthz/urls.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from django.urls import path

from . import views

urlpatterns = [
path("ping", views.PingApi.as_view(), name="api.ping"),
path("healthz", views.HealthzApi.as_view(), name="api.healthz"),
]
60 changes: 60 additions & 0 deletions src/bk-login/bklogin/monitoring/healthz/views.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from blue_krill.monitoring.probe.base import ProbeSet
from django.conf import settings
from django.http import HttpResponse
from django.views.generic import View

from bklogin.common.error_codes import error_codes
from bklogin.common.response import APISuccessResponse

from .probes import get_default_probes


class PingApi(View):
"""就绪&存活探测 API"""

def get(self, request, *args, **kwargs):
return HttpResponse("pong")


class HealthzApi(View):
"""健康探测 API"""

def get(self, request, *args, **kwargs):
token = request.GET.get("token", "")
if not settings.HEALTHZ_TOKEN:
raise error_codes.UNAUTHENTICATED.f(
"Healthz token was not configured in settings, request denied", replace=True
)
if not (token and token == settings.HEALTHZ_TOKEN):
raise error_codes.UNAUTHENTICATED.f("Please provide valid token", replace=True)

probe_set = ProbeSet(get_default_probes())
diagnosis_list = probe_set.examination()

if diagnosis_list.is_death:
# if something deadly exist, we have to make response non-200 which is easier to be found
# by monitor system and make response as a plain text
raise error_codes.SYSTEM_ERROR.f("internal server error", replace=True).set_data(
diagnosis_list.get_fatal_report()
)

results = [
{
"system_name": i.system_name,
"alive": i.alive,
"issues": [{"fatal": j.fatal, "description": j.description} for j in i.issues],
}
for i in diagnosis_list.items
]

return APISuccessResponse(data={"results": results})
10 changes: 10 additions & 0 deletions src/bk-login/bklogin/monitoring/metrics/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
15 changes: 15 additions & 0 deletions src/bk-login/bklogin/monitoring/metrics/urls.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from django.conf.urls import url

from . import views

urlpatterns = [url(r"^metrics$", views.metric_view, name="prometheus-django-metrics")]
27 changes: 27 additions & 0 deletions src/bk-login/bklogin/monitoring/metrics/views.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from django.conf import settings
from django_prometheus.exports import ExportToDjangoView

from bklogin.common.error_codes import error_codes


def metric_view(request):
"""metric view with basic auth"""
token = request.GET.get("token", "")
if not settings.METRIC_TOKEN:
raise error_codes.UNAUTHENTICATED.f(
"Metric token was not configured in settings, request denied", replace=True
)
if not (token and token == settings.METRIC_TOKEN):
raise error_codes.UNAUTHENTICATED.f("Please provide valid token", replace=True)

return ExportToDjangoView(request)
10 changes: 10 additions & 0 deletions src/bk-login/bklogin/monitoring/tracing/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
26 changes: 26 additions & 0 deletions src/bk-login/bklogin/monitoring/tracing/apps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from django.apps import AppConfig

from .otel import setup_by_settings
from .sentry import init_sentry_sdk


class TracingConfig(AppConfig):
name = "bklogin.monitoring.tracing"

def ready(self):
setup_by_settings()
init_sentry_sdk(
django_integrated=True,
redis_integrated=False,
celery_integrated=False,
)
120 changes: 120 additions & 0 deletions src/bk-login/bklogin/monitoring/tracing/hooks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import json
from typing import Dict

import requests
from django.http import HttpRequest, HttpResponse
from opentelemetry.trace import Span, StatusCode, format_trace_id


def handle_api_error(span: Span, result: Dict):
"""统一处理新版 HTTP API 协议中的错误详情"""
if "error" not in result:
return

err = result["error"]
span.set_attribute("error_code", err.get("code", ""))
span.set_attribute("error_message", err.get("message", ""))
span.set_attribute("error_system", err.get("system", ""))
# 错误详情若存在,则统一存到一个字段中
if err_details := err.get("details", []):
span.set_attribute("error_details", json.dumps(err_details))


def requests_response_hook(span: Span, request: requests.Request, response: requests.Response):
"""用于处理 requests 库发起的请求响应,需要兼容支持新旧 esb,apigw,新版 HTTP 协议"""
if (
# requests 请求异常, 例如访问超时等
response is None
# 并非所有返回内容都是 json 格式的, 因此需要根据返回头进行判断, 避免处理二进制格式的内容
or response.headers.get("Content-Type") != "application/json"
):
return

try:
result = json.loads(response.content)
except Exception: # pylint: disable=broad-except
return
if not isinstance(result, dict):
return

request_id = (
# new esb and apigateway
response.headers.get("x-bkapi-request-id")
# legacy api
or response.headers.get("X-Request-Id")
# old esb and other
or result.get("request_id", "")
)
if request_id:
span.set_attribute("request_id", request_id)

if "message" in result:
span.set_attribute("error_message", result["message"])

# 旧版本 API 中,code 为 0/'0'/'00' 表示成功
code = result.get("code")
if code is not None:
span.set_attribute("error_code", str(code))
if str(code) in ["0", "00"]:
span.set_status(StatusCode.OK)
else:
span.set_status(StatusCode.ERROR)

# 后续均为处理新版 API 协议逻辑,因此此处直接 return
return

# 根据新版本 HTTP API 协议,处理错误详情
handle_api_error(span, result)

if 200 <= response.status_code <= 299: # noqa: PLR2004
span.set_status(StatusCode.OK)
else:
span.set_status(StatusCode.ERROR)


def django_request_hook(span: Span, request: HttpRequest):
"""在 request 注入 trace_id,方便获取"""
trace_id = span.get_span_context().trace_id
request.otel_trace_id = format_trace_id(trace_id)


def django_response_hook(span: Span, request: HttpRequest, response: HttpResponse):
"""处理 Django 响应,因用户管理已经使用新版本 HTTP 协议,因此仅支持新版协议"""

if (
# requests 请求异常, 例如访问超时等
response is None
# 并非所有返回内容都是 json 格式的, 因此需要根据返回头进行判断, 避免处理二进制格式的内容
or response.headers.get("Content-Type") != "application/json"
):
return

# 新版本协议中按照标准 HTTP 协议,200 <= code < 300 的都是正常
if 200 <= response.status_code <= 299: # noqa: PLR2004
span.set_status(StatusCode.OK)
return

span.set_status(StatusCode.ERROR)
try:
result = json.loads(response.content)
except Exception: # pylint: disable=broad-except
return
if not isinstance(result, dict):
return

# 若能够获取到 request_id,则一并记录
request_id = response.headers.get("X-Request-Id") or result.get("request_id")
if request_id:
span.set_attribute("request_id", request_id)

handle_api_error(span, result)
59 changes: 59 additions & 0 deletions src/bk-login/bklogin/monitoring/tracing/instrumentor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import logging
from typing import Collection

from django.conf import settings
from opentelemetry.instrumentation import dbapi
from opentelemetry.instrumentation.celery import CeleryInstrumentor
from opentelemetry.instrumentation.django import DjangoInstrumentor
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
from opentelemetry.instrumentation.logging import LoggingInstrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor

from .hooks import django_request_hook, django_response_hook, requests_response_hook

logger = logging.getLogger(__name__)


class BKLoginInstrumentor(BaseInstrumentor):
def instrumentation_dependencies(self) -> Collection[str]:
return []

def _instrument(self, **kwargs):
LoggingInstrumentor().instrument()
logger.info("otel instructment: logging")
RequestsInstrumentor().instrument(response_hook=requests_response_hook)
logger.info("otel instructment: requests")
DjangoInstrumentor().instrument(request_hook=django_request_hook, response_hook=django_response_hook)
logger.info("otel instructment: django")
RedisInstrumentor().instrument()
logger.info("otel instructment: redis")
CeleryInstrumentor().instrument()
logger.info("otel instructment: celery")

if getattr(settings, "OTEL_INSTRUMENT_DB_API", False):
import MySQLdb # noqa

dbapi.wrap_connect(
__name__,
MySQLdb,
"connect",
"mysql",
{"database": "db", "port": "port", "host": "host", "user": "user"},
)
logger.info("otel instructment: database api")

def _uninstrument(self, **kwargs):
for instrumentor in self.instrumentors:
logger.info("otel uninstrument %s", instrumentor)
instrumentor.uninstrument()
Loading
Loading