优化 bkapi-client-core 异常消息

sdks/bkapi-client-core/CHANGE.md

@@ -1,5 +1,11 @@
 ## Change logs
 
+## 1.2.0
+- BK_API_CLIENT_ENABLE_SSL_VERIFY 默认值设置为 False
+- Client 添加辅助方法：check_response_apigateway_error
+- ResponseError 添加辅助方法：response_status_code, response_text, response_json
+- 日志中，curl 信息请求头 X-Bkapi-Authorization 脱敏
+
 ### 1.1.8
 - 使用 `CurlRequest` 封装转换 curl 命令的逻辑以优化性能
 

sdks/bkapi-client-core/Makefile

@@ -22,7 +22,7 @@ requirements.txt: poetry.lock
 	poetry export -f requirements.txt --without-hashes | grep -v "index-url" > requirements.txt
 
 requirements_dev.txt: requirements.txt
-	poetry export -f requirements.txt --without-hashes --dev | grep -v "index-url" > requirements_dev.txt
+	poetry export -f requirements.txt --without-hashes --with dev | grep -v "index-url" > requirements_dev.txt
 
 requirements_tox.txt: requirements_dev.txt
 	grep -E 'pytest|mock|django|prometheus-client' requirements_dev.txt > requirements_tox.txt

sdks/bkapi-client-core/README.md

@@ -6,7 +6,6 @@
 - 支持丰富的请求参数，包括魔法参数 data，requests.request 全部参数(除 data)、路径参数
   - 魔法参数 data，对于 GET/HEAD/OPTIONS 请求，参数将转换为 QueryString，其它请求方法，则转换为 json 格式的 Body
   - 支持从 django settings，Cookies 获取认证数据
-  - verify 默认值为 True，请求 HTTPS 接口更安全
 - 支持对 session 进行更新细粒度的控制，支持复用 session 的连接
 - 灵活的响应处理方式
   - 支持校验响应状态码，获取响应的 json 数据
@@ -111,7 +110,7 @@ SDK 支持通过配置更改一些默认的行为，Django settings 配置优先
 | BK_APP_CODE                          | 应用代号                               | string | `"my_app"`                        |                            | 支持        | 支持     | BKPAAS_APP_ID     |
 | BK_APP_SECRET                        | 应用密钥                               | string | `"my_secret"`                     |                            | 支持        | 支持     | BKPAAS_APP_SECRET |
 | DEFAULT_STAGE_MAPPINGS               | 指定对应网关的默认环境                 | dict   | `{"my_gateway": "prod"}`          |                            | 支持        |          |                   |
-| BK_API_CLIENT_ENABLE_SSL_VERIFY      | 是否开启 SSL 证书验证                  | bool   | `True`                            |                            | 支持        |          |                   |
+| BK_API_CLIENT_ENABLE_SSL_VERIFY      | 是否开启 SSL 证书验证                  | bool   | `True`                            | `False`                    | 支持        |          |                   |
 | BK_API_AUTHORIZATION_COOKIES_MAPPING | 指定 Cookie 和认证参数的映射关系       | dict   | `{"key": "cookie"}`               | `{"bk_token": "bk_token"}` | 支持        |          |                   |
 | BK_API_URL_TMPL                      | 网关地址模板，支持 `{api_name}` 占位符 | string | `"http://{api_name}.example.com"` |                            | 支持        | 支持     |                   |
 | BK_COMPONENT_API_URL                 | 组件 API 网关地址                      | string | `"http://esb.example.com"`        |                            | 支持        | 支持     |                   |
@@ -224,4 +223,4 @@ prometheus.enable()
 | bkapi_requests_body_bytes       | Histogram | 请求体大小   | operation,method        |
 | bkapi_responses_body_bytes      | Histogram | 响应体大小   | operation,method        |
 | bkapi_responses_total           | Counter   | 响应总数     | operation,method,status |
-| bkapi_failures_total            | Counter   | 请求失败总数 | operation,method,error  |
+| bkapi_failures_total            | Counter   | 请求失败总数 | operation,method,error  |

sdks/bkapi-client-core/bkapi_client_core/__init__.py

@@ -8,4 +8,4 @@
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
 """
-__version__ = "1.1.9"
+__version__ = "1.2.0"

sdks/bkapi-client-core/bkapi_client_core/client.py

@@ -25,6 +25,7 @@
     EndpointNotSetError,
     HTTPResponseError,
     JSONResponseError,
+    ResponseError,
 )
 from bkapi_client_core.session import Session
 from bkapi_client_core.utils import CurlRequest, urljoin
@@ -112,7 +113,15 @@ def __str__(self):
         if not self._headers:
             return ""
 
-        return "request_id: %s, error_code: %s, %s" % (self.request_id, self.error_code, self.error_message)
+        parts = ["request_id: %s" % self.request_id]
+
+        if self.error_code:
+            parts.append("error_code: %s" % self.error_code)
+
+        if self.error_message:
+            parts.append(self.error_message)
+
+        return ", ".join(parts)
 
 
 class BaseClient(object):
@@ -184,6 +193,23 @@ def parse_response(
         except RequestException as err:
             return self._handle_exception(operation, None, err)
 
+    def check_response_apigateway_error(
+        self,
+        response,  # type: Optional[Response]
+    ):
+        # type: (...) -> None
+        """检查是否包含 apigateway 层面的报错，如应用认证失败，无访问 API 权限等"""
+        if response is None:
+            return
+
+        response_headers_representer = ResponseHeadersRepresenter(response.headers)
+        if response_headers_representer.has_apigateway_error:
+            raise APIGatewayResponseError(
+                "Error responded by API Gateway",
+                response=response,
+                response_headers_representer=response_headers_representer,
+            )
+
     def update_headers(
         self,
         headers,  # type: Dict[str, str]
@@ -256,12 +282,16 @@ def _handle_exception(
     ):
         # type: (...) -> Optional[Response]
         # log exception
-        if isinstance(exception, RequestException):
+        if isinstance(exception, ResponseError):
+            logger.warning("%s\n%s", str(exception), CurlRequest(exception.request))
+        elif isinstance(exception, RequestException):
             response = exception.response
-            response_headers_representer = ResponseHeadersRepresenter(response and response.headers)
+            response_headers_representer = ResponseHeadersRepresenter(
+                response.headers if response is not None else None
+            )
             logger.exception(
-                "request bkapi failed. status_code: %s, %s\n%s",
-                response and response.status_code,
+                "Request bkapi error, status_code: %s, %s\n%s",
+                response.status_code if response is not None else None,
                 response_headers_representer,
                 CurlRequest(exception.request),
             )
@@ -298,24 +328,22 @@ def _handle_response_content(
         if response is None:
             return None
 
-        response_headers_representer = ResponseHeadersRepresenter(response.headers)
-        if response_headers_representer.has_apigateway_error:
-            raise APIGatewayResponseError(
-                "Request bkapi error, %s" % response_headers_representer.error_message,
-                response=response,
-                response_headers_representer=response_headers_representer,
-            )
+        self.check_response_apigateway_error(response)
 
         try:
             response.raise_for_status()
         except HTTPError as err:
+            response_headers_representer = ResponseHeadersRepresenter(response.headers)
             raise HTTPResponseError(
-                str(err), response=response, response_headers_representer=response_headers_representer
+                "Error responded by Backend api, %s" % str(err),
+                response=response,
+                response_headers_representer=response_headers_representer,
             )
 
         try:
             return response.json()
         except (TypeError, json.JSONDecodeError):
+            response_headers_representer = ResponseHeadersRepresenter(response.headers)
             raise JSONResponseError(
                 "The response is not a valid JSON",
                 response=response,

sdks/bkapi-client-core/bkapi_client_core/config.py

@@ -132,7 +132,7 @@ def declare_aliases(
     {
         SettingKeys.DEFAULT_BK_API_VER: "v2",
         SettingKeys.BK_API_USE_TEST_ENV: False,
-        SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY: True,
+        SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY: False,
         SettingKeys.BK_API_AUTHORIZATION_COOKIES_MAPPING: {
             "bk_token": "bk_token",
         },

sdks/bkapi-client-core/bkapi_client_core/django_helper.py

@@ -39,7 +39,7 @@ def _get_client_by_settings(
     )
 
     # disable global https verify
-    if settings.get(SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY):
+    if not settings.get(SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY):
         client.disable_ssl_verify()
 
     if accept_language:

sdks/bkapi-client-core/bkapi_client_core/exceptions.py

@@ -10,7 +10,6 @@
 """
 from typing import Optional
 
-import curlify
 from requests.exceptions import (
     ChunkedEncodingError,
     ConnectionError,
@@ -22,6 +21,9 @@
     RequestException,
     Timeout,
 )
+from six.moves.urllib.parse import urlparse, urlunparse
+
+from .utils import CurlRequest
 
 __all__ = [
     "ChunkedEncodingError",
@@ -77,9 +79,36 @@ def request_id(self):
 
     @property
     def curl_command(self):
-        if self.request is None:
+        return CurlRequest(self.request).to_curl()
+
+    @property
+    def request_method(self):
+        # type: (...) -> Optional[str]
+        return self.request and self.request.method
+
+    @property
+    def request_url(self):
+        # type: (...) -> str
+        url = self.request and self.request.url
+        if not url:
             return ""
-        return curlify.to_curl(self.request)
+
+        parsed_url = urlparse(url)
+        return urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path, "", "", ""))
+
+    @property
+    def response_status_code(self):
+        # type: (...) -> Optional[int]
+        # bool(response) is equal to response.ok
+        return self.response.status_code if self.response is not None else None
+
+    @property
+    def response_text(self):
+        # type: (...) -> Optional[str]
+        return self.response.text if self.response is not None else None
+
+    def response_json(self):
+        return self.response.json() if self.response is not None else None
 
     def __str__(self):
         if self.response is None:

sdks/bkapi-client-core/bkapi_client_core/utils.py

@@ -9,6 +9,7 @@
  * specific language governing permissions and limitations under the License.
 """
 import copy
+import json
 from functools import wraps
 from typing import Callable, Optional, Type, TypeVar
 
@@ -27,6 +28,56 @@ def urljoin(base_url, path):
     return "%s/%s" % (base_url.rstrip("/"), path.lstrip("/"))
 
 
+class _SensitiveCleaner:
+    """处理敏感信息"""
+
+    def __init__(self, sensitive_keys):
+        self.sensitive_keys = sensitive_keys
+
+    def clean(self, data):
+        data = copy.deepcopy(data)
+        self._clean(data)
+        return data
+
+    def _clean(self, data):
+        if isinstance(data, dict):
+            for key, value in data.items():
+                if isinstance(value, (dict, list)):
+                    self._clean(value)
+
+                elif key in self.sensitive_keys and value:
+                    data[key] = "***"
+
+        elif isinstance(data, list):
+            for item in data:
+                self._clean(item)
+
+
+class _WrappedRequest:
+    header_bkapi_authorization = "X-Bkapi-Authorization"
+
+    def __init__(self, request):
+        self._request = request
+
+        # 去除请求头中的敏感信息
+        self.headers = self._get_headers_without_sensitive(self._request.headers)
+
+    def __getattr__(self, name):
+        return getattr(self._request, name)
+
+    def _get_headers_without_sensitive(self, headers):
+        headers = copy.deepcopy(headers)
+
+        authorization = headers.get(self.header_bkapi_authorization)
+        if authorization:
+            sensitive_cleaner = _SensitiveCleaner(
+                ["bk_app_secret", "app_secret", "bk_token", "bk_ticket", "access_token"]
+            )
+            headers[self.header_bkapi_authorization] = json.dumps(sensitive_cleaner.clean(json.loads(authorization)))
+
+        return headers
+
+
 class CurlRequest:
     def __init__(
         self,
@@ -41,11 +92,11 @@ def to_curl(self):
 
         try:
             # if request.body contains binary content, it may not be decoded
-            return curlify.to_curl(self.request)
+            return curlify.to_curl(_WrappedRequest(self.request))
         except UnicodeDecodeError:
             copied_request = copy.deepcopy(self.request)
             copied_request.body = ""
-            return curlify.to_curl(copied_request)
+            return curlify.to_curl(_WrappedRequest(copied_request))
         except Exception:
             return ""
 

sdks/bkapi-client-core/demo/shortcuts.py

@@ -8,10 +8,11 @@
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
 """
-from .client import Client
 from bkapi_client_core.apigateway import generic_type_partial as _partial
 from bkapi_client_core.apigateway.django_helper import get_client_by_request as _get_client_by_request
 from bkapi_client_core.apigateway.django_helper import get_client_by_username as _get_client_by_username
 
+from .client import Client
+
 get_client_by_request = _partial(Client, _get_client_by_request)
 get_client_by_username = _partial(Client, _get_client_by_username)

sdks/bkapi-client-core/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bkapi-client-core"
-version = "1.1.9"
+version = "1.2.0"
 description = "A toolkit for buiding blueking API clients."
 readme = "README.md"
 authors = ["blueking <[email protected]>"]
@@ -22,6 +22,7 @@ curlify = ">=2.0"
 bkoauth = {version = ">=0.0.10", optional = true}
 typing-extensions = ">=3.7.4"
 prometheus-client = {version = ">=0.9.0", optional = true}
+six = "*"
 
 [tool.poetry.extras]
 django=["bkoauth", "prometheus-client"]

sdks/bkapi-client-core/requirements.txt

@@ -7,6 +7,7 @@ idna==2.10 ; python_version >= "2.7" and python_version < "3.0"
 idna==3.2 ; python_version >= "3.6" and python_version < "4.0"
 prometheus-client==0.12.0 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4.0"
 requests==2.26.0 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4.0"
+six==1.16.0 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4.0"
 typing-extensions==3.10.0.2 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4.0"
 typing==3.10.0.0 ; python_version >= "2.7" and python_version < "3.0"
 urllib3==1.26.6 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4"