From 6cc2bcc75819aa538216b6fdffa2034574fc9df8 Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Sun, 19 Nov 2023 12:21:29 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E4=BC=98=E5=8C=96=20bkapi-client-core=20?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E6=B6=88=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/bkapi-client-core/Makefile | 2 +- sdks/bkapi-client-core/README.md | 5 +-- .../bkapi_client_core/client.py | 34 ++++++++++++----- .../bkapi_client_core/config.py | 2 +- .../bkapi_client_core/django_helper.py | 2 +- .../bkapi_client_core/exceptions.py | 37 +++++++++++++++++-- .../bkapi_client_core/utils.py | 15 +++++++- sdks/bkapi-client-core/poetry.lock | 4 +- sdks/bkapi-client-core/pyproject.toml | 1 + sdks/bkapi-client-core/requirements.txt | 1 + sdks/bkapi-client-core/tests/test_config.py | 2 +- .../tests/test_django_helper.py | 8 ++-- 12 files changed, 85 insertions(+), 28 deletions(-) diff --git a/sdks/bkapi-client-core/Makefile b/sdks/bkapi-client-core/Makefile index 59be41a4..c65b1747 100644 --- a/sdks/bkapi-client-core/Makefile +++ b/sdks/bkapi-client-core/Makefile @@ -22,7 +22,7 @@ requirements.txt: poetry.lock poetry export -f requirements.txt --without-hashes | grep -v "index-url" > requirements.txt requirements_dev.txt: requirements.txt - poetry export -f requirements.txt --without-hashes --dev | grep -v "index-url" > requirements_dev.txt + poetry export -f requirements.txt --without-hashes --with dev | grep -v "index-url" > requirements_dev.txt requirements_tox.txt: requirements_dev.txt grep -E 'pytest|mock|django|prometheus-client' requirements_dev.txt > requirements_tox.txt diff --git a/sdks/bkapi-client-core/README.md b/sdks/bkapi-client-core/README.md index bafff46d..70c8abb3 100644 --- a/sdks/bkapi-client-core/README.md +++ b/sdks/bkapi-client-core/README.md @@ -6,7 +6,6 @@ - 支持丰富的请求参数,包括魔法参数 data,requests.request 全部参数(除 data)、路径参数 - 魔法参数 data,对于 GET/HEAD/OPTIONS 请求,参数将转换为 QueryString,其它请求方法,则转换为 json 格式的 Body - 支持从 django settings,Cookies 获取认证数据 - - verify 默认值为 True,请求 HTTPS 接口更安全 - 支持对 session 进行更新细粒度的控制,支持复用 session 的连接 - 灵活的响应处理方式 - 支持校验响应状态码,获取响应的 json 数据 @@ -111,7 +110,7 @@ SDK 支持通过配置更改一些默认的行为,Django settings 配置优先 | BK_APP_CODE | 应用代号 | string | `"my_app"` | | 支持 | 支持 | BKPAAS_APP_ID | | BK_APP_SECRET | 应用密钥 | string | `"my_secret"` | | 支持 | 支持 | BKPAAS_APP_SECRET | | DEFAULT_STAGE_MAPPINGS | 指定对应网关的默认环境 | dict | `{"my_gateway": "prod"}` | | 支持 | | | -| BK_API_CLIENT_ENABLE_SSL_VERIFY | 是否开启 SSL 证书验证 | bool | `True` | | 支持 | | | +| BK_API_CLIENT_ENABLE_SSL_VERIFY | 是否开启 SSL 证书验证 | bool | `True` | `False` | 支持 | | | | BK_API_AUTHORIZATION_COOKIES_MAPPING | 指定 Cookie 和认证参数的映射关系 | dict | `{"key": "cookie"}` | `{"bk_token": "bk_token"}` | 支持 | | | | BK_API_URL_TMPL | 网关地址模板,支持 `{api_name}` 占位符 | string | `"http://{api_name}.example.com"` | | 支持 | 支持 | | | BK_COMPONENT_API_URL | 组件 API 网关地址 | string | `"http://esb.example.com"` | | 支持 | 支持 | | @@ -224,4 +223,4 @@ prometheus.enable() | bkapi_requests_body_bytes | Histogram | 请求体大小 | operation,method | | bkapi_responses_body_bytes | Histogram | 响应体大小 | operation,method | | bkapi_responses_total | Counter | 响应总数 | operation,method,status | -| bkapi_failures_total | Counter | 请求失败总数 | operation,method,error | \ No newline at end of file +| bkapi_failures_total | Counter | 请求失败总数 | operation,method,error | diff --git a/sdks/bkapi-client-core/bkapi_client_core/client.py b/sdks/bkapi-client-core/bkapi_client_core/client.py index 74c7f722..697f979c 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/client.py +++ b/sdks/bkapi-client-core/bkapi_client_core/client.py @@ -13,7 +13,7 @@ from typing import Any, Dict, Optional from requests import Response -from requests.exceptions import HTTPError, RequestException +from requests.exceptions import HTTPError, RequestException, JSONDecodeError from requests.sessions import merge_setting from requests.structures import CaseInsensitiveDict @@ -112,7 +112,15 @@ def __str__(self): if not self._headers: return "" - return "request_id: %s, error_code: %s, %s" % (self.request_id, self.error_code, self.error_message) + parts = ["request_id: %s" % self.request_id] + + if self.error_code: + parts.append("error_code: %s" % self.error_code) + + if self.error_message: + parts.append(self.error_message) + + return ", ".join(parts) class BaseClient(object): @@ -306,22 +314,28 @@ def _handle_response_content( response_headers_representer=response_headers_representer, ) + # 先校验 json,状态码校验失败时,可以在 exception 中获取正常的 json 响应内容; + # 如此,方便调用者在同一层中处理 http 状态码和 json 两个异常 try: - response.raise_for_status() - except HTTPError as err: - raise HTTPResponseError( - str(err), response=response, response_headers_representer=response_headers_representer + response_json = response.json() + except (TypeError, json.JSONDecodeError, JSONDecodeError): + raise JSONResponseError( + "The response is not a valid JSON", + response=response, + response_headers_representer=response_headers_representer, ) try: - return response.json() - except (TypeError, json.JSONDecodeError): - raise JSONResponseError( - "The response is not a valid JSON", + response.raise_for_status() + except HTTPError as err: + raise HTTPResponseError( + str(err), response=response, response_headers_representer=response_headers_representer, ) + return response_json + def close(self): """Close the session""" self.session.close() diff --git a/sdks/bkapi-client-core/bkapi_client_core/config.py b/sdks/bkapi-client-core/bkapi_client_core/config.py index 5256c2a7..721a9034 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/config.py +++ b/sdks/bkapi-client-core/bkapi_client_core/config.py @@ -132,7 +132,7 @@ def declare_aliases( { SettingKeys.DEFAULT_BK_API_VER: "v2", SettingKeys.BK_API_USE_TEST_ENV: False, - SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY: True, + SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY: False, SettingKeys.BK_API_AUTHORIZATION_COOKIES_MAPPING: { "bk_token": "bk_token", }, diff --git a/sdks/bkapi-client-core/bkapi_client_core/django_helper.py b/sdks/bkapi-client-core/bkapi_client_core/django_helper.py index 49dbf279..26cd572c 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/django_helper.py +++ b/sdks/bkapi-client-core/bkapi_client_core/django_helper.py @@ -39,7 +39,7 @@ def _get_client_by_settings( ) # disable global https verify - if settings.get(SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY): + if not settings.get(SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY): client.disable_ssl_verify() if accept_language: diff --git a/sdks/bkapi-client-core/bkapi_client_core/exceptions.py b/sdks/bkapi-client-core/bkapi_client_core/exceptions.py index c268e198..27850a6d 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/exceptions.py +++ b/sdks/bkapi-client-core/bkapi_client_core/exceptions.py @@ -10,7 +10,6 @@ """ from typing import Optional -import curlify from requests.exceptions import ( ChunkedEncodingError, ConnectionError, @@ -22,6 +21,9 @@ RequestException, Timeout, ) +from six.moves.urllib.parse import urlparse, urlunparse + +from .utils import CurlRequest __all__ = [ "ChunkedEncodingError", @@ -77,9 +79,35 @@ def request_id(self): @property def curl_command(self): - if self.request is None: + return CurlRequest(self.request).to_curl() + + @property + def request_method(self): + # type: (...) -> Optional[str] + return self.request and self.request.method + + @property + def request_url(self): + # type: (...) -> str + url = self.request and self.request.url + if not url: return "" - return curlify.to_curl(self.request) + + parsed_url = urlparse(url) + return urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path, "", "", "")) + + @property + def response_status_code(self): + # type: (...) -> Optional[int] + return self.response and self.response.status_code + + @property + def response_text(self): + # type: (...) -> Optional[str] + try: + return self.response and self.response.text + except Exception: + return None def __str__(self): if self.response is None: @@ -99,6 +127,9 @@ class APIGatewayResponseError(ResponseError): class HTTPResponseError(ResponseError): """HTTP request status code error""" + def response_json(self): + return self.response.json() + class JSONResponseError(ResponseError): """The response content is not a valid json""" diff --git a/sdks/bkapi-client-core/bkapi_client_core/utils.py b/sdks/bkapi-client-core/bkapi_client_core/utils.py index 1529340f..7905b936 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/utils.py +++ b/sdks/bkapi-client-core/bkapi_client_core/utils.py @@ -27,6 +27,17 @@ def urljoin(base_url, path): return "%s/%s" % (base_url.rstrip("/"), path.lstrip("/")) +class _WrappedRequest: + def __init__(self, request): + self._request = request + + # 请求头中可能有敏感信息,不打印请求头 + self.headers = {} + + def __getattr__(self, name): + return getattr(self._request, name) + + class CurlRequest: def __init__( self, @@ -41,11 +52,11 @@ def to_curl(self): try: # if request.body contains binary content, it may not be decoded - return curlify.to_curl(self.request) + return curlify.to_curl(_WrappedRequest(self.request)) except UnicodeDecodeError: copied_request = copy.deepcopy(self.request) copied_request.body = "" - return curlify.to_curl(copied_request) + return curlify.to_curl(_WrappedRequest(copied_request)) except Exception: return "" diff --git a/sdks/bkapi-client-core/poetry.lock b/sdks/bkapi-client-core/poetry.lock index 83123e28..c7b4fc1d 100644 --- a/sdks/bkapi-client-core/poetry.lock +++ b/sdks/bkapi-client-core/poetry.lock @@ -2194,7 +2194,7 @@ reference = "tencent" name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2586,4 +2586,4 @@ monitor = ["prometheus-client"] [metadata] lock-version = "2.0" python-versions = "^2.7 || ^3.6" -content-hash = "3f78606f9de472b2c2746c3bebae9b61b234b3efcf9d6bd3336aa7f926833c5d" +content-hash = "a511699900912e4824cbcf81f571527eef5d4b7cb50c1cbcd6166ddffd0561d0" diff --git a/sdks/bkapi-client-core/pyproject.toml b/sdks/bkapi-client-core/pyproject.toml index 3520c6cd..993e25a4 100644 --- a/sdks/bkapi-client-core/pyproject.toml +++ b/sdks/bkapi-client-core/pyproject.toml @@ -22,6 +22,7 @@ curlify = ">=2.0" bkoauth = {version = ">=0.0.10", optional = true} typing-extensions = ">=3.7.4" prometheus-client = {version = ">=0.9.0", optional = true} +six = "*" [tool.poetry.extras] django=["bkoauth", "prometheus-client"] diff --git a/sdks/bkapi-client-core/requirements.txt b/sdks/bkapi-client-core/requirements.txt index 62318bd6..951f22f2 100644 --- a/sdks/bkapi-client-core/requirements.txt +++ b/sdks/bkapi-client-core/requirements.txt @@ -7,6 +7,7 @@ idna==2.10 ; python_version >= "2.7" and python_version < "3.0" idna==3.2 ; python_version >= "3.6" and python_version < "4.0" prometheus-client==0.12.0 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4.0" requests==2.26.0 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4.0" +six==1.16.0 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4.0" typing-extensions==3.10.0.2 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4.0" typing==3.10.0.0 ; python_version >= "2.7" and python_version < "3.0" urllib3==1.26.6 ; python_version >= "2.7" and python_version < "3.0" or python_version >= "3.6" and python_version < "4" diff --git a/sdks/bkapi-client-core/tests/test_config.py b/sdks/bkapi-client-core/tests/test_config.py index 489664a1..5998da60 100644 --- a/sdks/bkapi-client-core/tests/test_config.py +++ b/sdks/bkapi-client-core/tests/test_config.py @@ -125,5 +125,5 @@ def test_defaults(self, django_settings): def test_defaults(): assert settings.get(SettingKeys.DEFAULT_BK_API_VER) == "v2" assert settings.get(SettingKeys.BK_API_USE_TEST_ENV) is False - assert settings.get(SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY) is True + assert settings.get(SettingKeys.BK_API_CLIENT_ENABLE_SSL_VERIFY) is False assert settings.get(SettingKeys.BK_API_AUTHORIZATION_COOKIES_MAPPING) == {"bk_token": "bk_token"} diff --git a/sdks/bkapi-client-core/tests/test_django_helper.py b/sdks/bkapi-client-core/tests/test_django_helper.py index e4c7c381..7a3b50f2 100644 --- a/sdks/bkapi-client-core/tests/test_django_helper.py +++ b/sdks/bkapi-client-core/tests/test_django_helper.py @@ -151,9 +151,9 @@ def test_get_client_by_request_with_verify(verify, core_settings, client_cls, dj client = django_helper.get_client_by_request(client_cls, django_request) if verify: - client.disable_ssl_verify.assert_called_once_with() - else: client.disable_ssl_verify.assert_not_called() + else: + client.disable_ssl_verify.assert_called_once_with() @pytest.mark.parametrize( @@ -187,6 +187,6 @@ def test_get_client_by_username_with_verify(verify, core_settings, client_cls, d client = django_helper.get_client_by_username(client_cls, "test") if verify: - client.disable_ssl_verify.assert_called_once_with() - else: client.disable_ssl_verify.assert_not_called() + else: + client.disable_ssl_verify.assert_called_once_with() From 47af7fc2ca415918ca8faf9c023f783a6845fdde Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Mon, 20 Nov 2023 09:34:43 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkapi_client_core/client.py | 6 +- sdks/bkapi-client-core/tests/test_client.py | 48 ++++++-- .../tests/test_exceptions.py | 110 +++++++++++++++--- sdks/bkapi-client-core/tests/test_utils.py | 16 +-- 4 files changed, 146 insertions(+), 34 deletions(-) diff --git a/sdks/bkapi-client-core/bkapi_client_core/client.py b/sdks/bkapi-client-core/bkapi_client_core/client.py index 697f979c..df68632d 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/client.py +++ b/sdks/bkapi-client-core/bkapi_client_core/client.py @@ -13,7 +13,7 @@ from typing import Any, Dict, Optional from requests import Response -from requests.exceptions import HTTPError, RequestException, JSONDecodeError +from requests.exceptions import HTTPError, RequestException from requests.sessions import merge_setting from requests.structures import CaseInsensitiveDict @@ -267,7 +267,7 @@ def _handle_exception( if isinstance(exception, RequestException): response = exception.response response_headers_representer = ResponseHeadersRepresenter(response and response.headers) - logger.exception( + logger.warning( "request bkapi failed. status_code: %s, %s\n%s", response and response.status_code, response_headers_representer, @@ -318,7 +318,7 @@ def _handle_response_content( # 如此,方便调用者在同一层中处理 http 状态码和 json 两个异常 try: response_json = response.json() - except (TypeError, json.JSONDecodeError, JSONDecodeError): + except (TypeError, json.JSONDecodeError): raise JSONResponseError( "The response is not a valid JSON", response=response, diff --git a/sdks/bkapi-client-core/tests/test_client.py b/sdks/bkapi-client-core/tests/test_client.py index 00ba79d7..cd0545a4 100644 --- a/sdks/bkapi-client-core/tests/test_client.py +++ b/sdks/bkapi-client-core/tests/test_client.py @@ -213,15 +213,35 @@ def test_has_apigateway_error(self, headers, expected): headers = ResponseHeadersRepresenter(headers) assert headers.has_apigateway_error == expected - def test_str(self, faker): - headers = ResponseHeadersRepresenter( - { - "X-Bkapi-Error-Code": faker.pystr(), - "X-Bkapi-Error-Message": faker.pystr(), - "X-Bkapi-Request-Id": faker.uuid4(), - } - ) - assert "request_id:" in str(headers) + @pytest.mark.parametrize( + "headers, expected", + [ + ( + { + "X-Bkapi-Request-Id": "abcdef", + "X-Bkapi-Error-Code": "foo", + "X-Bkapi-Error-Message": "error", + }, + "request_id: abcdef, error_code: foo, error", + ), + ( + { + "X-Bkapi-Request-Id": "abcdef", + "X-Bkapi-Error-Code": "foo", + }, + "request_id: abcdef, error_code: foo", + ), + ( + { + "X-Bkapi-Request-Id": "abcdef", + }, + "request_id: abcdef", + ), + ] + ) + def test_str(self, headers, expected): + representer = ResponseHeadersRepresenter(headers) + assert str(representer) == expected class TestBaseClient: @@ -428,12 +448,20 @@ def test_handler_exception(self, mocker): def test_handle_response_content(self, mocker): assert self.client._handle_response_content(mocker.MagicMock(), None) is None + response = { + "headers": {"X-Bkapi-Request-Id": "abcd"}, + "json.return_value": {"foo": "bar"}, + "raise_for_status.return_value": None, + } + result = self.client._handle_response_content(None, mocker.MagicMock(**response)) + assert result == {"foo": "bar"} + @pytest.mark.parametrize( "response", [ {"headers": {"X-Bkapi-Error-Code": "error"}}, + {"json.side_effect": TypeError}, {"raise_for_status.side_effect": RequestException("error")}, - {"raise_for_status.return_value": None, "json.side_effect": TypeError}, ], ) def test_handle_response_content_error(self, mocker, response): diff --git a/sdks/bkapi-client-core/tests/test_exceptions.py b/sdks/bkapi-client-core/tests/test_exceptions.py index 51d1e18a..3e14c729 100644 --- a/sdks/bkapi-client-core/tests/test_exceptions.py +++ b/sdks/bkapi-client-core/tests/test_exceptions.py @@ -8,28 +8,54 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. """ +import pytest +import requests + from bkapi_client_core import exceptions from bkapi_client_core.client import ResponseHeadersRepresenter class TestResponseError: - def test_str(self, mocker, faker): - err = exceptions.ResponseError("error", response=None) - assert str(err) == "error" - - err = exceptions.ResponseError( - "error", - response=mocker.MagicMock(), - response_headers_representer=ResponseHeadersRepresenter( + @pytest.mark.parametrize( + "error, response, headers, expected", + [ + ( + "error", + { + "status_code": 400, + }, + { + "X-Bkapi-Error-Code": "code", + "X-Bkapi-Error-Message": "error-message", + "X-Bkapi-Request-Id": "abcdef", + }, + "error, status_code: 400, request_id: abcdef, error_code: code, error-message", + ), + ( + "error", + None, + None, + "error", + ), + ( + "error", + { + "status_code": 400, + }, { - "X-Bkapi-Error-Code": faker.pystr(), - "X-Bkapi-Error-Message": faker.pystr(), - "X-Bkapi-Request-Id": faker.uuid4(), - } + "X-Bkapi-Request-Id": "abcdef", + }, + "error, status_code: 400, request_id: abcdef", ), + ] + ) + def test_str(self, mocker, error, response, headers, expected): + err = exceptions.ResponseError( + error, + response=response and mocker.MagicMock(**response), + response_headers_representer=ResponseHeadersRepresenter(headers), ) - assert str(err).startswith("error") - assert "request_id:" in str(err) + assert str(err) == expected def test_error_code(self, mocker): err = exceptions.ResponseError( @@ -54,3 +80,59 @@ def test_request_id(self, mocker): response_headers_representer=ResponseHeadersRepresenter({"X-Bkapi-Request-Id": "abcdef"}), ) assert err.request_id == "abcdef" + + def test_curl_command(self, mocker, faker): + mocker.patch( + "bkapi_client_core.exceptions.CurlRequest.to_curl", return_value="curl -X GET http://example.com", + ) + err = exceptions.ResponseError( + faker.pystr(), + response=mocker.MagicMock(), + response_headers_representer=ResponseHeadersRepresenter({"X-Bkapi-Request-Id": "abcdef"}), + ) + assert err.curl_command == "curl -X GET http://example.com" + + @pytest.mark.parametrize( + "request_, expected", + [ + (None, None), + ({"method": "GET"}, "GET"), + ] + ) + def test_request_method(self, mocker, faker, request_, expected): + err = exceptions.ResponseError( + faker.pystr(), + request=request_ and mocker.MagicMock(**request_), + response_headers_representer=ResponseHeadersRepresenter({"X-Bkapi-Request-Id": "abcdef"}), + ) + assert err.request_method == expected + + @pytest.mark.parametrize( + "request_, expected", + [ + (None, ""), + ({"url": "http://example.com/test?foo=bar"}, "http://example.com/test"), + ] + ) + def test_request_url(self, mocker, faker, request_, expected): + err = exceptions.ResponseError( + faker.pystr(), + request=request_ and mocker.MagicMock(**request_), + response_headers_representer=ResponseHeadersRepresenter({"X-Bkapi-Request-Id": "abcdef"}), + ) + assert err.request_url == expected + + @pytest.mark.parametrize( + "response, expected", + [ + (None, None), + ({"status_code": 200}, 200), + ] + ) + def test_request_status_code(self, mocker, faker, response, expected): + err = exceptions.ResponseError( + faker.pystr(), + response=response and mocker.MagicMock(**response), + response_headers_representer=ResponseHeadersRepresenter({"X-Bkapi-Request-Id": "abcdef"}), + ) + assert err.response_status_code == expected diff --git a/sdks/bkapi-client-core/tests/test_utils.py b/sdks/bkapi-client-core/tests/test_utils.py index f7ebf49e..4d23a6af 100644 --- a/sdks/bkapi-client-core/tests/test_utils.py +++ b/sdks/bkapi-client-core/tests/test_utils.py @@ -9,6 +9,7 @@ * specific language governing permissions and limitations under the License. """ import pytest +import requests from bkapi_client_core import utils @@ -42,10 +43,11 @@ def test_to_curl(mocker, curlify, request_, expected): class TestCurlRequest: - def test_str(self, mocker): - curlify = mocker.patch.object(utils, "curlify") - curlify.to_curl.return_value = "curl http://example.com" - - request = mocker.MagicMock(url="http://example.com") - result = utils.CurlRequest(request) - assert str(result) == "curl http://example.com" + def test_str(self): + request = requests.Request("GET", "https://example.com/get").prepare() + result = utils.CurlRequest(request).to_curl() + assert result == "curl -X GET https://example.com/get" + + request = requests.Request("GET", "https://example.com/get", params={"foo": "bar"}, headers={"x-token": "test"}).prepare() + result = utils.CurlRequest(request).to_curl() + assert result == "curl -X GET 'https://example.com/get?foo=bar'" \ No newline at end of file From fdf082689a4750940c8211a1fbf9497abc6b0e7a Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Mon, 20 Nov 2023 15:39:30 +0800 Subject: [PATCH 03/10] add check_response_status/check_response_apigateway_error --- .../bkapi_client_core/client.py | 36 ++++++++++++++++ sdks/bkapi-client-core/tests/test_client.py | 42 ++++++++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/sdks/bkapi-client-core/bkapi_client_core/client.py b/sdks/bkapi-client-core/bkapi_client_core/client.py index df68632d..3ff3a6a9 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/client.py +++ b/sdks/bkapi-client-core/bkapi_client_core/client.py @@ -192,6 +192,42 @@ def parse_response( except RequestException as err: return self._handle_exception(operation, None, err) + def check_response_apigateway_error( + self, + response, # type: Optional[Response] + ): + # type: (...) -> None + """检查是否包含 apigateway 层面的报错,如应用认证失败,无访问 API 权限等""" + if response is None: + return + + response_headers_representer = ResponseHeadersRepresenter(response.headers) + if response_headers_representer.has_apigateway_error: + raise APIGatewayResponseError( + "Request bkapi error, %s" % response_headers_representer.error_message, + response=response, + response_headers_representer=response_headers_representer, + ) + + def check_response_status( + self, + response, # type: Optional[Response] + ): + # type: (...) -> None + """检查响应状态码,即 requests raise_for_status 校验""" + if response is None: + return + + try: + response.raise_for_status() + except HTTPError as err: + response_headers_representer = ResponseHeadersRepresenter(response.headers) + raise HTTPResponseError( + str(err), + response=response, + response_headers_representer=response_headers_representer, + ) + def update_headers( self, headers, # type: Dict[str, str] diff --git a/sdks/bkapi-client-core/tests/test_client.py b/sdks/bkapi-client-core/tests/test_client.py index cd0545a4..db7db363 100644 --- a/sdks/bkapi-client-core/tests/test_client.py +++ b/sdks/bkapi-client-core/tests/test_client.py @@ -9,12 +9,12 @@ * specific language governing permissions and limitations under the License. """ import pytest -from requests.exceptions import RequestException +from requests.exceptions import HTTPError, RequestException from bkapi_client_core.base import Operation, OperationGroup from bkapi_client_core.client import BaseClient, RequestContextBuilder, ResponseHeadersRepresenter from bkapi_client_core.config import HookEvent -from bkapi_client_core.exceptions import EndpointNotSetError, ResponseError +from bkapi_client_core.exceptions import EndpointNotSetError, ResponseError, APIGatewayResponseError, HTTPResponseError from bkapi_client_core.property import bind_property from bkapi_client_core.session import Session @@ -372,6 +372,44 @@ def test_parse_response_error(self, mocker): with pytest.raises(RequestException): # type: ignore self.client.parse_response(mocker.MagicMock(), mocker.MagicMock()) + @pytest.mark.parametrize( + "response, expected_error", + [ + ( + None, + None, + ), + ( + { + "headers": {}, + }, + None, + ), + ( + { + "headers": {"X-Bkapi-Error-Code": "error"}, + }, + APIGatewayResponseError, + ) + ] + ) + def test_check_response_apigateway_error(self, mocker, response, expected_error): + if not expected_error: + self.client.check_response_apigateway_error(response and mocker.MagicMock(**response)) + return + + with pytest.raises(expected_error): + self.client.check_response_apigateway_error(mocker.MagicMock(**response)) + + def test_check_response_status(self, mocker): + self.client.check_response_status(None) + self.client.check_response_status(mocker.MagicMock()) + + response = {"raise_for_status.side_effect": HTTPError()} + with pytest.raises(HTTPResponseError): + self.client.check_response_status(mocker.MagicMock(**response)) + + @pytest.mark.parametrize( "session_headers, headers, expected", [ From 3c9fde1c8e57ef24ea3a1a5ae15ea8c8578faaef Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Mon, 20 Nov 2023 17:10:11 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20change.log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/bkapi-client-core/CHANGE.md | 6 ++++++ .../bkapi_client_core/__init__.py | 2 +- sdks/bkapi-client-core/bkapi_client_core/client.py | 4 ++-- .../bkapi_client_core/exceptions.py | 14 ++++++-------- sdks/bkapi-client-core/pyproject.toml | 2 +- sdks/bkapi-client-core/tests/test_exceptions.py | 13 ++++++++++++- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/sdks/bkapi-client-core/CHANGE.md b/sdks/bkapi-client-core/CHANGE.md index 192f3f11..7d0468a2 100644 --- a/sdks/bkapi-client-core/CHANGE.md +++ b/sdks/bkapi-client-core/CHANGE.md @@ -1,5 +1,11 @@ ## Change logs +## 1.2.0 +- BK_API_CLIENT_ENABLE_SSL_VERIFY 默认值设置为 False +- BaseClient 处理响应内容时优先检查 json,添加辅助方法:check_response_apigateway_error, check_response_status +- ResponseError 添加 response_status_code/response_text/response_json 等辅助方法 +- 日志中,curl 信息中不再携带请求头 + ### 1.1.8 - 使用 `CurlRequest` 封装转换 curl 命令的逻辑以优化性能 diff --git a/sdks/bkapi-client-core/bkapi_client_core/__init__.py b/sdks/bkapi-client-core/bkapi_client_core/__init__.py index 19c3c6ec..d17d264b 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/__init__.py +++ b/sdks/bkapi-client-core/bkapi_client_core/__init__.py @@ -8,4 +8,4 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. """ -__version__ = "1.1.9" +__version__ = "1.2.0" diff --git a/sdks/bkapi-client-core/bkapi_client_core/client.py b/sdks/bkapi-client-core/bkapi_client_core/client.py index 3ff3a6a9..dc81598a 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/client.py +++ b/sdks/bkapi-client-core/bkapi_client_core/client.py @@ -302,10 +302,10 @@ def _handle_exception( # log exception if isinstance(exception, RequestException): response = exception.response - response_headers_representer = ResponseHeadersRepresenter(response and response.headers) + response_headers_representer = ResponseHeadersRepresenter(response.headers if response is not None else None) logger.warning( "request bkapi failed. status_code: %s, %s\n%s", - response and response.status_code, + response.status_code if response is not None else None, response_headers_representer, CurlRequest(exception.request), ) diff --git a/sdks/bkapi-client-core/bkapi_client_core/exceptions.py b/sdks/bkapi-client-core/bkapi_client_core/exceptions.py index 27850a6d..d5dbb095 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/exceptions.py +++ b/sdks/bkapi-client-core/bkapi_client_core/exceptions.py @@ -99,15 +99,16 @@ def request_url(self): @property def response_status_code(self): # type: (...) -> Optional[int] - return self.response and self.response.status_code + # bool(response) is equal to response.ok + return self.response.status_code if self.response is not None else None @property def response_text(self): # type: (...) -> Optional[str] - try: - return self.response and self.response.text - except Exception: - return None + return self.response.text if self.response is not None else None + + def response_json(self): + return self.response.json() if self.response is not None else None def __str__(self): if self.response is None: @@ -127,9 +128,6 @@ class APIGatewayResponseError(ResponseError): class HTTPResponseError(ResponseError): """HTTP request status code error""" - def response_json(self): - return self.response.json() - class JSONResponseError(ResponseError): """The response content is not a valid json""" diff --git a/sdks/bkapi-client-core/pyproject.toml b/sdks/bkapi-client-core/pyproject.toml index 993e25a4..88405b0a 100644 --- a/sdks/bkapi-client-core/pyproject.toml +++ b/sdks/bkapi-client-core/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bkapi-client-core" -version = "1.1.9" +version = "1.2.0" description = "A toolkit for buiding blueking API clients." readme = "README.md" authors = ["blueking "] diff --git a/sdks/bkapi-client-core/tests/test_exceptions.py b/sdks/bkapi-client-core/tests/test_exceptions.py index 3e14c729..71bb5858 100644 --- a/sdks/bkapi-client-core/tests/test_exceptions.py +++ b/sdks/bkapi-client-core/tests/test_exceptions.py @@ -127,12 +127,23 @@ def test_request_url(self, mocker, faker, request_, expected): [ (None, None), ({"status_code": 200}, 200), + ({"status_code": 400}, 400), ] ) - def test_request_status_code(self, mocker, faker, response, expected): + def test_response_status_code(self, mocker, faker, response, expected): err = exceptions.ResponseError( faker.pystr(), response=response and mocker.MagicMock(**response), response_headers_representer=ResponseHeadersRepresenter({"X-Bkapi-Request-Id": "abcdef"}), ) assert err.response_status_code == expected + + def test_response_json(self, mocker, faker): + err = exceptions.ResponseError(faker.pystr(), response=None) + assert err.response_json() == None + + err = exceptions.ResponseError( + faker.pystr(), + response=mocker.MagicMock(**{"json.return_value": {"foo": "bar"}}) + ) + assert err.response_json() == {"foo": "bar"} \ No newline at end of file From ef6c4034b36278da6706b252e0e09ef4f30c02e0 Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Mon, 20 Nov 2023 17:15:20 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E6=8F=90=E5=8F=96=E6=96=B9=E6=B3=95=20?= =?UTF-8?q?=5Fcheck=5Fresponse=5Fjson?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkapi_client_core/client.py | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/sdks/bkapi-client-core/bkapi_client_core/client.py b/sdks/bkapi-client-core/bkapi_client_core/client.py index dc81598a..9fe1649c 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/client.py +++ b/sdks/bkapi-client-core/bkapi_client_core/client.py @@ -342,35 +342,27 @@ def _handle_response_content( if response is None: return None - response_headers_representer = ResponseHeadersRepresenter(response.headers) - if response_headers_representer.has_apigateway_error: - raise APIGatewayResponseError( - "Request bkapi error, %s" % response_headers_representer.error_message, - response=response, - response_headers_representer=response_headers_representer, - ) + self.check_response_apigateway_error(response) # 先校验 json,状态码校验失败时,可以在 exception 中获取正常的 json 响应内容; # 如此,方便调用者在同一层中处理 http 状态码和 json 两个异常 + response_json = self._check_response_json(response) + + self.check_response_status(response) + + return response_json + + def _check_response_json(self, response): try: - response_json = response.json() + return response.json() except (TypeError, json.JSONDecodeError): + response_headers_representer = ResponseHeadersRepresenter(response.headers) raise JSONResponseError( "The response is not a valid JSON", response=response, response_headers_representer=response_headers_representer, ) - try: - response.raise_for_status() - except HTTPError as err: - raise HTTPResponseError( - str(err), - response=response, - response_headers_representer=response_headers_representer, - ) - - return response_json def close(self): """Close the session""" From c58c718e1528fac384693f0887b8882a8ea8f8e5 Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Mon, 20 Nov 2023 18:12:58 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E4=B8=8D=E6=9A=B4=E9=9C=B2=20check=5Fres?= =?UTF-8?q?ponse=5Fstatus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/bkapi-client-core/CHANGE.md | 6 +-- .../bkapi_client_core/client.py | 47 ++++++++----------- sdks/bkapi-client-core/tests/test_client.py | 11 +---- 3 files changed, 24 insertions(+), 40 deletions(-) diff --git a/sdks/bkapi-client-core/CHANGE.md b/sdks/bkapi-client-core/CHANGE.md index 7d0468a2..69fcd8bb 100644 --- a/sdks/bkapi-client-core/CHANGE.md +++ b/sdks/bkapi-client-core/CHANGE.md @@ -2,9 +2,9 @@ ## 1.2.0 - BK_API_CLIENT_ENABLE_SSL_VERIFY 默认值设置为 False -- BaseClient 处理响应内容时优先检查 json,添加辅助方法:check_response_apigateway_error, check_response_status -- ResponseError 添加 response_status_code/response_text/response_json 等辅助方法 -- 日志中,curl 信息中不再携带请求头 +- Client 添加辅助方法:check_response_apigateway_error +- ResponseError 添加辅助方法:response_status_code, response_text, response_json +- 日志中,curl 信息不再携带请求头 ### 1.1.8 - 使用 `CurlRequest` 封装转换 curl 命令的逻辑以优化性能 diff --git a/sdks/bkapi-client-core/bkapi_client_core/client.py b/sdks/bkapi-client-core/bkapi_client_core/client.py index 9fe1649c..18a28d44 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/client.py +++ b/sdks/bkapi-client-core/bkapi_client_core/client.py @@ -25,6 +25,7 @@ EndpointNotSetError, HTTPResponseError, JSONResponseError, + ResponseError, ) from bkapi_client_core.session import Session from bkapi_client_core.utils import CurlRequest, urljoin @@ -209,24 +210,6 @@ def check_response_apigateway_error( response_headers_representer=response_headers_representer, ) - def check_response_status( - self, - response, # type: Optional[Response] - ): - # type: (...) -> None - """检查响应状态码,即 requests raise_for_status 校验""" - if response is None: - return - - try: - response.raise_for_status() - except HTTPError as err: - response_headers_representer = ResponseHeadersRepresenter(response.headers) - raise HTTPResponseError( - str(err), - response=response, - response_headers_representer=response_headers_representer, - ) def update_headers( self, @@ -300,7 +283,7 @@ def _handle_exception( ): # type: (...) -> Optional[Response] # log exception - if isinstance(exception, RequestException): + if isinstance(exception, ResponseError): response = exception.response response_headers_representer = ResponseHeadersRepresenter(response.headers if response is not None else None) logger.warning( @@ -309,6 +292,15 @@ def _handle_exception( response_headers_representer, CurlRequest(exception.request), ) + elif isinstance(exception, RequestException): + response = exception.response + response_headers_representer = ResponseHeadersRepresenter(response.headers if response is not None else None) + logger.exception( + "request bkapi error. status_code: %s, %s\n%s", + response.status_code if response is not None else None, + response_headers_representer, + CurlRequest(exception.request), + ) else: logger.exception("request operation failed. operation: %s, context: %s", operation, context) @@ -344,15 +336,16 @@ def _handle_response_content( self.check_response_apigateway_error(response) - # 先校验 json,状态码校验失败时,可以在 exception 中获取正常的 json 响应内容; - # 如此,方便调用者在同一层中处理 http 状态码和 json 两个异常 - response_json = self._check_response_json(response) - - self.check_response_status(response) - - return response_json + try: + response.raise_for_status() + except HTTPError as err: + response_headers_representer = ResponseHeadersRepresenter(response.headers) + raise HTTPResponseError( + str(err), + response=response, + response_headers_representer=response_headers_representer, + ) - def _check_response_json(self, response): try: return response.json() except (TypeError, json.JSONDecodeError): diff --git a/sdks/bkapi-client-core/tests/test_client.py b/sdks/bkapi-client-core/tests/test_client.py index db7db363..28bd0107 100644 --- a/sdks/bkapi-client-core/tests/test_client.py +++ b/sdks/bkapi-client-core/tests/test_client.py @@ -401,15 +401,6 @@ def test_check_response_apigateway_error(self, mocker, response, expected_error) with pytest.raises(expected_error): self.client.check_response_apigateway_error(mocker.MagicMock(**response)) - def test_check_response_status(self, mocker): - self.client.check_response_status(None) - self.client.check_response_status(mocker.MagicMock()) - - response = {"raise_for_status.side_effect": HTTPError()} - with pytest.raises(HTTPResponseError): - self.client.check_response_status(mocker.MagicMock(**response)) - - @pytest.mark.parametrize( "session_headers, headers, expected", [ @@ -498,8 +489,8 @@ def test_handle_response_content(self, mocker): "response", [ {"headers": {"X-Bkapi-Error-Code": "error"}}, - {"json.side_effect": TypeError}, {"raise_for_status.side_effect": RequestException("error")}, + {"json.side_effect": TypeError}, ], ) def test_handle_response_content_error(self, mocker, response): From 2fd1d6a5c90c45f4ba494cd33cef18a1ef29f180 Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Mon, 20 Nov 2023 20:27:35 +0800 Subject: [PATCH 07/10] lint --- .../bkapi_client_core/client.py | 12 +++++++----- sdks/bkapi-client-core/demo/shortcuts.py | 3 ++- sdks/bkapi-client-core/tests/test_client.py | 14 +++++++------- .../tests/test_exceptions.py | 19 +++++++++---------- sdks/bkapi-client-core/tests/test_utils.py | 9 +++++++-- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/sdks/bkapi-client-core/bkapi_client_core/client.py b/sdks/bkapi-client-core/bkapi_client_core/client.py index 18a28d44..7016ced3 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/client.py +++ b/sdks/bkapi-client-core/bkapi_client_core/client.py @@ -195,7 +195,7 @@ def parse_response( def check_response_apigateway_error( self, - response, # type: Optional[Response] + response, # type: Optional[Response] ): # type: (...) -> None """检查是否包含 apigateway 层面的报错,如应用认证失败,无访问 API 权限等""" @@ -210,7 +210,6 @@ def check_response_apigateway_error( response_headers_representer=response_headers_representer, ) - def update_headers( self, headers, # type: Dict[str, str] @@ -285,7 +284,9 @@ def _handle_exception( # log exception if isinstance(exception, ResponseError): response = exception.response - response_headers_representer = ResponseHeadersRepresenter(response.headers if response is not None else None) + response_headers_representer = ResponseHeadersRepresenter( + response.headers if response is not None else None + ) logger.warning( "request bkapi failed. status_code: %s, %s\n%s", response.status_code if response is not None else None, @@ -294,7 +295,9 @@ def _handle_exception( ) elif isinstance(exception, RequestException): response = exception.response - response_headers_representer = ResponseHeadersRepresenter(response.headers if response is not None else None) + response_headers_representer = ResponseHeadersRepresenter( + response.headers if response is not None else None + ) logger.exception( "request bkapi error. status_code: %s, %s\n%s", response.status_code if response is not None else None, @@ -356,7 +359,6 @@ def _handle_response_content( response_headers_representer=response_headers_representer, ) - def close(self): """Close the session""" self.session.close() diff --git a/sdks/bkapi-client-core/demo/shortcuts.py b/sdks/bkapi-client-core/demo/shortcuts.py index 945c42c9..8f3a2989 100644 --- a/sdks/bkapi-client-core/demo/shortcuts.py +++ b/sdks/bkapi-client-core/demo/shortcuts.py @@ -8,10 +8,11 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. """ -from .client import Client from bkapi_client_core.apigateway import generic_type_partial as _partial from bkapi_client_core.apigateway.django_helper import get_client_by_request as _get_client_by_request from bkapi_client_core.apigateway.django_helper import get_client_by_username as _get_client_by_username +from .client import Client + get_client_by_request = _partial(Client, _get_client_by_request) get_client_by_username = _partial(Client, _get_client_by_username) diff --git a/sdks/bkapi-client-core/tests/test_client.py b/sdks/bkapi-client-core/tests/test_client.py index 28bd0107..0d68254d 100644 --- a/sdks/bkapi-client-core/tests/test_client.py +++ b/sdks/bkapi-client-core/tests/test_client.py @@ -9,19 +9,19 @@ * specific language governing permissions and limitations under the License. """ import pytest -from requests.exceptions import HTTPError, RequestException +from requests.exceptions import RequestException from bkapi_client_core.base import Operation, OperationGroup from bkapi_client_core.client import BaseClient, RequestContextBuilder, ResponseHeadersRepresenter from bkapi_client_core.config import HookEvent -from bkapi_client_core.exceptions import EndpointNotSetError, ResponseError, APIGatewayResponseError, HTTPResponseError +from bkapi_client_core.exceptions import APIGatewayResponseError, EndpointNotSetError, ResponseError from bkapi_client_core.property import bind_property from bkapi_client_core.session import Session class TestRequestContextBuilder: @pytest.fixture(autouse=True) - def setup(self, faker): + def setup(self): self.builder = RequestContextBuilder() @pytest.mark.parametrize( @@ -237,7 +237,7 @@ def test_has_apigateway_error(self, headers, expected): }, "request_id: abcdef", ), - ] + ], ) def test_str(self, headers, expected): representer = ResponseHeadersRepresenter(headers) @@ -381,7 +381,7 @@ def test_parse_response_error(self, mocker): ), ( { - "headers": {}, + "headers": {}, }, None, ), @@ -390,8 +390,8 @@ def test_parse_response_error(self, mocker): "headers": {"X-Bkapi-Error-Code": "error"}, }, APIGatewayResponseError, - ) - ] + ), + ], ) def test_check_response_apigateway_error(self, mocker, response, expected_error): if not expected_error: diff --git a/sdks/bkapi-client-core/tests/test_exceptions.py b/sdks/bkapi-client-core/tests/test_exceptions.py index 71bb5858..bc337132 100644 --- a/sdks/bkapi-client-core/tests/test_exceptions.py +++ b/sdks/bkapi-client-core/tests/test_exceptions.py @@ -9,7 +9,6 @@ * specific language governing permissions and limitations under the License. """ import pytest -import requests from bkapi_client_core import exceptions from bkapi_client_core.client import ResponseHeadersRepresenter @@ -47,7 +46,7 @@ class TestResponseError: }, "error, status_code: 400, request_id: abcdef", ), - ] + ], ) def test_str(self, mocker, error, response, headers, expected): err = exceptions.ResponseError( @@ -83,7 +82,8 @@ def test_request_id(self, mocker): def test_curl_command(self, mocker, faker): mocker.patch( - "bkapi_client_core.exceptions.CurlRequest.to_curl", return_value="curl -X GET http://example.com", + "bkapi_client_core.exceptions.CurlRequest.to_curl", + return_value="curl -X GET http://example.com", ) err = exceptions.ResponseError( faker.pystr(), @@ -97,7 +97,7 @@ def test_curl_command(self, mocker, faker): [ (None, None), ({"method": "GET"}, "GET"), - ] + ], ) def test_request_method(self, mocker, faker, request_, expected): err = exceptions.ResponseError( @@ -112,7 +112,7 @@ def test_request_method(self, mocker, faker, request_, expected): [ (None, ""), ({"url": "http://example.com/test?foo=bar"}, "http://example.com/test"), - ] + ], ) def test_request_url(self, mocker, faker, request_, expected): err = exceptions.ResponseError( @@ -128,7 +128,7 @@ def test_request_url(self, mocker, faker, request_, expected): (None, None), ({"status_code": 200}, 200), ({"status_code": 400}, 400), - ] + ], ) def test_response_status_code(self, mocker, faker, response, expected): err = exceptions.ResponseError( @@ -140,10 +140,9 @@ def test_response_status_code(self, mocker, faker, response, expected): def test_response_json(self, mocker, faker): err = exceptions.ResponseError(faker.pystr(), response=None) - assert err.response_json() == None + assert err.response_json() is None err = exceptions.ResponseError( - faker.pystr(), - response=mocker.MagicMock(**{"json.return_value": {"foo": "bar"}}) + faker.pystr(), response=mocker.MagicMock(**{"json.return_value": {"foo": "bar"}}) ) - assert err.response_json() == {"foo": "bar"} \ No newline at end of file + assert err.response_json() == {"foo": "bar"} diff --git a/sdks/bkapi-client-core/tests/test_utils.py b/sdks/bkapi-client-core/tests/test_utils.py index 4d23a6af..714bc355 100644 --- a/sdks/bkapi-client-core/tests/test_utils.py +++ b/sdks/bkapi-client-core/tests/test_utils.py @@ -48,6 +48,11 @@ def test_str(self): result = utils.CurlRequest(request).to_curl() assert result == "curl -X GET https://example.com/get" - request = requests.Request("GET", "https://example.com/get", params={"foo": "bar"}, headers={"x-token": "test"}).prepare() + request = requests.Request( + "GET", + "https://example.com/get", + params={"foo": "bar"}, + headers={"x-token": "test"}, + ).prepare() result = utils.CurlRequest(request).to_curl() - assert result == "curl -X GET 'https://example.com/get?foo=bar'" \ No newline at end of file + assert result == "curl -X GET 'https://example.com/get?foo=bar'" From 9130e40305c424c5434fcf91d94e1a9896c1a910 Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Mon, 20 Nov 2023 23:01:17 +0800 Subject: [PATCH 08/10] =?UTF-8?q?curl=20=E4=B8=AD=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=AF=B7=E6=B1=82=E5=A4=B4=EF=BC=8C=E4=BD=86=E6=98=AF=E5=8E=BB?= =?UTF-8?q?=E9=99=A4=E5=85=B6=E4=B8=AD=E7=9A=84=E6=95=8F=E6=84=9F=E4=BF=A1?= =?UTF-8?q?=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkapi_client_core/utils.py | 42 ++++++- sdks/bkapi-client-core/tests/test_utils.py | 104 ++++++++++++++---- 2 files changed, 123 insertions(+), 23 deletions(-) diff --git a/sdks/bkapi-client-core/bkapi_client_core/utils.py b/sdks/bkapi-client-core/bkapi_client_core/utils.py index 7905b936..9a2361e4 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/utils.py +++ b/sdks/bkapi-client-core/bkapi_client_core/utils.py @@ -9,6 +9,7 @@ * specific language governing permissions and limitations under the License. """ import copy +import json from functools import wraps from typing import Callable, Optional, Type, TypeVar @@ -27,16 +28,55 @@ def urljoin(base_url, path): return "%s/%s" % (base_url.rstrip("/"), path.lstrip("/")) +class _SensitiveCleaner: + """处理敏感信息""" + + def __init__(self, sensitive_keys): + self.sensitive_keys = sensitive_keys + + def clean(self, data): + data = copy.deepcopy(data) + self._clean(data) + return data + + def _clean(self, data): + if isinstance(data, dict): + for key, value in data.items(): + if isinstance(value, (dict, list)): + self._clean(value) + + elif key in self.sensitive_keys and value: + data[key] = "***" + + elif isinstance(data, list): + for item in data: + self._clean(item) + + class _WrappedRequest: + header_bkapi_authorization = "X-Bkapi-Authorization" + def __init__(self, request): self._request = request # 请求头中可能有敏感信息,不打印请求头 - self.headers = {} + self.headers = self._get_headers_without_sensitive(self._request.headers) def __getattr__(self, name): return getattr(self._request, name) + def _get_headers_without_sensitive(self, headers): + headers = copy.deepcopy(headers) + + authorization = headers.get(self.header_bkapi_authorization) + if authorization: + sensitive_cleaner = _SensitiveCleaner( + ["bk_app_secret", "app_secret", "bk_token", "bk_ticket", "access_token"] + ) + headers[self.header_bkapi_authorization] = json.dumps(sensitive_cleaner.clean(json.loads(authorization))) + + return headers + class CurlRequest: def __init__( diff --git a/sdks/bkapi-client-core/tests/test_utils.py b/sdks/bkapi-client-core/tests/test_utils.py index 714bc355..4c25304e 100644 --- a/sdks/bkapi-client-core/tests/test_utils.py +++ b/sdks/bkapi-client-core/tests/test_utils.py @@ -14,6 +14,11 @@ from bkapi_client_core import utils +@pytest.fixture +def fake_request(): + return requests.Request("GET", "https://example.com/get").prepare() + + @pytest.mark.parametrize( "base_url, path, expected", [ @@ -29,30 +34,85 @@ def test_urljoin(base_url, path, expected): assert result == expected -@pytest.mark.parametrize( - "curlify, request_, expected", - [ - ({"to_curl.return_value": "curl http://example.com"}, None, ""), - ({"to_curl.return_value": "curl http://example.com"}, {"a": "b"}, "curl http://example.com"), - ], -) -def test_to_curl(mocker, curlify, request_, expected): - mocker.patch.object(utils, "curlify", curlify and mocker.MagicMock(**curlify)) - result = utils.to_curl(request_ and mocker.MagicMock(**request_)) - assert result == expected +def test_to_curl(fake_request): + request = None + result = utils.to_curl(request) + assert result == "" + + fake_request.prepare_headers({"foo": "bar"}) + result = utils.to_curl(fake_request) + assert result == "curl -X GET -H 'foo: bar' https://example.com/get" + + +class TestSensitiveCleaner: + @pytest.mark.parametrize( + "data, expected", + [ + ( + { + "secret": "bar", + "foo1": { + "secret": "baz", + "foo2": { + "secret": "ba2", + }, + }, + "foo3": [{"secret": "s3"}], + "foo4": [1, 2], + }, + { + "secret": "***", + "foo1": { + "secret": "***", + "foo2": { + "secret": "***", + }, + }, + "foo3": [{"secret": "***"}], + "foo4": [1, 2], + }, + ) + ], + ) + def test_clean(self, data, expected): + sensitive_cleaner = utils._SensitiveCleaner(["secret"]) + result = sensitive_cleaner.clean(data) + assert result == expected + + +class TestWrappedRequest: + @pytest.mark.parametrize( + "headers, expected", + [ + ( + {}, + {}, + ), + ( + {"x-bkapi-authorization": '{"bk_app_code": "foo", "bk_app_secret": "bar"}'}, + {"X-Bkapi-Authorization": '{"bk_app_code": "foo", "bk_app_secret": "***"}'}, + ), + ( + {"x-bkapi-authorization": '{"bk_app_code": "foo", "bk_app_secret": "bar", "bk_token": ""}'}, + {"X-Bkapi-Authorization": '{"bk_app_code": "foo", "bk_app_secret": "***", "bk_token": ""}'}, + ), + ], + ) + def test_get_headers_without_sensitive(self, fake_request, headers, expected): + fake_request.prepare_headers(headers) + wrapped_request = utils._WrappedRequest(fake_request) + assert wrapped_request.headers == expected class TestCurlRequest: - def test_str(self): - request = requests.Request("GET", "https://example.com/get").prepare() - result = utils.CurlRequest(request).to_curl() + def test_str(self, fake_request): + result = utils.CurlRequest(fake_request).to_curl() assert result == "curl -X GET https://example.com/get" - request = requests.Request( - "GET", - "https://example.com/get", - params={"foo": "bar"}, - headers={"x-token": "test"}, - ).prepare() - result = utils.CurlRequest(request).to_curl() - assert result == "curl -X GET 'https://example.com/get?foo=bar'" + fake_request.prepare_url("https://example.com/get", {"foo": "bar"}) + fake_request.prepare_headers({"x-token": "test", "x-bkapi-authorization": '{"bk_app_secret": "bar"}'}) + result = utils.CurlRequest(fake_request).to_curl() + assert result == ( + "curl -X GET -H 'X-Bkapi-Authorization: {\"bk_app_secret\": \"***\"}' " + "-H 'x-token: test' 'https://example.com/get?foo=bar'" + ) From 0a364941a8ecf9a066f5222cc209bd792809fe24 Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Mon, 20 Nov 2023 23:34:19 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20change.log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/bkapi-client-core/CHANGE.md | 2 +- sdks/bkapi-client-core/bkapi_client_core/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/bkapi-client-core/CHANGE.md b/sdks/bkapi-client-core/CHANGE.md index 69fcd8bb..c3f7a756 100644 --- a/sdks/bkapi-client-core/CHANGE.md +++ b/sdks/bkapi-client-core/CHANGE.md @@ -4,7 +4,7 @@ - BK_API_CLIENT_ENABLE_SSL_VERIFY 默认值设置为 False - Client 添加辅助方法:check_response_apigateway_error - ResponseError 添加辅助方法:response_status_code, response_text, response_json -- 日志中,curl 信息不再携带请求头 +- 日志中,curl 信息请求头 X-Bkapi-Authorization 脱敏 ### 1.1.8 - 使用 `CurlRequest` 封装转换 curl 命令的逻辑以优化性能 diff --git a/sdks/bkapi-client-core/bkapi_client_core/utils.py b/sdks/bkapi-client-core/bkapi_client_core/utils.py index 9a2361e4..ebb02056 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/utils.py +++ b/sdks/bkapi-client-core/bkapi_client_core/utils.py @@ -59,7 +59,7 @@ class _WrappedRequest: def __init__(self, request): self._request = request - # 请求头中可能有敏感信息,不打印请求头 + # 去除请求头中的敏感信息 self.headers = self._get_headers_without_sensitive(self._request.headers) def __getattr__(self, name): From 0f7ca5a96fa236177ed56bc0800b54109b9ba0d0 Mon Sep 17 00:00:00 2001 From: alex-smile <443677891@qq.com> Date: Tue, 21 Nov 2023 11:14:55 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E6=B6=88=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkapi_client_core/client.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/sdks/bkapi-client-core/bkapi_client_core/client.py b/sdks/bkapi-client-core/bkapi_client_core/client.py index 7016ced3..33beb24b 100644 --- a/sdks/bkapi-client-core/bkapi_client_core/client.py +++ b/sdks/bkapi-client-core/bkapi_client_core/client.py @@ -205,7 +205,7 @@ def check_response_apigateway_error( response_headers_representer = ResponseHeadersRepresenter(response.headers) if response_headers_representer.has_apigateway_error: raise APIGatewayResponseError( - "Request bkapi error, %s" % response_headers_representer.error_message, + "Error responded by API Gateway", response=response, response_headers_representer=response_headers_representer, ) @@ -283,23 +283,14 @@ def _handle_exception( # type: (...) -> Optional[Response] # log exception if isinstance(exception, ResponseError): - response = exception.response - response_headers_representer = ResponseHeadersRepresenter( - response.headers if response is not None else None - ) - logger.warning( - "request bkapi failed. status_code: %s, %s\n%s", - response.status_code if response is not None else None, - response_headers_representer, - CurlRequest(exception.request), - ) + logger.warning("%s\n%s", str(exception), CurlRequest(exception.request)) elif isinstance(exception, RequestException): response = exception.response response_headers_representer = ResponseHeadersRepresenter( response.headers if response is not None else None ) logger.exception( - "request bkapi error. status_code: %s, %s\n%s", + "Request bkapi error, status_code: %s, %s\n%s", response.status_code if response is not None else None, response_headers_representer, CurlRequest(exception.request), @@ -344,7 +335,7 @@ def _handle_response_content( except HTTPError as err: response_headers_representer = ResponseHeadersRepresenter(response.headers) raise HTTPResponseError( - str(err), + "Error responded by Backend api, %s" % str(err), response=response, response_headers_representer=response_headers_representer, )