diff --git a/generate/config-template.json b/generate/config-template.json index c9a15f2..8c5f648 100644 --- a/generate/config-template.json +++ b/generate/config-template.json @@ -50,6 +50,9 @@ }, "extensions/file_access_token.mustache": { "destinationFilename": "${PACKAGE_NAME}/extensions/file_access_token.py" + }, + "extensions/configuration_options.mustache": { + "destinationFilename": "${PACKAGE_NAME}/extensions/configuration_options.py" } } } \ No newline at end of file diff --git a/generate/templates/api.mustache b/generate/templates/api.mustache index ffd391e..3200a57 100644 --- a/generate/templates/api.mustache +++ b/generate/templates/api.mustache @@ -21,6 +21,7 @@ from {{packageName}}.exceptions import ( # noqa: F401 ApiTypeError, ApiValueError ) +from {{packageName}}.extensions.configuration_options import ConfigurationOptions {{#operations}} @@ -66,10 +67,9 @@ class {{classname}}: {{/allParams}} :param async_req: Whether to execute the request asynchronously. :type async_req: bool, optional - :param _request_timeout: timeout setting for this request. - If one number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional :return: Returns the result object. If the method is called asynchronously, returns the request thread. @@ -112,10 +112,9 @@ class {{classname}}: :param _return_http_data_only: response data instead of ApiResponse object with status code, headers, etc :type _return_http_data_only: bool, optional - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional :param _request_auth: set to override the auth_settings for an a single request; this effectively ignores the authentication in the spec for a single request. @@ -162,7 +161,8 @@ class {{classname}}: '_request_timeout', '_request_auth', '_content_type', - '_headers' + '_headers', + 'opts' ] ) @@ -304,6 +304,7 @@ class {{classname}}: _return_http_data_only=_params.get('_return_http_data_only'), # noqa: E501 _preload_content=_params.get('_preload_content', True), _request_timeout=_params.get('_request_timeout'), + opts=_params.get('opts'), {{#servers.0}} _host=_host, {{/servers.0}} diff --git a/generate/templates/api_client.mustache b/generate/templates/api_client.mustache index 5988d7e..81d8e13 100644 --- a/generate/templates/api_client.mustache +++ b/generate/templates/api_client.mustache @@ -162,7 +162,7 @@ class ApiClient: files=None, response_types_map=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None, - _request_auth=None): + _request_auth=None, opts=None): config = self.configuration @@ -230,7 +230,8 @@ class ApiClient: headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, - _request_timeout=_request_timeout) + _request_timeout=_request_timeout, + opts=opts) except ApiException as e: if e.body: e.body = e.body.decode('utf-8') @@ -390,7 +391,8 @@ class ApiClient: response_types_map=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, - _request_timeout=None, _host=None, _request_auth=None): + _request_timeout=None, _host=None, _request_auth=None, + opts=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. @@ -417,10 +419,9 @@ class ApiClient: Default is True. :param collection_formats: dict of collection formats for path, query, header, and post parameters. - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional :param _request_auth: set to override the auth_settings for an a single request; this effectively ignores the authentication in the spec for a single request. @@ -439,7 +440,7 @@ class ApiClient: response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, - _request_auth) + _request_auth, opts) return self.pool.apply_async(self.__call_api, (resource_path, method, path_params, @@ -452,30 +453,34 @@ class ApiClient: collection_formats, _preload_content, _request_timeout, - _host, _request_auth)) + _host, _request_auth, + opts)) def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): """Makes the HTTP request using RESTClient.""" if method == "GET": return self.rest_client.get_request(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - headers=headers) + headers=headers, + opts=opts) elif method == "HEAD": return self.rest_client.head_request(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - headers=headers) + headers=headers, + opts=opts) elif method == "OPTIONS": return self.rest_client.options_request(url, query_params=query_params, headers=headers, _preload_content=_preload_content, - _request_timeout=_request_timeout) + _request_timeout=_request_timeout, + opts=opts) elif method == "POST": return self.rest_client.post_request(url, query_params=query_params, @@ -483,7 +488,8 @@ class ApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "PUT": return self.rest_client.put_request(url, query_params=query_params, @@ -491,7 +497,8 @@ class ApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "PATCH": return self.rest_client.patch_request(url, query_params=query_params, @@ -499,14 +506,16 @@ class ApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "DELETE": return self.rest_client.delete_request(url, query_params=query_params, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) else: raise ApiValueError( "http method must be `GET`, `HEAD`, `OPTIONS`," diff --git a/generate/templates/asyncio/rest.mustache b/generate/templates/asyncio/rest.mustache index 314b404..100024e 100644 --- a/generate/templates/asyncio/rest.mustache +++ b/generate/templates/asyncio/rest.mustache @@ -58,6 +58,11 @@ class RESTClientObject: self.proxy = configuration.proxy self.proxy_headers = configuration.proxy_headers + self.timeout = aiohttp.ClientTimeout( + total=configuration.timeouts.total_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.total_timeout_ms != None else None, + connect=configuration.timeouts.connect_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.connect_timeout_ms != None else None, + sock_read=configuration.timeouts.read_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.read_timeout_ms != None else None, + ) # https pool manager self.pool_manager = aiohttp.ClientSession( @@ -70,7 +75,7 @@ class RESTClientObject: async def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): """Execute request :param method: http request method @@ -83,10 +88,9 @@ class RESTClientObject: and `multipart/form-data` :param _preload_content: this is a non-applicable field for the AiohttpClient. - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', @@ -102,7 +106,44 @@ class RESTClientObject: # url already contains the URL query string # so reset query_params to empty dict query_params = {} - timeout = _request_timeout or 5 * 60 + + # _request_timeout param cannot be removed for backwards compatability + # values from opts override values from _request_timeout + # try to get values from opts first, then _request_timeout, then self.timeout, else set to None + # timeout = _request_timeout or self.timeout + timeout = None + opts_total_timeout = opts.total_timeout_ms / 1000.0 if opts and opts.total_timeout_ms != None else None + opts_connect_timeout = opts.connect_timeout_ms / 1000.0 if opts and opts.connect_timeout_ms != None else None + opts_read_timeout = opts.read_timeout_ms / 1000.0 if opts and opts.read_timeout_ms != None else None + if not _request_timeout: + timeout = aiohttp.ClientTimeout( + total=opts_total_timeout if opts_total_timeout != None + else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None + else self.timeout.connect, + sock_read=opts_read_timeout if opts_read_timeout != None + else self.timeout.sock_read) + elif isinstance(_request_timeout, aiohttp.ClientTimeout): + timeout = aiohttp.ClientTimeout( + total=opts_total_timeout if opts_total_timeout != None + else _request_timeout.total if _request_timeout.total != None + else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None + else _request_timeout.connect if _request_timeout.connect != None + else self.timeout.connect, + sock_read=opts_read_timeout if opts_read_timeout != None + else _request_timeout.sock_read if _request_timeout.sock_read != None + else self.timeout.sock_read) + elif isinstance(_request_timeout, (int, float)): + timeout = aiohttp.ClientTimeout( + total=opts_total_timeout if opts_total_timeout != None + else _request_timeout, + connect=opts_connect_timeout if opts_connect_timeout != None + else self.timeout.connect, + sock_read=opts_read_timeout if opts_read_timeout != None + else self.timeout.sock_read) + else: + raise f"unexpected type '{type(_request_timeout)}' for _request_timeout" if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' @@ -176,69 +217,78 @@ class RESTClientObject: return r async def get_request(self, url, headers=None, query_params=None, - _preload_content=True, _request_timeout=None): + _preload_content=True, _request_timeout=None, opts=None): return (await self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params)) + query_params=query_params, + opts=opts)) async def head_request(self, url, headers=None, query_params=None, - _preload_content=True, _request_timeout=None): + _preload_content=True, _request_timeout=None, opts=None): return (await self.request("HEAD", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params)) + query_params=query_params, + opts=opts)) async def options_request(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return (await self.request("OPTIONS", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def delete_request(self, url, headers=None, query_params=None, body=None, - _preload_content=True, _request_timeout=None): + _preload_content=True, _request_timeout=None, + opts=None): return (await self.request("DELETE", url, headers=headers, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def post_request(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return (await self.request("POST", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def put_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, + opts=None): return (await self.request("PUT", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def patch_request(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return (await self.request("PATCH", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) diff --git a/generate/templates/configuration.mustache b/generate/templates/configuration.mustache index 706909b..60820f5 100644 --- a/generate/templates/configuration.mustache +++ b/generate/templates/configuration.mustache @@ -19,6 +19,56 @@ JSON_SCHEMA_VALIDATION_KEYWORDS = { 'minLength', 'pattern', 'maxItems', 'minItems' } +class Timeouts: + + @property + def total_timeout_ms(self): + return self.__total_timeout_ms + + @total_timeout_ms.setter + def total_timeout_ms(self, value): + if not isinstance(value, int): + raise TypeError(f"total_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise TypeError(f"total_timeout_ms must be an integer greater than or equal to zero") + self.__total_timeout_ms = value + + @property + def connect_timeout_ms(self): + return self.__connect_timeout_ms + + @connect_timeout_ms.setter + def connect_timeout_ms(self, value): + if not isinstance(value, int): + raise TypeError(f"connect_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise TypeError(f"connect_timeout_ms must be an integer greater than or equal to zero") + self.__connect_timeout_ms = value + + @property + def read_timeout_ms(self): + return self.__read_timeout_ms + + @read_timeout_ms.setter + def read_timeout_ms(self, value): + if not isinstance(value, int): + raise TypeError(f"read_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise TypeError(f"read_timeout_ms must be an integer greater than or equal to zero") + self.__read_timeout_ms = value + + def __init__(self, total_timeout_ms: int, connect_timeout_ms: int, read_timeout_ms: int): + self.total_timeout_ms = total_timeout_ms + self.connect_timeout_ms = connect_timeout_ms + self.read_timeout_ms = read_timeout_ms + + def get_default(): + return Timeouts( + total_timeout_ms=Configuration.DEFAULT_TOTAL_TIMEOUT_MS, + connect_timeout_ms=Configuration.DEFAULT_CONNECT_TIMEOUT_MS, + read_timeout_ms=Configuration.DEFAULT_READ_TIMEOUT_MS + ) + class Configuration: """This class contains various settings of the API client. @@ -134,6 +184,12 @@ conf = {{{packageName}}}.Configuration( {{/hasHttpSignatureMethods}} {{/hasAuthMethods}} """ + + DEFAULT_TOTAL_TIMEOUT_MS: int = 1800_000 + DEFAULT_CONNECT_TIMEOUT_MS: int = 0 + DEFAULT_READ_TIMEOUT_MS: int = 0 + DEFAULT_RATE_LIMIT_RETRIES: int = 2 + DEFAULT_RETRIES: int = 3 _default = None @@ -147,7 +203,11 @@ conf = {{{packageName}}}.Configuration( server_index=None, server_variables=None, server_operation_index=None, server_operation_variables=None, ssl_ca_cert=None, - ) -> None: + timeouts=Timeouts( + total_timeout_ms=DEFAULT_TOTAL_TIMEOUT_MS, + connect_timeout_ms=DEFAULT_CONNECT_TIMEOUT_MS, + read_timeout_ms=DEFAULT_READ_TIMEOUT_MS), + rate_limit_retries=DEFAULT_RATE_LIMIT_RETRIES) -> None: """Constructor """ self._base_path = "{{{basePath}}}" if host is None else host @@ -279,6 +339,9 @@ conf = {{{packageName}}}.Configuration( self.date_format = "{{{dateFormat}}}" """date format """ + + self.timeouts = timeouts + self.rate_limit_retries = rate_limit_retries def __deepcopy__(self, memo): cls = self.__class__ @@ -641,3 +704,23 @@ conf = {{{packageName}}}.Configuration( """Fix base path.""" self._base_path = value self.server_index = None + + @property + def timeouts(self): + return self._timeouts + + @timeouts.setter + def timeouts(self, value): + if not isinstance(value, Timeouts): + raise TypeError(f"timeouts must be type Timeouts but type '{type(value)}' used") + self._timeouts = value + + @property + def rate_limit_retries(self): + return self._rate_limit_retries + + @rate_limit_retries.setter + def rate_limit_retries(self, value): + if not isinstance(value, int): + raise TypeError(f"rate_limit_retries must be type int but type '{type(value)}' used") + self._rate_limit_retries = value diff --git a/generate/templates/extensions/api_client.mustache b/generate/templates/extensions/api_client.mustache index af936fb..3179709 100644 --- a/generate/templates/extensions/api_client.mustache +++ b/generate/templates/extensions/api_client.mustache @@ -150,7 +150,7 @@ class SyncApiClient: files=None, response_types_map=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None, - _request_auth=None): + _request_auth=None, opts=None): config = self.configuration @@ -218,7 +218,8 @@ class SyncApiClient: headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, - _request_timeout=_request_timeout) + _request_timeout=_request_timeout, + opts=opts) except ApiException as e: if e.body: e.body = e.body.decode('utf-8') @@ -378,7 +379,8 @@ class SyncApiClient: response_types_map=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, - _request_timeout=None, _host=None, _request_auth=None): + _request_timeout=None, _host=None, _request_auth=None, + opts=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. @@ -405,10 +407,9 @@ class SyncApiClient: Default is True. :param collection_formats: dict of collection formats for path, query, header, and post parameters. - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional :param _request_auth: set to override the auth_settings for an a single request; this effectively ignores the authentication in the spec for a single request. @@ -427,7 +428,7 @@ class SyncApiClient: response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, - _request_auth) + _request_auth, opts) return self.pool.apply_async(self.__call_api, (resource_path, method, path_params, @@ -440,30 +441,33 @@ class SyncApiClient: collection_formats, _preload_content, _request_timeout, - _host, _request_auth)) + _host, _request_auth, opts)) def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): """Makes the HTTP request using RESTClient.""" if method == "GET": return self.rest_client.get_request(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - headers=headers) + headers=headers, + opts=opts) elif method == "HEAD": return self.rest_client.head_request(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - headers=headers) + headers=headers, + opts=opts) elif method == "OPTIONS": return self.rest_client.options_request(url, query_params=query_params, headers=headers, _preload_content=_preload_content, - _request_timeout=_request_timeout) + _request_timeout=_request_timeout, + opts=opts) elif method == "POST": return self.rest_client.post_request(url, query_params=query_params, @@ -471,7 +475,8 @@ class SyncApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "PUT": return self.rest_client.put_request(url, query_params=query_params, @@ -479,7 +484,8 @@ class SyncApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "PATCH": return self.rest_client.patch_request(url, query_params=query_params, @@ -487,14 +493,16 @@ class SyncApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "DELETE": return self.rest_client.delete_request(url, query_params=query_params, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) else: raise ApiValueError( "http method must be `GET`, `HEAD`, `OPTIONS`," diff --git a/generate/templates/extensions/api_client_factory.mustache b/generate/templates/extensions/api_client_factory.mustache index f1c6372..0dd9a8e 100644 --- a/generate/templates/extensions/api_client_factory.mustache +++ b/generate/templates/extensions/api_client_factory.mustache @@ -1,5 +1,6 @@ from __future__ import annotations from {{packageName}}.api_client import ApiClient +from {{packageName}}.extensions.configuration_options import ConfigurationOptions from {{packageName}}.extensions.retry import ( RetryingRestWrapper, RetryingRestWrapperAsync @@ -68,6 +69,7 @@ class SyncApiClientFactory: ] = keep_alive_socket_options(), correlation_id: Optional[str] = None, app_name: Optional[str] = None, + opts: Optional[ConfigurationOptions] = None, ): """Create an ApiClientFactory which can build api objects with a configured ApiClient object @@ -96,7 +98,8 @@ class SyncApiClientFactory: api_client_config = api_config.build_api_client_config( tcp_keep_alive=tcp_keep_alive, socket_options=socket_options, - id_provider_response_handler=id_provider_response_handler + id_provider_response_handler=id_provider_response_handler, + opts=opts ) self.__api_client = SyncApiClient( configuration=api_client_config, @@ -107,7 +110,9 @@ class SyncApiClientFactory: if tcp_keep_alive: rc.pool_manager.pool_classes_by_scheme = {"http": TCPKeepAliveHTTPConnectionPool, "https": TCPKeepAliveHTTPSConnectionPool} - wrapped_rest_client = rest_client_wrapper(rc) + wrapped_rest_client = rest_client_wrapper( + rest_object=rc, + rate_limit_retries=api_client_config.rate_limit_retries) self.__api_client.rest_client = wrapped_rest_client set_additional_api_client_headers( @@ -153,7 +158,8 @@ class ApiClientFactory: correlation_id: Optional[str] = None, app_name: Optional[str] = None, client_session: Optional[ClientSession] = None, - trace_configs: Optional[List[TraceConfig]] = None + trace_configs: Optional[List[TraceConfig]] = None, + opts: Optional[ConfigurationOptions] = None, ): """Create an ApiClientFactory which can build api objects with a configured ApiClient object @@ -189,7 +195,8 @@ class ApiClientFactory: api_client_config = api_config.build_api_client_config( tcp_keep_alive=tcp_keep_alive, socket_options=socket_options, - id_provider_response_handler=id_provider_response_handler + id_provider_response_handler=id_provider_response_handler, + opts=opts ) self.__api_client = ApiClient( configuration=api_client_config, @@ -218,7 +225,9 @@ class ApiClientFactory: logger.exception("client_session must be an aiohttp.ClientSession" " object with an initialised TCP Connector") rest_client_wrapper = RetryingRestWrapperAsync - wrapped_rest_client = rest_client_wrapper(rc) + wrapped_rest_client = rest_client_wrapper( + rest_object=rc, + rate_limit_retries=api_client_config.rate_limit_retries) self.__api_client.rest_client = wrapped_rest_client set_additional_api_client_headers( self.__api_client, app_name=app_name, correlation_id=correlation_id diff --git a/generate/templates/extensions/api_configuration.mustache b/generate/templates/extensions/api_configuration.mustache index 9cfd0b1..49282ae 100644 --- a/generate/templates/extensions/api_configuration.mustache +++ b/generate/templates/extensions/api_configuration.mustache @@ -1,7 +1,8 @@ import re import logging from typing import Optional, Union, Tuple, Any, Callable -from {{packageName}}.configuration import Configuration +from {{packageName}}.configuration import Configuration, Timeouts +from {{packageName}}.extensions.configuration_options import ConfigurationOptions from {{packageName}}.extensions.refreshing_token import RefreshingToken from {{packageName}}.extensions.socket_keep_alive import keep_alive_socket_options from {{packageName}}.extensions.proxy_config import ProxyConfig @@ -22,6 +23,10 @@ class ApiConfiguration: certificate_filename=None, proxy_config:Optional[ProxyConfig]=None, access_token=None, + total_timeout_ms=None, + connect_timeout_ms=None, + read_timeout_ms=None, + rate_limit_retries=None, ): """ The configuration required to access LUSID, read more at https://support.finbourne.com/getting-started-with-apis-sdks @@ -46,6 +51,10 @@ class ApiConfiguration: self.__certificate_filename = certificate_filename self.__proxy_config = proxy_config self.__access_token = access_token + self.__total_timeout_ms = total_timeout_ms + self.__connect_timeout_ms = connect_timeout_ms + self.__read_timeout_ms = read_timeout_ms + self.__rate_limit_retries = rate_limit_retries @property def token_url(self): @@ -181,6 +190,35 @@ class ApiConfiguration: ) raise + @property + def total_timeout_ms(self): + return self.__total_timeout_ms + + @total_timeout_ms.setter + def total_timeout_ms(self, value): + self.__total_timeout_ms = value + + @property + def connect_timeout_ms(self): + return self.__connect_timeout_ms + + @connect_timeout_ms.setter + def connect_timeout_ms(self, value): + self.__connect_timeout_ms = value + + @property + def read_timeout_ms(self): + return self.__read_timeout_ms + + @read_timeout_ms.setter + def read_timeout_ms(self, value): + self.__read_timeout_ms = value + + @property + def rate_limit_retries(self): + return self.__rate_limit_retries + + def build_api_client_config( self, tcp_keep_alive: bool = True, @@ -188,6 +226,7 @@ class ApiConfiguration: Union[Tuple[Any, Any, Any], Tuple[Any, Any, None, int]] ] = keep_alive_socket_options(), id_provider_response_handler: Optional[Callable[[Response], None]] = None, + opts: ConfigurationOptions = None, ) -> Configuration: """Builds lusid.Configuration for initialising an api client. @@ -219,6 +258,16 @@ class ApiConfiguration: access_token=access_token, host=self.api_url, ssl_ca_cert=self.certificate_filename, + timeouts=Timeouts( + total_timeout_ms=opts.total_timeout_ms if opts and opts.total_timeout_ms != None else + self.total_timeout_ms if self.total_timeout_ms != None else Configuration.DEFAULT_TOTAL_TIMEOUT_MS, + connect_timeout_ms=opts.connect_timeout_ms if opts and opts.connect_timeout_ms != None else + self.connect_timeout_ms if self.connect_timeout_ms != None else Configuration.DEFAULT_CONNECT_TIMEOUT_MS, + read_timeout_ms=opts.read_timeout_ms if opts and opts.read_timeout_ms != None else + self.read_timeout_ms if self.read_timeout_ms != None else Configuration.DEFAULT_READ_TIMEOUT_MS + ), + rate_limit_retries=opts.rate_limit_retries if opts != None and opts.rate_limit_retries else + self.rate_limit_retries if self.rate_limit_retries != None else Configuration.DEFAULT_RATE_LIMIT_RETRIES ) if tcp_keep_alive: config.socket_options = socket_options or keep_alive_socket_options() diff --git a/generate/templates/extensions/configuration_loaders.mustache b/generate/templates/extensions/configuration_loaders.mustache index 6560d67..bff3b66 100644 --- a/generate/templates/extensions/configuration_loaders.mustache +++ b/generate/templates/extensions/configuration_loaders.mustache @@ -5,6 +5,7 @@ import logging from {{packageName}}.extensions.proxy_config import ProxyConfig from {{packageName}}.extensions.api_configuration import ApiConfiguration from {{packageName}}.extensions.file_access_token import FileAccessToken +from {{packageName}}.extensions.configuration_options import ConfigurationOptions logger = logging.getLogger(__name__) @@ -21,7 +22,11 @@ ENVIRONMENT_CONFIG_KEYS = { "proxy_address": "FBN_PROXY_ADDRESS", "proxy_username": "FBN_PROXY_USERNAME", "proxy_password": "FBN_PROXY_PASSWORD", - "access_token": "FBN_ACCESS_TOKEN" + "access_token": "FBN_ACCESS_TOKEN", + "total_timeout_ms": "FBN_TOTAL_TIMEOUT_MS", + "connect_timeout_ms": "FBN_CONNECT_TIMEOUT_MS", + "read_timeout_ms": "FBN_READ_TIMEOUT_MS", + "rate_limit_retries": "FBN_RATE_LIMIT_RETRIES", } SECRETS_FILE_CONFIG_KEYS = { @@ -37,7 +42,11 @@ SECRETS_FILE_CONFIG_KEYS = { "proxy_address": "address", "proxy_username": "username", "proxy_password": "password", - "access_token": "accessToken" + "access_token": "accessToken", + "total_timeout_ms": "totalTimeoutMs", + "connect_timeout_ms": "connectTimeoutMs", + "read_timeout_ms": "readTimeoutMs", + "rate_limit_retries": "rateLimitRetries", } @@ -47,7 +56,7 @@ class ConfigurationLoader(Protocol): environment variables with preference given to the secrets file. """ - def load_config(self) -> Dict[str, str]: + def load_config(self) -> Dict[str, object]: pass @@ -67,12 +76,12 @@ class SecretsFileConfigurationLoader: """ self._api_secrets_file = api_secrets_file or "" - def load_config(self) -> Dict[str, str]: + def load_config(self) -> Dict[str, object]: """reads config from the provided secrets file Returns ------- - Dict[str, str] + Dict[str, object] dictionary that can be loaded into an ApiConfiguration object """ # The secrets file is a nested dictionary, set the names of the top level keys @@ -117,12 +126,12 @@ class SecretsFileConfigurationLoader: class EnvironmentVariablesConfigurationLoader: """ConfigurationLoader which reads config from environment variables """ - def load_config(self) -> Dict[str, str]: + def load_config(self) -> Dict[str, object]: """reads config from environment variables Returns ------- - Dict[str, str] + Dict[str, object] dictionary that can be loaded into an ApiConfiguration object """ logger.debug("loading config from environment variables") @@ -134,6 +143,13 @@ class EnvironmentVariablesConfigurationLoader: } if not populated_api_config_values["api_url"]: populated_api_config_values["api_url"] = populated_api_config_values["previous_api_url"] + # ensure that these values are ints + for key in ["total_timeout_ms", "connect_timeout_ms", "read_timeout_ms", "rate_limit_retries"]: + if populated_api_config_values[key]: + try: + populated_api_config_values[key] = int(populated_api_config_values[key]) + except ValueError as e: + raise ValueError(f"invalid value for '{key}' - value must be an integer if set") del(populated_api_config_values["previous_api_url"]) populated_proxy_values = { key: os.environ.get(value) @@ -162,7 +178,11 @@ class ArgsConfigurationLoader: proxy_address:Optional[str]=None, proxy_username:Optional[str]=None, proxy_password:Optional[str]=None, - access_token:Optional[str]=None + access_token:Optional[str]=None, + total_timeout_ms:Optional[int]=None, + connect_timeout_ms:Optional[int]=None, + read_timeout_ms:Optional[int]=None, + rate_limit_retries:Optional[int]=None, ): """kwargs passed to this constructor used to build ApiConfiguration """ @@ -178,13 +198,17 @@ class ArgsConfigurationLoader: self.__proxy_username = proxy_username self.__proxy_password = proxy_password self.__access_token = access_token + self.__total_timeout_ms = total_timeout_ms + self.__connect_timeout_ms = connect_timeout_ms + self.__read_timeout_ms = read_timeout_ms + self.__rate_limit_retries = rate_limit_retries - def load_config(self) -> Dict[str, str]: + def load_config(self) -> Dict[str, object]: """load configuration from kwargs passed to constructor Returns ------- - Dict[str, str] + Dict[str, object] dictionary that can be loaded into an ApiConfiguration object """ logger.debug("loading config from arguments passed to ArgsConfigurationLoader") @@ -200,7 +224,11 @@ class ArgsConfigurationLoader: "proxy_address" : self.__proxy_address, "proxy_username" : self.__proxy_username, "proxy_password" : self.__proxy_password, - "access_token" : self.__access_token + "access_token" : self.__access_token, + "total_timeout_ms" : self.__total_timeout_ms, + "connect_timeout_ms" : self.__connect_timeout_ms, + "read_timeout_ms" : self.__read_timeout_ms, + "rate_limit_retries" : self.__rate_limit_retries, } @@ -243,6 +271,8 @@ def get_api_configuration(config_loaders: Iterable[ConfigurationLoader]) -> ApiC ---------- config_loaders : Iterable[ConfigurationLoader] Objects that can be used to fetch config with a load_config function returning a dict. + opts: ConfigurationOptions + Optional configuration Returns ------- diff --git a/generate/templates/extensions/configuration_options.mustache b/generate/templates/extensions/configuration_options.mustache new file mode 100644 index 0000000..f9e416c --- /dev/null +++ b/generate/templates/extensions/configuration_options.mustache @@ -0,0 +1,67 @@ +from typing import Optional + +class ConfigurationOptions: + + def __init__( + self, + total_timeout_ms: Optional[int] = None, + connect_timeout_ms: Optional[int] = None, + read_timeout_ms: Optional[int] = None, + rate_limit_retries: Optional[int] = None + ): + self.total_timeout_ms = total_timeout_ms + self.connect_timeout_ms = connect_timeout_ms + self.read_timeout_ms = read_timeout_ms + self.rate_limit_retries = rate_limit_retries + + @property + def total_timeout_ms(self): + return self.__total_timeout_ms + + @total_timeout_ms.setter + def total_timeout_ms(self, value): + if value: + if not isinstance(value, int): + raise TypeError(f"total_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise TypeError(f"total_timeout_ms must be an integer greater than or equal to zero") + self.__total_timeout_ms = value + + @property + def connect_timeout_ms(self): + return self.__connect_timeout_ms + + @connect_timeout_ms.setter + def connect_timeout_ms(self, value): + if value: + if not isinstance(value, int): + raise TypeError(f"connect_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise TypeError(f"connect_timeout_ms must be an integer greater than or equal to zero") + self.__connect_timeout_ms = value + + @property + def read_timeout_ms(self): + return self.__read_timeout_ms + + @read_timeout_ms.setter + def read_timeout_ms(self, value): + if value: + if not isinstance(value, int): + raise TypeError(f"read_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise TypeError(f"read_timeout_ms must be an integer greater than or equal to zero") + self.__read_timeout_ms = value + + @property + def rate_limit_retries(self): + return self.__rate_limit_retries + + @rate_limit_retries.setter + def rate_limit_retries(self, value): + if value: + if not isinstance(value, int): + raise TypeError(f"rate_limit_retries must be type int but type '{type(value)}' used") + if value < 0: + raise TypeError(f"rate_limit_retries must be an integer greater than or equal to zero") + self.__rate_limit_retries = value \ No newline at end of file diff --git a/generate/templates/extensions/rest.mustache b/generate/templates/extensions/rest.mustache index de70e55..f91a4ff 100644 --- a/generate/templates/extensions/rest.mustache +++ b/generate/templates/extensions/rest.mustache @@ -103,9 +103,15 @@ class RESTClientObject(object): **addition_pool_args ) + self.timeout = self.get_timeout( + total=configuration.timeouts.total_timeout_ms / 1000.0, + connect=configuration.timeouts.connect_timeout_ms / 1000.0, + read=configuration.timeouts.read_timeout_ms / 1000.0, + ) + def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): """Perform requests. :param method: http request method @@ -119,10 +125,9 @@ class RESTClientObject(object): :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', @@ -139,14 +144,45 @@ class RESTClientObject(object): # so reset query_params to empty dict query_params = {} + # _request_timeout param cannot be removed for backwards compatability + # values from opts override values from _request_timeout + # try to get values from opts first, then _request_timeout, then self.timeout, else set to None + # timeout = _request_timeout or self.timeout timeout = None - if _request_timeout: - if isinstance(_request_timeout, (int,float)): # noqa: E501,F821 - timeout = urllib3.Timeout(total=_request_timeout) - elif (isinstance(_request_timeout, tuple) and - len(_request_timeout) == 2): - timeout = urllib3.Timeout( - connect=_request_timeout[0], read=_request_timeout[1]) + opts_total_timeout = opts.total_timeout_ms / 1000.0 if opts and opts.total_timeout_ms != None else None + opts_connect_timeout = opts.connect_timeout_ms / 1000.0 if opts and opts.connect_timeout_ms != None else None + opts_read_timeout = opts.read_timeout_ms / 1000.0 if opts and opts.read_timeout_ms != None else None + if not _request_timeout: + timeout = self.get_timeout( + total=opts_total_timeout if opts_total_timeout != None + else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None + else self.timeout._connect, + read=opts_read_timeout if opts_read_timeout != None + else self.timeout._read) + elif isinstance(_request_timeout, urllib3.Timeout): + timeout = self.get_timeout( + total=opts_total_timeout if opts_total_timeout != None + else _request_timeout.total if _request_timeout.total != None + else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None + else _request_timeout._connect if _request_timeout._connect != None + else self.timeout._connect, + read=opts_read_timeout if opts_read_timeout != None + else _request_timeout._read if _request_timeout._read != None + else self.timeout._read) + elif isinstance(_request_timeout, (int, float)): + timeout = self.get_timeout( + total=opts_total_timeout if opts_total_timeout != None else _request_timeout, + connect=opts_connect_timeout if opts_connect_timeout != None else self.timeout._connect, + read=opts_read_timeout if opts_read_timeout != None else self.timeout._read) + elif (isinstance(_request_timeout, tuple) and len(_request_timeout) == 2): + timeout = self.get_timeout( + total=opts_total_timeout if opts_total_timeout != None else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None else _request_timeout[0], + read=opts_read_timeout if opts_read_timeout != None else _request_timeout[1]) + else: + raise f"unexpected type '{type(_request_timeout)}' for _request_timeout" try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` @@ -235,66 +271,82 @@ class RESTClientObject(object): return r def get_request(self, url, headers=None, query_params=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params) + query_params=query_params, + opts=opts) def head_request(self, url, headers=None, query_params=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return self.request("HEAD", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params) + query_params=query_params, + opts=opts) def options_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, opts=None): return self.request("OPTIONS", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def delete_request(self, url, headers=None, query_params=None, body=None, - _preload_content=True, _request_timeout=None): + _preload_content=True, _request_timeout=None, opts=None): return self.request("DELETE", url, headers=headers, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def post_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, opts=None): return self.request("POST", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def put_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, opts=None): return self.request("PUT", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def patch_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, opts=None): return self.request("PATCH", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) + + def get_timeout(self, total: int, connect: int, read: int): + # zero is used in the sdk config to explicitly set to infinity (and None is used to indicate the value has not been set) + # but zero is not an allowed value for urllib3.Timeout so change any zeros to ones + return urllib3.Timeout( + total=total if total != 0 else None, + connect=connect if connect != 0 else None, + read=read if read != 0 else None, + ) diff --git a/generate/templates/extensions/retry.mustache b/generate/templates/extensions/retry.mustache index ba41a5f..d239d9b 100644 --- a/generate/templates/extensions/retry.mustache +++ b/generate/templates/extensions/retry.mustache @@ -1,19 +1,29 @@ import time -from typing import overload +from typing import Optional -from {{packageName}} import ApiException import asyncio +from {{packageName}}.configuration import Configuration +from {{packageName}}.exceptions import ApiException + class RetryingRestWrapper: """Wrapper for HTTP requests Which retries on failure And waits the amount of time specified in the Retry After header. """ - def __init__(self, rest_object, retries: int = 3): + def __init__( + self, + rest_object, + retries: int = Configuration.DEFAULT_RETRIES, + rate_limit_retries: int = Configuration.DEFAULT_RATE_LIMIT_RETRIES + ): if not isinstance(retries, int): - raise ValueError(f"retries should be an int, found {type(self.retries)}") + raise ValueError(f"retries should be an int, found {type(retries)}") + if not isinstance(rate_limit_retries, int): + raise ValueError(f"retries should be an int, found {type(rate_limit_retries)}") self.retries: int = retries + self.rate_limit_retries: int = rate_limit_retries self.rest_object = rest_object def request( @@ -26,9 +36,12 @@ class RetryingRestWrapper: post_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): - tries = 0 - while tries < self.retries + 1: + retries_count = 0 + rate_limit_retries_count = 0 + + while True: try: return self.rest_object.request( method, @@ -39,17 +52,26 @@ class RetryingRestWrapper: post_params, _preload_content, _request_timeout, + opts, ) except ApiException as ex: - - retry_after = ex.headers.get("Retry-After") - # have done max number of retries - if tries >= self.retries: - raise + if ex.status == 429 and ((opts != None and opts.rate_limit_retries != None) or self.rate_limit_retries != None): + # check for limit of rate limit retries + limit = opts.rate_limit_retries if (opts and opts.rate_limit_retries) else self.rate_limit_retries + if rate_limit_retries_count >= limit: + raise + rate_limit_retries_count += 1 + else: + # check for limit of all other retries + if retries_count >= self.retries: + raise + retries_count += 1 + + retry_after = ex.headers.get("Retry-After") # try after delay - elif retry_after is not None: + if retry_after is not None: if not isinstance(retry_after, float): try: retry_after = float(retry_after) @@ -61,7 +83,6 @@ class RetryingRestWrapper: # no retry header else: raise - tries += 1 def get_request( self, @@ -70,12 +91,14 @@ class RetryingRestWrapper: query_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params) + query_params=query_params, + opts=opts) def head_request( self, @@ -84,12 +107,14 @@ class RetryingRestWrapper: query_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("HEAD", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params) + query_params=query_params, + opts=opts) def options_request( self, @@ -100,6 +125,7 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("OPTIONS", url, headers=headers, @@ -107,7 +133,8 @@ class RetryingRestWrapper: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def delete_request( self, @@ -117,13 +144,15 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("DELETE", url, headers=headers, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def post_request( self, @@ -134,6 +163,7 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("POST", url, headers=headers, @@ -141,7 +171,8 @@ class RetryingRestWrapper: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def put_request( self, @@ -152,6 +183,7 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("PUT", url, headers=headers, @@ -159,7 +191,8 @@ class RetryingRestWrapper: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def patch_request( self, @@ -170,6 +203,7 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("PATCH", url, headers=headers, @@ -177,7 +211,8 @@ class RetryingRestWrapper: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) class RetryingRestWrapperAsync: @@ -185,10 +220,11 @@ class RetryingRestWrapperAsync: Which retries on failure And waits the amount of time specified in the Retry After header. """ - def __init__(self, rest_object, retries: int = 3): + def __init__(self, rest_object, retries: int = 3, rate_limit_retries: Optional[int] = Configuration.DEFAULT_RATE_LIMIT_RETRIES): if not isinstance(retries, int): - raise ValueError(f"retries should be an int, found {type(self.retries)}") + raise ValueError(f"retries should be an int, found {type(retries)}") self.retries: int = retries + self.rate_limit_retries: Optional[int] = rate_limit_retries self.rest_object = rest_object async def close(self): @@ -204,9 +240,12 @@ class RetryingRestWrapperAsync: post_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): - tries = 0 - while tries < self.retries + 1: + retries_count = 0 + rate_limit_retries_count = 0 + + while True: try: return await self.rest_object.request( method, @@ -217,16 +256,26 @@ class RetryingRestWrapperAsync: post_params, _preload_content, _request_timeout, + opts ) except ApiException as ex: - retry_after = ex.headers.get("Retry-After") - # have done max number of retries - if tries >= self.retries: - raise + if ex.status == 429 and ((opts and opts.rate_limit_retries) or self.rate_limit_retries != None): + # check for limit of rate limit retries + limit = opts.rate_limit_retries if (opts and opts.rate_limit_retries) else self.rate_limit_retries + if rate_limit_retries_count >= limit: + raise + rate_limit_retries_count += 1 + else: + # check for limit of all other retries + if retries_count >= self.retries: + raise + retries_count += 1 + + retry_after = ex.headers.get("Retry-After") # try after delay - elif retry_after is not None: + if retry_after is not None: if not isinstance(retry_after, float): try: retry_after = float(retry_after) @@ -238,7 +287,6 @@ class RetryingRestWrapperAsync: # no retry header else: raise - tries += 1 async def get_request( self, @@ -247,12 +295,14 @@ class RetryingRestWrapperAsync: query_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params)) + query_params=query_params, + opts=opts)) async def head_request( self, @@ -261,12 +311,14 @@ class RetryingRestWrapperAsync: query_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("HEAD", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params)) + query_params=query_params, + opts=opts)) async def options_request( self, @@ -277,6 +329,7 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("OPTIONS", url, headers=headers, @@ -284,7 +337,8 @@ class RetryingRestWrapperAsync: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def delete_request( self, @@ -294,13 +348,15 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("DELETE", url, headers=headers, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def post_request( self, @@ -311,6 +367,7 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("POST", url, headers=headers, @@ -318,7 +375,8 @@ class RetryingRestWrapperAsync: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def put_request( self, @@ -329,6 +387,7 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("PUT", url, headers=headers, @@ -336,7 +395,8 @@ class RetryingRestWrapperAsync: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def patch_request( self, @@ -347,6 +407,7 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("PATCH", url, headers=headers, @@ -354,4 +415,5 @@ class RetryingRestWrapperAsync: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) diff --git a/justfile b/justfile index e6a28ad..fde9d35 100644 --- a/justfile +++ b/justfile @@ -22,6 +22,7 @@ export FBN_CLIENT_ID := `echo ${FBN_CLIENT_ID:-client-id}` export FBN_CLIENT_SECRET := `echo ${FBN_CLIENT_SECRET:-client-secret}` export TEST_API_MODULE := `echo ${TEST_API_MODULE:-api.application_metadata_api}` export TEST_API := `echo ${TEST_API:-ApplicationMetadataApi}` +export TEST_METHOD := `echo ${TEST_METHOD:-'list_access_controlled_resources('}` export GENERATE_API_TESTS := `echo ${GENERATE_API_TESTS:-false}` swagger_path := "./swagger.json" @@ -72,10 +73,11 @@ add-tests: # rename to match values for the sdk being tested find {{justfile_directory()}}/generate/.output/sdk/test -type f -exec sed -i -e "s/TO_BE_REPLACED/${PACKAGE_NAME}/g" {} \; - # note these values won't work for the horizon sdk - # (it doesn't have this api) + # note the default values at the top of this justfile won't work for the horizon or luminesce sdk + # (they don't have this api/method) find {{justfile_directory()}}/generate/.output/sdk/test -type f -exec sed -i -e "s/TEST_API_MODULE/${TEST_API_MODULE}/g" {} \; find {{justfile_directory()}}/generate/.output/sdk/test -type f -exec sed -i -e "s/TEST_API/${TEST_API}/g" {} \; + find {{justfile_directory()}}/generate/.output/sdk/test -type f -exec sed -i -e "s/TEST_METHOD/${TEST_METHOD}/g" {} \; link-tests-cicd TARGET_DIR: mkdir -p {{TARGET_DIR}}/sdk/test/ @@ -87,6 +89,7 @@ link-tests-cicd TARGET_DIR: find {{justfile_directory()}}/test_sdk -type f -exec sed -i -e "s/TO_BE_REPLACED/${PLACEHOLDER_VALUE_FOR_TESTS}/g" {} \; find {{justfile_directory()}}/test_sdk -type f -exec sed -i -e "s/TEST_API_MODULE/${TEST_API_MODULE}/g" {} \; find {{justfile_directory()}}/test_sdk -type f -exec sed -i -e "s/TEST_API/${TEST_API}/g" {} \; + find {{justfile_directory()}}/test_sdk -type f -exec sed -i -e "s/TEST_METHOD/${TEST_METHOD}/g" {} \; setup-test-local: @just generate-local diff --git a/test_sdk/unit/test_api_client_factory.py b/test_sdk/unit/test_api_client_factory.py index 445cd80..4b16e8b 100644 --- a/test_sdk/unit/test_api_client_factory.py +++ b/test_sdk/unit/test_api_client_factory.py @@ -87,6 +87,7 @@ def test_build_returns_api_instance_with_sync_client(self): tcp_keep_alive=True, socket_options=keep_alive_socket_options(), id_provider_response_handler=None, + opts=None, ) args, kwargs = set_additional_api_client_headers_mock.call_args assert isinstance(args[0], SyncApiClient) @@ -114,6 +115,7 @@ def test_build_with_socket_options_and_tcp_keep_alive_True_sets_socket_options_o tcp_keep_alive=True, socket_options=socket_options, id_provider_response_handler=None, + opts=None, ) assert ( { @@ -145,6 +147,7 @@ async def test_build_returns_api_instance_with_async_client(self): tcp_keep_alive=True, socket_options=keep_alive_socket_options(), id_provider_response_handler=None, + opts=None, ) args, kwargs = set_additional_api_client_headers_mock.call_args assert isinstance(args[0], AsyncApiClient) diff --git a/test_sdk/unit/test_api_configuration.py b/test_sdk/unit/test_api_configuration.py index 741b33f..425166e 100644 --- a/test_sdk/unit/test_api_configuration.py +++ b/test_sdk/unit/test_api_configuration.py @@ -1,4 +1,6 @@ from unittest.mock import patch +from TO_BE_REPLACED.configuration import Configuration +from TO_BE_REPLACED.extensions.configuration_options import ConfigurationOptions from TO_BE_REPLACED.extensions.proxy_config import ProxyConfig from TO_BE_REPLACED.extensions.api_configuration import ApiConfiguration import pytest @@ -103,3 +105,49 @@ def test_build_api_client_config_with_api_url_sets_api_client_host( api_client = config.build_api_client_config() assert "sample_api_url" == api_client.host + +def test_build_api_client_config_sets_defaults_if_unset( + patch_get_access_token, +): + config = ApiConfiguration( + password="password", + username="me", + client_id="test_id", + client_secret="shhhh", + token_url="fake_url", + api_url="sample_api_url", + ) + + api_client = config.build_api_client_config() + assert api_client.timeouts.total_timeout_ms == Configuration.DEFAULT_TOTAL_TIMEOUT_MS + assert api_client.timeouts.connect_timeout_ms == Configuration.DEFAULT_CONNECT_TIMEOUT_MS + assert api_client.timeouts.read_timeout_ms == Configuration.DEFAULT_READ_TIMEOUT_MS + assert api_client.rate_limit_retries == Configuration.DEFAULT_RATE_LIMIT_RETRIES + +def test_build_api_client_uses_opts_overrides( + patch_get_access_token, +): + config = ApiConfiguration( + password="password", + username="me", + client_id="test_id", + client_secret="shhhh", + token_url="fake_url", + api_url="sample_api_url", + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000, + rate_limit_retries=5, + ) + + opts = ConfigurationOptions( + total_timeout_ms=6000, + connect_timeout_ms=5000, + read_timeout_ms=4000, + rate_limit_retries=10 + ) + api_client = config.build_api_client_config(opts=opts) + assert api_client.timeouts.total_timeout_ms == 6000 + assert api_client.timeouts.connect_timeout_ms == 5000 + assert api_client.timeouts.read_timeout_ms == 4000 + assert api_client.rate_limit_retries == 10 diff --git a/test_sdk/unit/test_config_overrides.py b/test_sdk/unit/test_config_overrides.py new file mode 100644 index 0000000..07d2ef3 --- /dev/null +++ b/test_sdk/unit/test_config_overrides.py @@ -0,0 +1,850 @@ +from typing import Optional +import aiohttp +import pytest +import urllib3 +import TO_BE_REPLACED +from TO_BE_REPLACED.api import TEST_API +from TO_BE_REPLACED.api_client import ApiClient +from TO_BE_REPLACED.configuration import Configuration, Timeouts +from TO_BE_REPLACED.exceptions import ApiException +from TO_BE_REPLACED.extensions.api_client import SyncApiClient +from TO_BE_REPLACED.extensions.api_client_factory import ApiClientFactory, SyncApiClientFactory +from TO_BE_REPLACED.extensions.configuration_loaders import ArgsConfigurationLoader +from TO_BE_REPLACED.extensions.configuration_options import ConfigurationOptions + +class MockPoolManager: + def __init__(self, return_status_code: int = 200): + self.timeouts = [] + self.callCount = 0 + self.return_status_code = return_status_code + + def request(self, method, url, **kwargs): + self.callCount += 1 + if "timeout" in kwargs: + self.timeouts.append(kwargs["timeout"]) + return MockResponse(self.return_status_code) + +class MockResponse: + def __init__(self, status_code: int): + self.status = status_code + self.reason = "test" + self.data = "test".encode('utf-8') + self.headers = {"Retry-After":0} + + def getheader(self, header: str): + if header == "Retry-After": + return 0 + return None + + def getheaders(self): + return self.headers + + async def read(self): + return self.data + + def __await__(self): + async def closure(): + return self + return closure().__await__() + + +# contains tests that are common to both async and sync api clients, whether created manually or with an api client factory +class BaseTests: + + def _override_deserialize(_, __): + return "" + + async def _run_timeout_test( + self, + create_api_client_with_factory: bool, + async_req: Optional[bool], + is_async: bool, + configuration_timeouts: Timeouts, + factory_opts: ConfigurationOptions, + request_timeout, + request_opts: ConfigurationOptions, + expected_total_timeout: Optional[float], + expected_connect_timeout: Optional[float], + expected_read_timeout: Optional[float] + ): + mock_pool_manager = MockPoolManager() + if create_api_client_with_factory: + api = self._create_api_instance_with_factory( + is_async=is_async, + rate_limit_retries_config=None, + configuration_timeouts=configuration_timeouts, + opts=factory_opts, + mock_pool_manager=mock_pool_manager) + else: + api = self._create_api_instance_without_factory(is_async, configuration_timeouts, mock_pool_manager) + + # to avoid having to return data that can be deserialised into real types + # (which will vary from api to api and can change over time) + # override the deserialize method + api.api_client.deserialize = BaseTests._override_deserialize + + result = api.TEST_METHOD + _request_timeout=request_timeout, + async_req=async_req, + opts=request_opts) + + if async_req: + result = result.get(timeout=60) + + if is_async: + await result + + assert mock_pool_manager.callCount == 1 + assert len(mock_pool_manager.timeouts) == 1 + timeout = mock_pool_manager.timeouts[0] + + if is_async: + # the timeout is of type aiohttp.ClientTimeout + assert timeout.total == expected_total_timeout + assert timeout.connect == expected_connect_timeout + assert timeout.sock_read == expected_read_timeout + else: + # the timeout is of type urllib3.Timeout + assert timeout.total == expected_total_timeout + assert timeout._connect == expected_connect_timeout + assert timeout._read == expected_read_timeout + + def _create_api_instance_without_factory( + self, + is_async: bool, + configuration_timeouts: Timeouts, + mock_pool_manager: MockPoolManager, + ): + configuration = Configuration(access_token="token", host="http://localhost", timeouts=configuration_timeouts) + api_client_type = ApiClient if is_async else SyncApiClient + api_client = api_client_type(configuration=configuration) + api_client.rest_client.pool_manager = mock_pool_manager + return TEST_API(api_client) + + def _create_api_instance_with_factory( + self, + is_async: bool, + rate_limit_retries_config: Optional[int], + configuration_timeouts: Timeouts, + opts: ConfigurationOptions, + mock_pool_manager: MockPoolManager, + ): + configuration_loader = ArgsConfigurationLoader( + access_token="token", + api_url="http://localhost", + total_timeout_ms=configuration_timeouts.total_timeout_ms, + connect_timeout_ms=configuration_timeouts.connect_timeout_ms, + read_timeout_ms=configuration_timeouts.read_timeout_ms, + rate_limit_retries=rate_limit_retries_config + ) + api_client_factory_type = ApiClientFactory if is_async else SyncApiClientFactory + api_client_factory = api_client_factory_type( + config_loaders=[configuration_loader], + opts=opts + ) + api = api_client_factory.build(TEST_API) + api.api_client.rest_client.rest_object.pool_manager = mock_pool_manager + return api + + async def test_timeout_when_set_on_request_timeout_via_int(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(3000, 2000, 1000), + factory_opts=None, + request_timeout=10, + request_opts=None, + expected_total_timeout=10, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + async def test_timeout_when_set_on_request_timeout_via_float(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(3000, 2000, 1000), + factory_opts=None, + request_timeout=10.1, + request_opts=None, + expected_total_timeout=10.1, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + async def test_timeout_when_set_on_request_opts_only(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=1100, + read_timeout_ms=2100 + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.1, + expected_read_timeout=2.1 + ) + + # some timeouts specified in opts + # no request_timeout + # all timeouts specified in configuration + # expect values from configuration to be used where not specified in opts + async def test_timeout_when_partially_set_on_request_opts(self, create_api_client_with_factory, async_req, is_async): + + # specify total and connect + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=1100, + read_timeout_ms=None + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.1, + expected_read_timeout=3.4 + ) + + # specify connect and read + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=1100, + read_timeout_ms=2100 + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.1, + expected_read_timeout=2.1 + ) + + # specify total and read + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=None, + read_timeout_ms=2100 + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.2, + expected_read_timeout=2.1 + ) + + # specify total only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=None, + read_timeout_ms=None + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.2, + expected_read_timeout=3.4 + ) + + # specify connect only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=1100, + read_timeout_ms=None + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.1, + expected_read_timeout=3.4 + ) + + # specify read only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=None, + read_timeout_ms=2100 + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.2, + expected_read_timeout=2.1 + ) + + async def test_timeout_when_set_on_config_only(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000), + factory_opts=None, + request_timeout=None, + request_opts=None, + expected_total_timeout=3, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + async def test_timeout_when_set_on_request_timeout_only(self, create_api_client_with_factory, async_req, is_async): + request_timeout = aiohttp.ClientTimeout(total=10.2, connect=1.2, sock_read=3.4) if is_async else urllib3.Timeout(total=10.2, connect=1.2, read=3.4) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=None, + expected_total_timeout=10.2, + expected_connect_timeout=1.2, + expected_read_timeout=3.4 + ) + + # all timeouts specified in opts + # all timeouts specified in request_timeout + # expect values from opts to be used + async def test_request_opts_override_request_timeout(self, create_api_client_with_factory, async_req, is_async): + request_timeout = aiohttp.ClientTimeout(total=10.2, connect=1.2, sock_read=3.4) if is_async else urllib3.Timeout(total=10.2, connect=1.2, read=3.4) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=1100, + read_timeout_ms=2100 + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.1, + expected_read_timeout=2.1 + ) + + # some timeouts specified in opts + # all timeouts specified in request_timeout + # expect values from request_timeout to be used where not specified in opts + async def test_timeout_when_set_on_request_timeout_and_partially_set_on_request_opts(self, create_api_client_with_factory, async_req, is_async): + + # specify total and connect + request_timeout = aiohttp.ClientTimeout(total=10.2, connect=1.2, sock_read=3.4) if is_async else urllib3.Timeout(total=10.2, connect=1.2, read=3.4) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=1100, + read_timeout_ms=None + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.1, + expected_read_timeout=3.4 + ) + + # specify connect and read + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=1100, + read_timeout_ms=2100 + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.1, + expected_read_timeout=2.1 + ) + + # specify total and read + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=None, + read_timeout_ms=2100 + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.2, + expected_read_timeout=2.1 + ) + + # specify total only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=None, + read_timeout_ms=None + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.2, + expected_read_timeout=3.4 + ) + + # specify connect only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=1100, + read_timeout_ms=None + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.1, + expected_read_timeout=3.4 + ) + + # specify read only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=None, + read_timeout_ms=2100 + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.2, + expected_read_timeout=2.1 + ) + + # some timeouts specified in opts + # some timeouts specified in request_timeout + # some timeouts specified in configuration + # expect precedence to be opts > request_timeout > configuration + async def test_timeout_when_partially_set_on_request_timeout_and_partially_set_on_request_opts(self, create_api_client_with_factory, async_req, is_async): + request_timeout = aiohttp.ClientTimeout(total=6, connect=5, sock_read=None) if is_async else urllib3.Timeout(total=6, connect=5, read=None) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=9000, + connect_timeout_ms=None, + read_timeout_ms=None + ), + expected_total_timeout=9, + expected_connect_timeout=5, + expected_read_timeout=1 + ) + + async def test_timeout_when_both_request_timeout_and_request_opts_all_values_set_to_none(self, create_api_client_with_factory, async_req, is_async): + request_timeout = aiohttp.ClientTimeout(total=None, connect=None, sock_read=None) if is_async else urllib3.Timeout(total=None, connect=None, read=None) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=None, + read_timeout_ms=None + ), + expected_total_timeout=3, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_timeouts_respected_in_config(self, create_api_client_with_factory, async_req, is_async): + # urllib3 used in sync does not allow the timeout to be set to zero - ensure it's converted to None + expected_timeout = 0 if is_async else None + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(0, 0, 0), + factory_opts=None, + request_timeout=None, + request_opts=None, + expected_total_timeout=expected_timeout, + expected_connect_timeout=expected_timeout, + expected_read_timeout=expected_timeout + ) + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_timeouts_respected_in_factory_opts(self, create_api_client_with_factory, async_req, is_async): + if not create_api_client_with_factory: + pytest.skip("test with factory options only valid when creating api client with factory") + + # urllib3 used in sync does not allow the timeout to be set to zero - ensure it's converted to None + expected_timeout = 0 if is_async else None + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(5000, 5000, 5000), + factory_opts=ConfigurationOptions( + total_timeout_ms=0, + connect_timeout_ms=0, + read_timeout_ms=0 + ), + request_timeout=None, + request_opts=None, + expected_total_timeout=expected_timeout, + expected_connect_timeout=expected_timeout, + expected_read_timeout=expected_timeout + ) + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_timeouts_respected_in_request_opts(self, create_api_client_with_factory, async_req, is_async): + # urllib3 used in sync does not allow the timeout to be set to zero - ensure it's converted to None + expected_timeout = 0 if is_async else None + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(5000, 5000, 5000), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=0, + connect_timeout_ms=0, + read_timeout_ms=0 + ), + expected_total_timeout=expected_timeout, + expected_connect_timeout=expected_timeout, + expected_read_timeout=expected_timeout + ) + +# contains tests that are common to async clients created either manually or through the api client factory +class BaseAsyncTests(BaseTests): + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_timeouts_respected_in_request_timeout(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(5000, 5000, 5000), + factory_opts=None, + request_timeout=aiohttp.ClientTimeout(total=0, connect=0, sock_read=0), + request_opts=None, + expected_total_timeout=0, + expected_connect_timeout=0, + expected_read_timeout=0 + ) + +# contains tests that are common to sync clients created either manually or through the api client factory +class BaseSyncTests(BaseTests): + + # this tuple is only valid with urllib3.Timeout + async def test_timeout_when_set_on_request_via_tuple(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=(1,2), + request_opts=None, + expected_total_timeout=Configuration.DEFAULT_TOTAL_TIMEOUT_MS / 1000.0, + expected_connect_timeout=1, + expected_read_timeout=2 + ) + +# contains test that are common to sync and async clients created through the api client factory +# NOTE: that the retries tests are not in BaseTests because the api client created manually does not have the retrying rest client +class BaseApiClientFromFactoryTests(BaseTests): + + async def _run_rate_limit_test( + self, + async_req: Optional[bool], + is_async: bool, + rate_limit_retries_config: int, + factory_opts: ConfigurationOptions, + request_opts: ConfigurationOptions, + return_status_code: int, + expected_times_called: int + ): + mock_pool_manager = MockPoolManager(return_status_code) + api = self._create_api_instance_with_factory( + is_async=is_async, + rate_limit_retries_config=rate_limit_retries_config, + configuration_timeouts=Timeouts.get_default(), + opts=factory_opts, + mock_pool_manager=mock_pool_manager) + + try: + result = api.TEST_METHOD + async_req=async_req, + opts=request_opts) + + if async_req: + result = result.get(timeout=60) + + if is_async: + await result + except(ApiException): + ... + + assert mock_pool_manager.callCount == expected_times_called + + async def test_timeout_when_set_on_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=ConfigurationOptions( + total_timeout_ms=5002, + connect_timeout_ms=4002, + read_timeout_ms=3002 + ), + request_timeout=None, + request_opts=None, + expected_total_timeout=5.002, + expected_connect_timeout=4.002, + expected_read_timeout=3.002 + ) + + async def test_timeout_when_partially_set_on_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=ConfigurationOptions( + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000 + ), + factory_opts=ConfigurationOptions( + total_timeout_ms=5002 + ), + request_timeout=None, + request_opts=None, + expected_total_timeout=5.002, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + async def test_default_rate_limit_retries_if_none_set(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=None, + request_opts=None, + return_status_code=429, + expected_times_called=Configuration.DEFAULT_RATE_LIMIT_RETRIES + 1 + ) + + async def test_retries_config_used_when_not_429_response(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=10, + factory_opts=None, + request_opts=None, + return_status_code=409, + expected_times_called=Configuration.DEFAULT_RETRIES + 1 + ) + + async def test_rate_limit_retries_set_on_request_when_set_in_config(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=5, + factory_opts=None, + request_opts=None, + return_status_code=429, + expected_times_called=6 + ) + + async def test_rate_limit_retries_set_on_request_when_set_in_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=ConfigurationOptions(rate_limit_retries=6), + request_opts=None, + return_status_code=429, + expected_times_called=7 + ) + + async def test_rate_limit_retries_set_on_request_when_set_in_request_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=None, + request_opts=ConfigurationOptions(rate_limit_retries=7), + return_status_code=429, + expected_times_called=8 + ) + + async def test_rate_limit_retries_set_on_request_when_factory_opts_override_config(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=5, + factory_opts=ConfigurationOptions(rate_limit_retries=6), + request_opts=None, + return_status_code=429, + expected_times_called=7 + ) + + async def test_rate_limit_retries_set_on_request_when_request_opts_override_config(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=5, + factory_opts=None, + request_opts=ConfigurationOptions(rate_limit_retries=7), + return_status_code=429, + expected_times_called=8 + ) + + async def test_rate_limit_retries_set_on_request_when_request_opts_override_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=ConfigurationOptions(rate_limit_retries=6), + request_opts=ConfigurationOptions(rate_limit_retries=7), + return_status_code=429, + expected_times_called=8 + ) + + async def test_rate_limit_retries_set_on_request_when_request_opts_override_factory_opts_override_config(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=5, + factory_opts=ConfigurationOptions(rate_limit_retries=6), + request_opts=ConfigurationOptions(rate_limit_retries=7), + return_status_code=429, + expected_times_called=8 + ) + +@pytest.mark.parametrize("create_api_client_with_factory, async_req, is_async", + [(True, True, True), + (True, False, True), + (True, None, True)]) +@pytest.mark.asyncio +class TestWithAsyncApiClientFromFactory(BaseApiClientFromFactoryTests, BaseAsyncTests): + ... + +@pytest.mark.parametrize("create_api_client_with_factory, async_req, is_async", + [(False, True, True), + (False, False, True), + (False, None, True)]) +@pytest.mark.asyncio +class TestWithAsyncApiClientManualCreation(BaseAsyncTests): + ... + +@pytest.mark.parametrize("create_api_client_with_factory, async_req, is_async", + [(True, True, False), + (True, False, False), + (True, None, False)]) +@pytest.mark.asyncio +class TestWithSyncApiClientFromFactory(BaseApiClientFromFactoryTests, BaseSyncTests): + ... + +@pytest.mark.parametrize("create_api_client_with_factory, async_req, is_async", + [(False, True, False), + (False, False, False), + (False, None, False)]) +@pytest.mark.asyncio +class TestWithSyncApiClientManualCreation(BaseSyncTests): + ... diff --git a/test_sdk/unit/test_configuration_loaders.py b/test_sdk/unit/test_configuration_loaders.py index 9d7ab0f..8fef1ef 100644 --- a/test_sdk/unit/test_configuration_loaders.py +++ b/test_sdk/unit/test_configuration_loaders.py @@ -1,4 +1,5 @@ import json +import pytest from TO_BE_REPLACED.extensions import ( SecretsFileConfigurationLoader, EnvironmentVariablesConfigurationLoader, @@ -24,6 +25,10 @@ def test_load_config_loads_api_config(self): "proxy_address": None, "proxy_username": None, "proxy_password": None, + "total_timeout_ms": 3000, + "connect_timeout_ms": 2000, + "read_timeout_ms": 1000, + "rate_limit_retries": 5, } secrets_file_contents = { @@ -37,6 +42,10 @@ def test_load_config_loads_api_config(self): "applicationName": "sample_applicationName", "clientCertificate": "sample_clientCertificate", "accessToken": "accessToken", + "totalTimeoutMs": 3000, + "connectTimeoutMs": 2000, + "readTimeoutMs": 1000, + "rateLimitRetries": 5, } } api_secrets_file = "secrets.json" @@ -62,6 +71,10 @@ def test_load_config_loads_proxy_config(self): "proxy_address": "sample_address", "proxy_username": "sample_username", "proxy_password": "sample_password", + "total_timeout_ms": None, + "connect_timeout_ms": None, + "read_timeout_ms": None, + "rate_limit_retries": None, } secrets_file_contents = { @@ -94,6 +107,10 @@ def test_load_config_loads_api_config(self): "FBN_APP_NAME": "sample_applicationName", "FBN_CLIENT_CERTIFICATE": "sample_clientCertificate", "FBN_ACCESS_TOKEN": "accessToken", + "FBN_TOTAL_TIMEOUT_MS": "3000", + "FBN_CONNECT_TIMEOUT_MS": "2000", + "FBN_READ_TIMEOUT_MS": "1000", + "FBN_RATE_LIMIT_RETRIES": "5" } expected_config = { "token_url": "sample_tokenUrl", @@ -108,6 +125,10 @@ def test_load_config_loads_api_config(self): "proxy_address": None, "proxy_username": None, "proxy_password": None, + "total_timeout_ms": 3000, + "connect_timeout_ms": 2000, + "read_timeout_ms": 1000, + "rate_limit_retries": 5, } with mock.patch.dict("os.environ", environment_variables, clear=True): config_loader = EnvironmentVariablesConfigurationLoader() @@ -133,12 +154,30 @@ def test_load_config_loads_proxy_config(self): "proxy_address": "sample_address", "proxy_username": "sample_username", "proxy_password": "sample_password", + "total_timeout_ms": None, + "connect_timeout_ms": None, + "read_timeout_ms": None, + "rate_limit_retries": None, } with mock.patch.dict("os.environ", environment_variables, clear=True): config_loader = EnvironmentVariablesConfigurationLoader() result = config_loader.load_config() assert expected_config == result + @pytest.mark.parametrize("key, config_name", + [("FBN_TOTAL_TIMEOUT_MS", "total_timeout_ms"), + ("FBN_CONNECT_TIMEOUT_MS", "connect_timeout_ms"), + ("FBN_READ_TIMEOUT_MS", "read_timeout_ms"), + ("FBN_RATE_LIMIT_RETRIES", "rate_limit_retries")]) + def test_load_config_throws_if_invalid_int_values(self, key, config_name): + environment_variables = { + key: "not-a-number" + } + with mock.patch.dict("os.environ", environment_variables, clear=True): + with pytest.raises(ValueError) as e: + config_loader = EnvironmentVariablesConfigurationLoader() + config_loader.load_config() + assert str(e.value) == f"invalid value for '{config_name}' - value must be an integer if set" class TestArgsConfigurationLoader: def test_load_config_gets_config_dict(self): @@ -155,6 +194,10 @@ def test_load_config_gets_config_dict(self): "proxy_address": "sample_address", "proxy_username": "sample_username", "proxy_password": "sample_password", + "total_timeout_ms": 3000, + "connect_timeout_ms": 2000, + "read_timeout_ms": 1000, + "rate_limit_retries": 5, } expected_config = { "token_url": "sample_tokenUrl", @@ -169,6 +212,10 @@ def test_load_config_gets_config_dict(self): "proxy_address": "sample_address", "proxy_username": "sample_username", "proxy_password": "sample_password", + "total_timeout_ms": 3000, + "connect_timeout_ms": 2000, + "read_timeout_ms": 1000, + "rate_limit_retries": 5, } config_loader = ArgsConfigurationLoader(**kwargs) result = config_loader.load_config() diff --git a/test_sdk/unit/test_retry.py b/test_sdk/unit/test_retry.py index 1d01f0c..aa3c8fd 100644 --- a/test_sdk/unit/test_retry.py +++ b/test_sdk/unit/test_retry.py @@ -15,7 +15,7 @@ def test_on_success_returns_result_and_makes_request_once(self): retry_object = RetryingRestWrapper(rest_object=rest_object_mock) result = retry_object.request("GET", "") assert result == "OK" - rest_object_mock.request.assert_called_once_with('GET', '', None, None, None, None, True, None) + rest_object_mock.request.assert_called_once_with('GET', '', None, None, None, None, True, None, None) def test_on_failure_retries_once(self): rest_object_mock = RESTClientObject(Configuration()) @@ -25,7 +25,7 @@ def test_on_failure_retries_once(self): retry_object = RetryingRestWrapper(rest_object=rest_object_mock, retries=1) result = retry_object.request("GET", "") assert result == "OK" - rest_object_mock.request.assert_called_with('GET', '', None, None, None, None, True, None) + rest_object_mock.request.assert_called_with('GET', '', None, None, None, None, True, None, None) assert rest_object_mock.request.call_count == 2 def test_on_repeated_failures_throws_ApiException(self): @@ -49,7 +49,7 @@ async def test_on_success_returns_result_and_makes_request_once(self): retry_object = RetryingRestWrapperAsync(rest_object=rest_object_mock) result = await retry_object.request("GET", "") assert result == "OK" - rest_object_mock.request.assert_called_once_with('GET', '', None, None, None, None, True, None) + rest_object_mock.request.assert_called_once_with('GET', '', None, None, None, None, True, None, None) await rest_object_mock.close() async def test_on_failure_retries_once(self): @@ -60,7 +60,7 @@ async def test_on_failure_retries_once(self): retry_object = RetryingRestWrapperAsync(rest_object=rest_object_mock, retries=1) result = await retry_object.request("GET", "") assert result == "OK" - rest_object_mock.request.assert_called_with('GET', '', None, None, None, None, True, None) + rest_object_mock.request.assert_called_with('GET', '', None, None, None, None, True, None, None) assert rest_object_mock.request.call_count == 2 await rest_object_mock.close()