diff --git a/generate/config-template.json b/generate/config-template.json index c9a15f2..8c5f648 100644 --- a/generate/config-template.json +++ b/generate/config-template.json @@ -50,6 +50,9 @@ }, "extensions/file_access_token.mustache": { "destinationFilename": "${PACKAGE_NAME}/extensions/file_access_token.py" + }, + "extensions/configuration_options.mustache": { + "destinationFilename": "${PACKAGE_NAME}/extensions/configuration_options.py" } } } \ No newline at end of file diff --git a/generate/templates/README.mustache b/generate/templates/README.mustache index 7bd6ff8..e293341 100644 --- a/generate/templates/README.mustache +++ b/generate/templates/README.mustache @@ -56,7 +56,9 @@ import {{{packageName}}} ## Getting Started -You'll need to provide some configuration to connect to the {{appName}} - see the articles about [short-lived access tokens](https://support.lusid.com/knowledgebase/article/KA-01654) and a [long-lived Personal Access Token](https://support.lusid.com/knowledgebase/article/KA-01774). This configuration can be provided using a secrets file or environment variables. +You'll need to provide some configuration to connect to the {{appName}} - see the articles about [short-lived access tokens](https://support.lusid.com/knowledgebase/article/KA-01654) and a [long-lived Personal Access Token](https://support.lusid.com/knowledgebase/article/KA-01774). This configuration can be provided using a secrets file or environment variables. + +For some configuration it is also possible to override the global configuration at the ApiClientFactory level, or at the request level. For the set of configuration which can be overridden, please see [ConfigurationOptions](sdk/{{packageName}}/extensions/configuration_options.py). For a code illustration of this configuration being overridden, please see the [example](#example). ### Environment variables @@ -78,6 +80,18 @@ FBN_ACCESS_TOKEN You can send your requests to the {{appName}} via a proxy, by setting `FBN_PROXY_ADDRESS`. If your proxy has basic auth enabled, you must also set `FBN_PROXY_USERNAME` and `FBN_PROXY_PASSWORD`. +Other optional configuration + +```bash +# sdk client timeouts in milliseconds - a value of 0 means no timeout, otherwise timeout values must be a positive integer +# please note - the chances of seeing a network issue increase with the duration of the request +# for this reason, rather than increasing the timeout, it will be more reliable to use an alternate polling style endpoint where these exist +FBN_TOTAL_TIMEOUT_MS # the default is 1800000 (30 minutes) +FBN_CONNECT_TIMEOUT_MS # the default is 0 (no timeout) +FBN_READ_TIMEOUT_MS # the default is 0 (no timeout) +FBN_RATE_LIMIT_RETRIES # the default is 2 +``` + ### Secrets file The secrets file must be in the current working directory. @@ -126,6 +140,23 @@ You can send your requests to the {{appName}} via a proxy, by adding a proxy sec } ``` +Other optional configuration + +```javascript +{ + "api": + { + // sdk client timeouts in milliseconds - a value of 0 means no timeout, otherwise timeout values must be a positive integer + // please note - the chances of seeing a network issue increase with the duration of the request + // for this reason, rather than increasing the timeout, it will be more reliable to use an alternate polling style endpoint where these exist + "totalTimeoutMs":, // the default is 1800000 (30 minutes) + "connectTimeoutMs":, // the default is 0 (no timeout) + "readTimeoutMs":, // the default is 0 (no timeout) + "rateLimitRetries": // the default is 2 + } +} +``` + {{#apiInfo}}{{#apis}}{{#-first}}{{#operations}}{{#operation}}{{#-first}}### Example {{> example}} {{/-first}}{{/operation}}{{/operations}}{{/-first}}{{/apis}}{{/apiInfo}} diff --git a/generate/templates/api.mustache b/generate/templates/api.mustache index ffd391e..3200a57 100644 --- a/generate/templates/api.mustache +++ b/generate/templates/api.mustache @@ -21,6 +21,7 @@ from {{packageName}}.exceptions import ( # noqa: F401 ApiTypeError, ApiValueError ) +from {{packageName}}.extensions.configuration_options import ConfigurationOptions {{#operations}} @@ -66,10 +67,9 @@ class {{classname}}: {{/allParams}} :param async_req: Whether to execute the request asynchronously. :type async_req: bool, optional - :param _request_timeout: timeout setting for this request. - If one number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional :return: Returns the result object. If the method is called asynchronously, returns the request thread. @@ -112,10 +112,9 @@ class {{classname}}: :param _return_http_data_only: response data instead of ApiResponse object with status code, headers, etc :type _return_http_data_only: bool, optional - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional :param _request_auth: set to override the auth_settings for an a single request; this effectively ignores the authentication in the spec for a single request. @@ -162,7 +161,8 @@ class {{classname}}: '_request_timeout', '_request_auth', '_content_type', - '_headers' + '_headers', + 'opts' ] ) @@ -304,6 +304,7 @@ class {{classname}}: _return_http_data_only=_params.get('_return_http_data_only'), # noqa: E501 _preload_content=_params.get('_preload_content', True), _request_timeout=_params.get('_request_timeout'), + opts=_params.get('opts'), {{#servers.0}} _host=_host, {{/servers.0}} diff --git a/generate/templates/api_client.mustache b/generate/templates/api_client.mustache index 5988d7e..81d8e13 100644 --- a/generate/templates/api_client.mustache +++ b/generate/templates/api_client.mustache @@ -162,7 +162,7 @@ class ApiClient: files=None, response_types_map=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None, - _request_auth=None): + _request_auth=None, opts=None): config = self.configuration @@ -230,7 +230,8 @@ class ApiClient: headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, - _request_timeout=_request_timeout) + _request_timeout=_request_timeout, + opts=opts) except ApiException as e: if e.body: e.body = e.body.decode('utf-8') @@ -390,7 +391,8 @@ class ApiClient: response_types_map=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, - _request_timeout=None, _host=None, _request_auth=None): + _request_timeout=None, _host=None, _request_auth=None, + opts=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. @@ -417,10 +419,9 @@ class ApiClient: Default is True. :param collection_formats: dict of collection formats for path, query, header, and post parameters. - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional :param _request_auth: set to override the auth_settings for an a single request; this effectively ignores the authentication in the spec for a single request. @@ -439,7 +440,7 @@ class ApiClient: response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, - _request_auth) + _request_auth, opts) return self.pool.apply_async(self.__call_api, (resource_path, method, path_params, @@ -452,30 +453,34 @@ class ApiClient: collection_formats, _preload_content, _request_timeout, - _host, _request_auth)) + _host, _request_auth, + opts)) def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): """Makes the HTTP request using RESTClient.""" if method == "GET": return self.rest_client.get_request(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - headers=headers) + headers=headers, + opts=opts) elif method == "HEAD": return self.rest_client.head_request(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - headers=headers) + headers=headers, + opts=opts) elif method == "OPTIONS": return self.rest_client.options_request(url, query_params=query_params, headers=headers, _preload_content=_preload_content, - _request_timeout=_request_timeout) + _request_timeout=_request_timeout, + opts=opts) elif method == "POST": return self.rest_client.post_request(url, query_params=query_params, @@ -483,7 +488,8 @@ class ApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "PUT": return self.rest_client.put_request(url, query_params=query_params, @@ -491,7 +497,8 @@ class ApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "PATCH": return self.rest_client.patch_request(url, query_params=query_params, @@ -499,14 +506,16 @@ class ApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "DELETE": return self.rest_client.delete_request(url, query_params=query_params, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) else: raise ApiValueError( "http method must be `GET`, `HEAD`, `OPTIONS`," diff --git a/generate/templates/asyncio/rest.mustache b/generate/templates/asyncio/rest.mustache index 314b404..100024e 100644 --- a/generate/templates/asyncio/rest.mustache +++ b/generate/templates/asyncio/rest.mustache @@ -58,6 +58,11 @@ class RESTClientObject: self.proxy = configuration.proxy self.proxy_headers = configuration.proxy_headers + self.timeout = aiohttp.ClientTimeout( + total=configuration.timeouts.total_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.total_timeout_ms != None else None, + connect=configuration.timeouts.connect_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.connect_timeout_ms != None else None, + sock_read=configuration.timeouts.read_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.read_timeout_ms != None else None, + ) # https pool manager self.pool_manager = aiohttp.ClientSession( @@ -70,7 +75,7 @@ class RESTClientObject: async def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): """Execute request :param method: http request method @@ -83,10 +88,9 @@ class RESTClientObject: and `multipart/form-data` :param _preload_content: this is a non-applicable field for the AiohttpClient. - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', @@ -102,7 +106,44 @@ class RESTClientObject: # url already contains the URL query string # so reset query_params to empty dict query_params = {} - timeout = _request_timeout or 5 * 60 + + # _request_timeout param cannot be removed for backwards compatability + # values from opts override values from _request_timeout + # try to get values from opts first, then _request_timeout, then self.timeout, else set to None + # timeout = _request_timeout or self.timeout + timeout = None + opts_total_timeout = opts.total_timeout_ms / 1000.0 if opts and opts.total_timeout_ms != None else None + opts_connect_timeout = opts.connect_timeout_ms / 1000.0 if opts and opts.connect_timeout_ms != None else None + opts_read_timeout = opts.read_timeout_ms / 1000.0 if opts and opts.read_timeout_ms != None else None + if not _request_timeout: + timeout = aiohttp.ClientTimeout( + total=opts_total_timeout if opts_total_timeout != None + else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None + else self.timeout.connect, + sock_read=opts_read_timeout if opts_read_timeout != None + else self.timeout.sock_read) + elif isinstance(_request_timeout, aiohttp.ClientTimeout): + timeout = aiohttp.ClientTimeout( + total=opts_total_timeout if opts_total_timeout != None + else _request_timeout.total if _request_timeout.total != None + else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None + else _request_timeout.connect if _request_timeout.connect != None + else self.timeout.connect, + sock_read=opts_read_timeout if opts_read_timeout != None + else _request_timeout.sock_read if _request_timeout.sock_read != None + else self.timeout.sock_read) + elif isinstance(_request_timeout, (int, float)): + timeout = aiohttp.ClientTimeout( + total=opts_total_timeout if opts_total_timeout != None + else _request_timeout, + connect=opts_connect_timeout if opts_connect_timeout != None + else self.timeout.connect, + sock_read=opts_read_timeout if opts_read_timeout != None + else self.timeout.sock_read) + else: + raise f"unexpected type '{type(_request_timeout)}' for _request_timeout" if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' @@ -176,69 +217,78 @@ class RESTClientObject: return r async def get_request(self, url, headers=None, query_params=None, - _preload_content=True, _request_timeout=None): + _preload_content=True, _request_timeout=None, opts=None): return (await self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params)) + query_params=query_params, + opts=opts)) async def head_request(self, url, headers=None, query_params=None, - _preload_content=True, _request_timeout=None): + _preload_content=True, _request_timeout=None, opts=None): return (await self.request("HEAD", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params)) + query_params=query_params, + opts=opts)) async def options_request(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return (await self.request("OPTIONS", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def delete_request(self, url, headers=None, query_params=None, body=None, - _preload_content=True, _request_timeout=None): + _preload_content=True, _request_timeout=None, + opts=None): return (await self.request("DELETE", url, headers=headers, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def post_request(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return (await self.request("POST", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def put_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, + opts=None): return (await self.request("PUT", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def patch_request(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return (await self.request("PATCH", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) diff --git a/generate/templates/configuration.mustache b/generate/templates/configuration.mustache index 706909b..8630267 100644 --- a/generate/templates/configuration.mustache +++ b/generate/templates/configuration.mustache @@ -19,6 +19,56 @@ JSON_SCHEMA_VALIDATION_KEYWORDS = { 'minLength', 'pattern', 'maxItems', 'minItems' } +class Timeouts: + + @property + def total_timeout_ms(self): + return self.__total_timeout_ms + + @total_timeout_ms.setter + def total_timeout_ms(self, value): + if not isinstance(value, int): + raise TypeError(f"total_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise ValueError(f"total_timeout_ms must be an integer greater than or equal to zero") + self.__total_timeout_ms = value + + @property + def connect_timeout_ms(self): + return self.__connect_timeout_ms + + @connect_timeout_ms.setter + def connect_timeout_ms(self, value): + if not isinstance(value, int): + raise TypeError(f"connect_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise ValueError(f"connect_timeout_ms must be an integer greater than or equal to zero") + self.__connect_timeout_ms = value + + @property + def read_timeout_ms(self): + return self.__read_timeout_ms + + @read_timeout_ms.setter + def read_timeout_ms(self, value): + if not isinstance(value, int): + raise TypeError(f"read_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise ValueError(f"read_timeout_ms must be an integer greater than or equal to zero") + self.__read_timeout_ms = value + + def __init__(self, total_timeout_ms: int, connect_timeout_ms: int, read_timeout_ms: int): + self.total_timeout_ms = total_timeout_ms + self.connect_timeout_ms = connect_timeout_ms + self.read_timeout_ms = read_timeout_ms + + def get_default(): + return Timeouts( + total_timeout_ms=Configuration.DEFAULT_TOTAL_TIMEOUT_MS, + connect_timeout_ms=Configuration.DEFAULT_CONNECT_TIMEOUT_MS, + read_timeout_ms=Configuration.DEFAULT_READ_TIMEOUT_MS + ) + class Configuration: """This class contains various settings of the API client. @@ -134,6 +184,12 @@ conf = {{{packageName}}}.Configuration( {{/hasHttpSignatureMethods}} {{/hasAuthMethods}} """ + + DEFAULT_TOTAL_TIMEOUT_MS: int = 1800_000 + DEFAULT_CONNECT_TIMEOUT_MS: int = 0 + DEFAULT_READ_TIMEOUT_MS: int = 0 + DEFAULT_RATE_LIMIT_RETRIES: int = 2 + DEFAULT_RETRIES: int = 3 _default = None @@ -147,7 +203,11 @@ conf = {{{packageName}}}.Configuration( server_index=None, server_variables=None, server_operation_index=None, server_operation_variables=None, ssl_ca_cert=None, - ) -> None: + timeouts=Timeouts( + total_timeout_ms=DEFAULT_TOTAL_TIMEOUT_MS, + connect_timeout_ms=DEFAULT_CONNECT_TIMEOUT_MS, + read_timeout_ms=DEFAULT_READ_TIMEOUT_MS), + rate_limit_retries=DEFAULT_RATE_LIMIT_RETRIES) -> None: """Constructor """ self._base_path = "{{{basePath}}}" if host is None else host @@ -279,6 +339,9 @@ conf = {{{packageName}}}.Configuration( self.date_format = "{{{dateFormat}}}" """date format """ + + self.timeouts = timeouts + self.rate_limit_retries = rate_limit_retries def __deepcopy__(self, memo): cls = self.__class__ @@ -641,3 +704,25 @@ conf = {{{packageName}}}.Configuration( """Fix base path.""" self._base_path = value self.server_index = None + + @property + def timeouts(self): + return self._timeouts + + @timeouts.setter + def timeouts(self, value): + if not isinstance(value, Timeouts): + raise TypeError(f"timeouts must be type Timeouts but type '{type(value)}' used") + self._timeouts = value + + @property + def rate_limit_retries(self): + return self._rate_limit_retries + + @rate_limit_retries.setter + def rate_limit_retries(self, value): + if not isinstance(value, int): + raise TypeError(f"rate_limit_retries must be type int but type '{type(value)}' used") + if value < 0: + raise ValueError(f"rate_limit_retries must be greater than or equal to zero but was '{value}'") + self._rate_limit_retries = value diff --git a/generate/templates/example.mustache b/generate/templates/example.mustache index 205f01f..1d987ae 100644 --- a/generate/templates/example.mustache +++ b/generate/templates/example.mustache @@ -1,6 +1,7 @@ ```python {{#asyncio}}import asyncio{{/asyncio}} from {{{packageName}}}.exceptions import ApiException +from {{{packageName}}}.extensions.configuration_options import ConfigurationOptions from {{{modelPackage}}} import * from pprint import pprint from {{packageName}} import ( @@ -27,6 +28,14 @@ from {{packageName}} import ( # Use the {{packageName}} ApiClientFactory to build Api instances with a configured api client # By default this will read config from environment variables # Then from a secrets.json file found in the current working directory + + # uncomment the below to use configuration overrides + # opts = ConfigurationOptions(); + # opts.total_timeout_ms = 30_000 + + # uncomment the below to use an api client factory with overrides + # api_client_factory = ApiClientFactory(opts=opts) + api_client_factory = ApiClientFactory() # Enter a context with an instance of the ApiClientFactory to ensure the connection pool is closed after use @@ -44,6 +53,9 @@ from {{packageName}} import ( {{/allParams}} try: + # uncomment the below to set overrides at the request level + # {{#returnType}}api_response = {{/returnType}}{{#asyncio}}await {{/asyncio}}api_instance.{{{operationId}}}({{#allParams}}{{#required}}{{paramName}}{{/required}}{{^required}}{{paramName}}={{paramName}}{{/required}}, {{/allParams}}opts=opts) + {{#summary}} # {{{.}}} {{/summary}} diff --git a/generate/templates/extensions/api_client.mustache b/generate/templates/extensions/api_client.mustache index af936fb..3179709 100644 --- a/generate/templates/extensions/api_client.mustache +++ b/generate/templates/extensions/api_client.mustache @@ -150,7 +150,7 @@ class SyncApiClient: files=None, response_types_map=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None, - _request_auth=None): + _request_auth=None, opts=None): config = self.configuration @@ -218,7 +218,8 @@ class SyncApiClient: headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, - _request_timeout=_request_timeout) + _request_timeout=_request_timeout, + opts=opts) except ApiException as e: if e.body: e.body = e.body.decode('utf-8') @@ -378,7 +379,8 @@ class SyncApiClient: response_types_map=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, - _request_timeout=None, _host=None, _request_auth=None): + _request_timeout=None, _host=None, _request_auth=None, + opts=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. @@ -405,10 +407,9 @@ class SyncApiClient: Default is True. :param collection_formats: dict of collection formats for path, query, header, and post parameters. - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional :param _request_auth: set to override the auth_settings for an a single request; this effectively ignores the authentication in the spec for a single request. @@ -427,7 +428,7 @@ class SyncApiClient: response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, - _request_auth) + _request_auth, opts) return self.pool.apply_async(self.__call_api, (resource_path, method, path_params, @@ -440,30 +441,33 @@ class SyncApiClient: collection_formats, _preload_content, _request_timeout, - _host, _request_auth)) + _host, _request_auth, opts)) def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): """Makes the HTTP request using RESTClient.""" if method == "GET": return self.rest_client.get_request(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - headers=headers) + headers=headers, + opts=opts) elif method == "HEAD": return self.rest_client.head_request(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - headers=headers) + headers=headers, + opts=opts) elif method == "OPTIONS": return self.rest_client.options_request(url, query_params=query_params, headers=headers, _preload_content=_preload_content, - _request_timeout=_request_timeout) + _request_timeout=_request_timeout, + opts=opts) elif method == "POST": return self.rest_client.post_request(url, query_params=query_params, @@ -471,7 +475,8 @@ class SyncApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "PUT": return self.rest_client.put_request(url, query_params=query_params, @@ -479,7 +484,8 @@ class SyncApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "PATCH": return self.rest_client.patch_request(url, query_params=query_params, @@ -487,14 +493,16 @@ class SyncApiClient: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) elif method == "DELETE": return self.rest_client.delete_request(url, query_params=query_params, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) else: raise ApiValueError( "http method must be `GET`, `HEAD`, `OPTIONS`," diff --git a/generate/templates/extensions/api_client_factory.mustache b/generate/templates/extensions/api_client_factory.mustache index f1c6372..0dd9a8e 100644 --- a/generate/templates/extensions/api_client_factory.mustache +++ b/generate/templates/extensions/api_client_factory.mustache @@ -1,5 +1,6 @@ from __future__ import annotations from {{packageName}}.api_client import ApiClient +from {{packageName}}.extensions.configuration_options import ConfigurationOptions from {{packageName}}.extensions.retry import ( RetryingRestWrapper, RetryingRestWrapperAsync @@ -68,6 +69,7 @@ class SyncApiClientFactory: ] = keep_alive_socket_options(), correlation_id: Optional[str] = None, app_name: Optional[str] = None, + opts: Optional[ConfigurationOptions] = None, ): """Create an ApiClientFactory which can build api objects with a configured ApiClient object @@ -96,7 +98,8 @@ class SyncApiClientFactory: api_client_config = api_config.build_api_client_config( tcp_keep_alive=tcp_keep_alive, socket_options=socket_options, - id_provider_response_handler=id_provider_response_handler + id_provider_response_handler=id_provider_response_handler, + opts=opts ) self.__api_client = SyncApiClient( configuration=api_client_config, @@ -107,7 +110,9 @@ class SyncApiClientFactory: if tcp_keep_alive: rc.pool_manager.pool_classes_by_scheme = {"http": TCPKeepAliveHTTPConnectionPool, "https": TCPKeepAliveHTTPSConnectionPool} - wrapped_rest_client = rest_client_wrapper(rc) + wrapped_rest_client = rest_client_wrapper( + rest_object=rc, + rate_limit_retries=api_client_config.rate_limit_retries) self.__api_client.rest_client = wrapped_rest_client set_additional_api_client_headers( @@ -153,7 +158,8 @@ class ApiClientFactory: correlation_id: Optional[str] = None, app_name: Optional[str] = None, client_session: Optional[ClientSession] = None, - trace_configs: Optional[List[TraceConfig]] = None + trace_configs: Optional[List[TraceConfig]] = None, + opts: Optional[ConfigurationOptions] = None, ): """Create an ApiClientFactory which can build api objects with a configured ApiClient object @@ -189,7 +195,8 @@ class ApiClientFactory: api_client_config = api_config.build_api_client_config( tcp_keep_alive=tcp_keep_alive, socket_options=socket_options, - id_provider_response_handler=id_provider_response_handler + id_provider_response_handler=id_provider_response_handler, + opts=opts ) self.__api_client = ApiClient( configuration=api_client_config, @@ -218,7 +225,9 @@ class ApiClientFactory: logger.exception("client_session must be an aiohttp.ClientSession" " object with an initialised TCP Connector") rest_client_wrapper = RetryingRestWrapperAsync - wrapped_rest_client = rest_client_wrapper(rc) + wrapped_rest_client = rest_client_wrapper( + rest_object=rc, + rate_limit_retries=api_client_config.rate_limit_retries) self.__api_client.rest_client = wrapped_rest_client set_additional_api_client_headers( self.__api_client, app_name=app_name, correlation_id=correlation_id diff --git a/generate/templates/extensions/api_configuration.mustache b/generate/templates/extensions/api_configuration.mustache index 9cfd0b1..27667dd 100644 --- a/generate/templates/extensions/api_configuration.mustache +++ b/generate/templates/extensions/api_configuration.mustache @@ -1,7 +1,8 @@ import re import logging from typing import Optional, Union, Tuple, Any, Callable -from {{packageName}}.configuration import Configuration +from {{packageName}}.configuration import Configuration, Timeouts +from {{packageName}}.extensions.configuration_options import ConfigurationOptions from {{packageName}}.extensions.refreshing_token import RefreshingToken from {{packageName}}.extensions.socket_keep_alive import keep_alive_socket_options from {{packageName}}.extensions.proxy_config import ProxyConfig @@ -22,6 +23,10 @@ class ApiConfiguration: certificate_filename=None, proxy_config:Optional[ProxyConfig]=None, access_token=None, + total_timeout_ms=None, + connect_timeout_ms=None, + read_timeout_ms=None, + rate_limit_retries=None, ): """ The configuration required to access LUSID, read more at https://support.finbourne.com/getting-started-with-apis-sdks @@ -46,6 +51,10 @@ class ApiConfiguration: self.__certificate_filename = certificate_filename self.__proxy_config = proxy_config self.__access_token = access_token + self.__total_timeout_ms = total_timeout_ms + self.__connect_timeout_ms = connect_timeout_ms + self.__read_timeout_ms = read_timeout_ms + self.__rate_limit_retries = rate_limit_retries @property def token_url(self): @@ -181,6 +190,35 @@ class ApiConfiguration: ) raise + @property + def total_timeout_ms(self): + return self.__total_timeout_ms + + @total_timeout_ms.setter + def total_timeout_ms(self, value): + self.__total_timeout_ms = value + + @property + def connect_timeout_ms(self): + return self.__connect_timeout_ms + + @connect_timeout_ms.setter + def connect_timeout_ms(self, value): + self.__connect_timeout_ms = value + + @property + def read_timeout_ms(self): + return self.__read_timeout_ms + + @read_timeout_ms.setter + def read_timeout_ms(self, value): + self.__read_timeout_ms = value + + @property + def rate_limit_retries(self): + return self.__rate_limit_retries + + def build_api_client_config( self, tcp_keep_alive: bool = True, @@ -188,6 +226,7 @@ class ApiConfiguration: Union[Tuple[Any, Any, Any], Tuple[Any, Any, None, int]] ] = keep_alive_socket_options(), id_provider_response_handler: Optional[Callable[[Response], None]] = None, + opts: ConfigurationOptions = None, ) -> Configuration: """Builds lusid.Configuration for initialising an api client. @@ -219,6 +258,16 @@ class ApiConfiguration: access_token=access_token, host=self.api_url, ssl_ca_cert=self.certificate_filename, + timeouts=Timeouts( + total_timeout_ms=opts.total_timeout_ms if opts and opts.total_timeout_ms != None else + self.total_timeout_ms if self.total_timeout_ms != None else Configuration.DEFAULT_TOTAL_TIMEOUT_MS, + connect_timeout_ms=opts.connect_timeout_ms if opts and opts.connect_timeout_ms != None else + self.connect_timeout_ms if self.connect_timeout_ms != None else Configuration.DEFAULT_CONNECT_TIMEOUT_MS, + read_timeout_ms=opts.read_timeout_ms if opts and opts.read_timeout_ms != None else + self.read_timeout_ms if self.read_timeout_ms != None else Configuration.DEFAULT_READ_TIMEOUT_MS + ), + rate_limit_retries=opts.rate_limit_retries if opts != None and opts.rate_limit_retries != None else + self.rate_limit_retries if self.rate_limit_retries != None else Configuration.DEFAULT_RATE_LIMIT_RETRIES ) if tcp_keep_alive: config.socket_options = socket_options or keep_alive_socket_options() diff --git a/generate/templates/extensions/configuration_loaders.mustache b/generate/templates/extensions/configuration_loaders.mustache index 6560d67..7b7796f 100644 --- a/generate/templates/extensions/configuration_loaders.mustache +++ b/generate/templates/extensions/configuration_loaders.mustache @@ -5,6 +5,7 @@ import logging from {{packageName}}.extensions.proxy_config import ProxyConfig from {{packageName}}.extensions.api_configuration import ApiConfiguration from {{packageName}}.extensions.file_access_token import FileAccessToken +from {{packageName}}.extensions.configuration_options import ConfigurationOptions logger = logging.getLogger(__name__) @@ -21,7 +22,11 @@ ENVIRONMENT_CONFIG_KEYS = { "proxy_address": "FBN_PROXY_ADDRESS", "proxy_username": "FBN_PROXY_USERNAME", "proxy_password": "FBN_PROXY_PASSWORD", - "access_token": "FBN_ACCESS_TOKEN" + "access_token": "FBN_ACCESS_TOKEN", + "total_timeout_ms": "FBN_TOTAL_TIMEOUT_MS", + "connect_timeout_ms": "FBN_CONNECT_TIMEOUT_MS", + "read_timeout_ms": "FBN_READ_TIMEOUT_MS", + "rate_limit_retries": "FBN_RATE_LIMIT_RETRIES", } SECRETS_FILE_CONFIG_KEYS = { @@ -37,7 +42,11 @@ SECRETS_FILE_CONFIG_KEYS = { "proxy_address": "address", "proxy_username": "username", "proxy_password": "password", - "access_token": "accessToken" + "access_token": "accessToken", + "total_timeout_ms": "totalTimeoutMs", + "connect_timeout_ms": "connectTimeoutMs", + "read_timeout_ms": "readTimeoutMs", + "rate_limit_retries": "rateLimitRetries", } @@ -47,7 +56,7 @@ class ConfigurationLoader(Protocol): environment variables with preference given to the secrets file. """ - def load_config(self) -> Dict[str, str]: + def load_config(self) -> Dict[str, object]: pass @@ -67,12 +76,12 @@ class SecretsFileConfigurationLoader: """ self._api_secrets_file = api_secrets_file or "" - def load_config(self) -> Dict[str, str]: + def load_config(self) -> Dict[str, object]: """reads config from the provided secrets file Returns ------- - Dict[str, str] + Dict[str, object] dictionary that can be loaded into an ApiConfiguration object """ # The secrets file is a nested dictionary, set the names of the top level keys @@ -117,12 +126,12 @@ class SecretsFileConfigurationLoader: class EnvironmentVariablesConfigurationLoader: """ConfigurationLoader which reads config from environment variables """ - def load_config(self) -> Dict[str, str]: + def load_config(self) -> Dict[str, object]: """reads config from environment variables Returns ------- - Dict[str, str] + Dict[str, object] dictionary that can be loaded into an ApiConfiguration object """ logger.debug("loading config from environment variables") @@ -134,6 +143,13 @@ class EnvironmentVariablesConfigurationLoader: } if not populated_api_config_values["api_url"]: populated_api_config_values["api_url"] = populated_api_config_values["previous_api_url"] + # ensure that these values are ints + for key in ["total_timeout_ms", "connect_timeout_ms", "read_timeout_ms", "rate_limit_retries"]: + if populated_api_config_values[key]: + try: + populated_api_config_values[key] = int(populated_api_config_values[key]) + except ValueError as e: + raise ValueError(f"invalid value for '{key}' - value must be an integer if set") del(populated_api_config_values["previous_api_url"]) populated_proxy_values = { key: os.environ.get(value) @@ -162,7 +178,11 @@ class ArgsConfigurationLoader: proxy_address:Optional[str]=None, proxy_username:Optional[str]=None, proxy_password:Optional[str]=None, - access_token:Optional[str]=None + access_token:Optional[str]=None, + total_timeout_ms:Optional[int]=None, + connect_timeout_ms:Optional[int]=None, + read_timeout_ms:Optional[int]=None, + rate_limit_retries:Optional[int]=None, ): """kwargs passed to this constructor used to build ApiConfiguration """ @@ -178,13 +198,17 @@ class ArgsConfigurationLoader: self.__proxy_username = proxy_username self.__proxy_password = proxy_password self.__access_token = access_token + self.__total_timeout_ms = total_timeout_ms + self.__connect_timeout_ms = connect_timeout_ms + self.__read_timeout_ms = read_timeout_ms + self.__rate_limit_retries = rate_limit_retries - def load_config(self) -> Dict[str, str]: + def load_config(self) -> Dict[str, object]: """load configuration from kwargs passed to constructor Returns ------- - Dict[str, str] + Dict[str, object] dictionary that can be loaded into an ApiConfiguration object """ logger.debug("loading config from arguments passed to ArgsConfigurationLoader") @@ -200,7 +224,11 @@ class ArgsConfigurationLoader: "proxy_address" : self.__proxy_address, "proxy_username" : self.__proxy_username, "proxy_password" : self.__proxy_password, - "access_token" : self.__access_token + "access_token" : self.__access_token, + "total_timeout_ms" : self.__total_timeout_ms, + "connect_timeout_ms" : self.__connect_timeout_ms, + "read_timeout_ms" : self.__read_timeout_ms, + "rate_limit_retries" : self.__rate_limit_retries, } diff --git a/generate/templates/extensions/configuration_options.mustache b/generate/templates/extensions/configuration_options.mustache new file mode 100644 index 0000000..55e04c1 --- /dev/null +++ b/generate/templates/extensions/configuration_options.mustache @@ -0,0 +1,67 @@ +from typing import Optional + +class ConfigurationOptions: + + def __init__( + self, + total_timeout_ms: Optional[int] = None, + connect_timeout_ms: Optional[int] = None, + read_timeout_ms: Optional[int] = None, + rate_limit_retries: Optional[int] = None + ): + self.total_timeout_ms = total_timeout_ms + self.connect_timeout_ms = connect_timeout_ms + self.read_timeout_ms = read_timeout_ms + self.rate_limit_retries = rate_limit_retries + + @property + def total_timeout_ms(self): + return self.__total_timeout_ms + + @total_timeout_ms.setter + def total_timeout_ms(self, value): + if value: + if not isinstance(value, int): + raise TypeError(f"total_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise ValueError(f"total_timeout_ms must be an integer greater than or equal to zero") + self.__total_timeout_ms = value + + @property + def connect_timeout_ms(self): + return self.__connect_timeout_ms + + @connect_timeout_ms.setter + def connect_timeout_ms(self, value): + if value: + if not isinstance(value, int): + raise TypeError(f"connect_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise ValueError(f"connect_timeout_ms must be an integer greater than or equal to zero") + self.__connect_timeout_ms = value + + @property + def read_timeout_ms(self): + return self.__read_timeout_ms + + @read_timeout_ms.setter + def read_timeout_ms(self, value): + if value: + if not isinstance(value, int): + raise TypeError(f"read_timeout_ms must be type int but type '{type(value)}' used") + if value < 0: + raise ValueError(f"read_timeout_ms must be an integer greater than or equal to zero") + self.__read_timeout_ms = value + + @property + def rate_limit_retries(self): + return self.__rate_limit_retries + + @rate_limit_retries.setter + def rate_limit_retries(self, value): + if value: + if not isinstance(value, int): + raise TypeError(f"rate_limit_retries must be type int but type '{type(value)}' used") + if value < 0: + raise ValueError(f"rate_limit_retries must be an integer greater than or equal to zero") + self.__rate_limit_retries = value \ No newline at end of file diff --git a/generate/templates/extensions/rest.mustache b/generate/templates/extensions/rest.mustache index de70e55..d358355 100644 --- a/generate/templates/extensions/rest.mustache +++ b/generate/templates/extensions/rest.mustache @@ -103,9 +103,15 @@ class RESTClientObject(object): **addition_pool_args ) + self.timeout = self.get_timeout( + total=configuration.timeouts.total_timeout_ms / 1000.0, + connect=configuration.timeouts.connect_timeout_ms / 1000.0, + read=configuration.timeouts.read_timeout_ms / 1000.0, + ) + def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): """Perform requests. :param method: http request method @@ -119,10 +125,9 @@ class RESTClientObject(object): :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. + :param _request_timeout: Timeout setting. Do not use - use the opts parameter instead + :param opts: Configuration options for this request + :type opts: ConfigurationOptions, optional """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', @@ -139,14 +144,45 @@ class RESTClientObject(object): # so reset query_params to empty dict query_params = {} + # _request_timeout param cannot be removed for backwards compatability + # values from opts override values from _request_timeout + # try to get values from opts first, then _request_timeout, then self.timeout, else set to None + # timeout = _request_timeout or self.timeout timeout = None - if _request_timeout: - if isinstance(_request_timeout, (int,float)): # noqa: E501,F821 - timeout = urllib3.Timeout(total=_request_timeout) - elif (isinstance(_request_timeout, tuple) and - len(_request_timeout) == 2): - timeout = urllib3.Timeout( - connect=_request_timeout[0], read=_request_timeout[1]) + opts_total_timeout = opts.total_timeout_ms / 1000.0 if opts and opts.total_timeout_ms != None else None + opts_connect_timeout = opts.connect_timeout_ms / 1000.0 if opts and opts.connect_timeout_ms != None else None + opts_read_timeout = opts.read_timeout_ms / 1000.0 if opts and opts.read_timeout_ms != None else None + if not _request_timeout: + timeout = self.get_timeout( + total=opts_total_timeout if opts_total_timeout != None + else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None + else self.timeout._connect, + read=opts_read_timeout if opts_read_timeout != None + else self.timeout._read) + elif isinstance(_request_timeout, urllib3.Timeout): + timeout = self.get_timeout( + total=opts_total_timeout if opts_total_timeout != None + else _request_timeout.total if _request_timeout.total != None + else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None + else _request_timeout._connect if _request_timeout._connect != None + else self.timeout._connect, + read=opts_read_timeout if opts_read_timeout != None + else _request_timeout._read if _request_timeout._read != None + else self.timeout._read) + elif isinstance(_request_timeout, (int, float)): + timeout = self.get_timeout( + total=opts_total_timeout if opts_total_timeout != None else _request_timeout, + connect=opts_connect_timeout if opts_connect_timeout != None else self.timeout._connect, + read=opts_read_timeout if opts_read_timeout != None else self.timeout._read) + elif (isinstance(_request_timeout, tuple) and len(_request_timeout) == 2): + timeout = self.get_timeout( + total=opts_total_timeout if opts_total_timeout != None else self.timeout.total, + connect=opts_connect_timeout if opts_connect_timeout != None else _request_timeout[0], + read=opts_read_timeout if opts_read_timeout != None else _request_timeout[1]) + else: + raise f"unexpected type '{type(_request_timeout)}' for _request_timeout" try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` @@ -235,66 +271,82 @@ class RESTClientObject(object): return r def get_request(self, url, headers=None, query_params=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params) + query_params=query_params, + opts=opts) def head_request(self, url, headers=None, query_params=None, _preload_content=True, - _request_timeout=None): + _request_timeout=None, opts=None): return self.request("HEAD", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params) + query_params=query_params, + opts=opts) def options_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, opts=None): return self.request("OPTIONS", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def delete_request(self, url, headers=None, query_params=None, body=None, - _preload_content=True, _request_timeout=None): + _preload_content=True, _request_timeout=None, opts=None): return self.request("DELETE", url, headers=headers, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def post_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, opts=None): return self.request("POST", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def put_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, opts=None): return self.request("PUT", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def patch_request(self, url, headers=None, query_params=None, post_params=None, - body=None, _preload_content=True, _request_timeout=None): + body=None, _preload_content=True, _request_timeout=None, opts=None): return self.request("PATCH", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) + + def get_timeout(self, total: int, connect: int, read: int): + # zero is used in the sdk config to explicitly set to infinity (and None is used to indicate the value has not been set) + # but zero is not an allowed value for urllib3.Timeout so change any zeros to Nones + return urllib3.Timeout( + total=total if total != 0 else None, + connect=connect if connect != 0 else None, + read=read if read != 0 else None, + ) diff --git a/generate/templates/extensions/retry.mustache b/generate/templates/extensions/retry.mustache index ba41a5f..59b87fe 100644 --- a/generate/templates/extensions/retry.mustache +++ b/generate/templates/extensions/retry.mustache @@ -1,19 +1,33 @@ import time -from typing import overload +from typing import Optional -from {{packageName}} import ApiException import asyncio +from {{packageName}}.configuration import Configuration +from {{packageName}}.exceptions import ApiException + class RetryingRestWrapper: """Wrapper for HTTP requests Which retries on failure And waits the amount of time specified in the Retry After header. """ - def __init__(self, rest_object, retries: int = 3): + def __init__( + self, + rest_object, + retries: int = Configuration.DEFAULT_RETRIES, + rate_limit_retries: int = Configuration.DEFAULT_RATE_LIMIT_RETRIES + ): if not isinstance(retries, int): - raise ValueError(f"retries should be an int, found {type(self.retries)}") + raise TypeError(f"retries should be an int, found {type(retries)}") + if retries < 0: + raise ValueError(f"retries should be greater than or equal to zero but was '{retries}'") + if not isinstance(rate_limit_retries, int): + raise TypeError(f"rate_limit_retries should be an int, found {type(rate_limit_retries)}") + if rate_limit_retries < 0: + raise ValueError(f"rate_limit_retries should be greater than or equal to zero but was '{rate_limit_retries}'") self.retries: int = retries + self.rate_limit_retries: int = rate_limit_retries self.rest_object = rest_object def request( @@ -26,9 +40,12 @@ class RetryingRestWrapper: post_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): - tries = 0 - while tries < self.retries + 1: + retries_count = 0 + rate_limit_retries_count = 0 + + while True: try: return self.rest_object.request( method, @@ -39,17 +56,26 @@ class RetryingRestWrapper: post_params, _preload_content, _request_timeout, + opts, ) except ApiException as ex: - - retry_after = ex.headers.get("Retry-After") - # have done max number of retries - if tries >= self.retries: - raise + if ex.status == 429 and ((opts != None and opts.rate_limit_retries != None) or self.rate_limit_retries != None): + # check for limit of rate limit retries + limit = opts.rate_limit_retries if (opts and opts.rate_limit_retries != None) else self.rate_limit_retries + if rate_limit_retries_count >= limit: + raise + rate_limit_retries_count += 1 + else: + # check for limit of all other retries + if retries_count >= self.retries: + raise + retries_count += 1 + + retry_after = ex.headers.get("Retry-After") # try after delay - elif retry_after is not None: + if retry_after is not None: if not isinstance(retry_after, float): try: retry_after = float(retry_after) @@ -61,7 +87,6 @@ class RetryingRestWrapper: # no retry header else: raise - tries += 1 def get_request( self, @@ -70,12 +95,14 @@ class RetryingRestWrapper: query_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params) + query_params=query_params, + opts=opts) def head_request( self, @@ -84,12 +111,14 @@ class RetryingRestWrapper: query_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("HEAD", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params) + query_params=query_params, + opts=opts) def options_request( self, @@ -100,6 +129,7 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("OPTIONS", url, headers=headers, @@ -107,7 +137,8 @@ class RetryingRestWrapper: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def delete_request( self, @@ -117,13 +148,15 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("DELETE", url, headers=headers, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def post_request( self, @@ -134,6 +167,7 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("POST", url, headers=headers, @@ -141,7 +175,8 @@ class RetryingRestWrapper: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def put_request( self, @@ -152,6 +187,7 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("PUT", url, headers=headers, @@ -159,7 +195,8 @@ class RetryingRestWrapper: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) def patch_request( self, @@ -170,6 +207,7 @@ class RetryingRestWrapper: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return self.request("PATCH", url, headers=headers, @@ -177,7 +215,8 @@ class RetryingRestWrapper: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body) + body=body, + opts=opts) class RetryingRestWrapperAsync: @@ -185,10 +224,17 @@ class RetryingRestWrapperAsync: Which retries on failure And waits the amount of time specified in the Retry After header. """ - def __init__(self, rest_object, retries: int = 3): + def __init__(self, rest_object, retries: int = 3, rate_limit_retries: Optional[int] = Configuration.DEFAULT_RATE_LIMIT_RETRIES): if not isinstance(retries, int): - raise ValueError(f"retries should be an int, found {type(self.retries)}") + raise TypeError(f"retries should be an int, found {type(retries)}") + if retries < 0: + raise ValueError(f"retries should be greater than or equal to zero but was '{retries}'") + if not isinstance(rate_limit_retries, int): + raise TypeError(f"rate_limit_retries should be an int, found {type(rate_limit_retries)}") + if rate_limit_retries < 0: + raise ValueError(f"rate_limit_retries should be greater than or equal to zero but was '{rate_limit_retries}'") self.retries: int = retries + self.rate_limit_retries: Optional[int] = rate_limit_retries self.rest_object = rest_object async def close(self): @@ -204,9 +250,12 @@ class RetryingRestWrapperAsync: post_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): - tries = 0 - while tries < self.retries + 1: + retries_count = 0 + rate_limit_retries_count = 0 + + while True: try: return await self.rest_object.request( method, @@ -217,16 +266,26 @@ class RetryingRestWrapperAsync: post_params, _preload_content, _request_timeout, + opts ) except ApiException as ex: - retry_after = ex.headers.get("Retry-After") - # have done max number of retries - if tries >= self.retries: - raise + if ex.status == 429 and ((opts and opts.rate_limit_retries != None) or self.rate_limit_retries != None): + # check for limit of rate limit retries + limit = opts.rate_limit_retries if (opts and opts.rate_limit_retries != None) else self.rate_limit_retries + if rate_limit_retries_count >= limit: + raise + rate_limit_retries_count += 1 + else: + # check for limit of all other retries + if retries_count >= self.retries: + raise + retries_count += 1 + + retry_after = ex.headers.get("Retry-After") # try after delay - elif retry_after is not None: + if retry_after is not None: if not isinstance(retry_after, float): try: retry_after = float(retry_after) @@ -238,7 +297,6 @@ class RetryingRestWrapperAsync: # no retry header else: raise - tries += 1 async def get_request( self, @@ -247,12 +305,14 @@ class RetryingRestWrapperAsync: query_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params)) + query_params=query_params, + opts=opts)) async def head_request( self, @@ -261,12 +321,14 @@ class RetryingRestWrapperAsync: query_params=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("HEAD", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, - query_params=query_params)) + query_params=query_params, + opts=opts)) async def options_request( self, @@ -277,6 +339,7 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("OPTIONS", url, headers=headers, @@ -284,7 +347,8 @@ class RetryingRestWrapperAsync: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def delete_request( self, @@ -294,13 +358,15 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("DELETE", url, headers=headers, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def post_request( self, @@ -311,6 +377,7 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("POST", url, headers=headers, @@ -318,7 +385,8 @@ class RetryingRestWrapperAsync: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def put_request( self, @@ -329,6 +397,7 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("PUT", url, headers=headers, @@ -336,7 +405,8 @@ class RetryingRestWrapperAsync: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) async def patch_request( self, @@ -347,6 +417,7 @@ class RetryingRestWrapperAsync: body=None, _preload_content=True, _request_timeout=None, + opts=None, ): return (await self.request("PATCH", url, headers=headers, @@ -354,4 +425,5 @@ class RetryingRestWrapperAsync: post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, - body=body)) + body=body, + opts=opts)) diff --git a/justfile b/justfile index e6a28ad..fde9d35 100644 --- a/justfile +++ b/justfile @@ -22,6 +22,7 @@ export FBN_CLIENT_ID := `echo ${FBN_CLIENT_ID:-client-id}` export FBN_CLIENT_SECRET := `echo ${FBN_CLIENT_SECRET:-client-secret}` export TEST_API_MODULE := `echo ${TEST_API_MODULE:-api.application_metadata_api}` export TEST_API := `echo ${TEST_API:-ApplicationMetadataApi}` +export TEST_METHOD := `echo ${TEST_METHOD:-'list_access_controlled_resources('}` export GENERATE_API_TESTS := `echo ${GENERATE_API_TESTS:-false}` swagger_path := "./swagger.json" @@ -72,10 +73,11 @@ add-tests: # rename to match values for the sdk being tested find {{justfile_directory()}}/generate/.output/sdk/test -type f -exec sed -i -e "s/TO_BE_REPLACED/${PACKAGE_NAME}/g" {} \; - # note these values won't work for the horizon sdk - # (it doesn't have this api) + # note the default values at the top of this justfile won't work for the horizon or luminesce sdk + # (they don't have this api/method) find {{justfile_directory()}}/generate/.output/sdk/test -type f -exec sed -i -e "s/TEST_API_MODULE/${TEST_API_MODULE}/g" {} \; find {{justfile_directory()}}/generate/.output/sdk/test -type f -exec sed -i -e "s/TEST_API/${TEST_API}/g" {} \; + find {{justfile_directory()}}/generate/.output/sdk/test -type f -exec sed -i -e "s/TEST_METHOD/${TEST_METHOD}/g" {} \; link-tests-cicd TARGET_DIR: mkdir -p {{TARGET_DIR}}/sdk/test/ @@ -87,6 +89,7 @@ link-tests-cicd TARGET_DIR: find {{justfile_directory()}}/test_sdk -type f -exec sed -i -e "s/TO_BE_REPLACED/${PLACEHOLDER_VALUE_FOR_TESTS}/g" {} \; find {{justfile_directory()}}/test_sdk -type f -exec sed -i -e "s/TEST_API_MODULE/${TEST_API_MODULE}/g" {} \; find {{justfile_directory()}}/test_sdk -type f -exec sed -i -e "s/TEST_API/${TEST_API}/g" {} \; + find {{justfile_directory()}}/test_sdk -type f -exec sed -i -e "s/TEST_METHOD/${TEST_METHOD}/g" {} \; setup-test-local: @just generate-local diff --git a/test_sdk/unit/test_api_client_factory.py b/test_sdk/unit/test_api_client_factory.py index 445cd80..4b16e8b 100644 --- a/test_sdk/unit/test_api_client_factory.py +++ b/test_sdk/unit/test_api_client_factory.py @@ -87,6 +87,7 @@ def test_build_returns_api_instance_with_sync_client(self): tcp_keep_alive=True, socket_options=keep_alive_socket_options(), id_provider_response_handler=None, + opts=None, ) args, kwargs = set_additional_api_client_headers_mock.call_args assert isinstance(args[0], SyncApiClient) @@ -114,6 +115,7 @@ def test_build_with_socket_options_and_tcp_keep_alive_True_sets_socket_options_o tcp_keep_alive=True, socket_options=socket_options, id_provider_response_handler=None, + opts=None, ) assert ( { @@ -145,6 +147,7 @@ async def test_build_returns_api_instance_with_async_client(self): tcp_keep_alive=True, socket_options=keep_alive_socket_options(), id_provider_response_handler=None, + opts=None, ) args, kwargs = set_additional_api_client_headers_mock.call_args assert isinstance(args[0], AsyncApiClient) diff --git a/test_sdk/unit/test_api_configuration.py b/test_sdk/unit/test_api_configuration.py index 741b33f..425166e 100644 --- a/test_sdk/unit/test_api_configuration.py +++ b/test_sdk/unit/test_api_configuration.py @@ -1,4 +1,6 @@ from unittest.mock import patch +from TO_BE_REPLACED.configuration import Configuration +from TO_BE_REPLACED.extensions.configuration_options import ConfigurationOptions from TO_BE_REPLACED.extensions.proxy_config import ProxyConfig from TO_BE_REPLACED.extensions.api_configuration import ApiConfiguration import pytest @@ -103,3 +105,49 @@ def test_build_api_client_config_with_api_url_sets_api_client_host( api_client = config.build_api_client_config() assert "sample_api_url" == api_client.host + +def test_build_api_client_config_sets_defaults_if_unset( + patch_get_access_token, +): + config = ApiConfiguration( + password="password", + username="me", + client_id="test_id", + client_secret="shhhh", + token_url="fake_url", + api_url="sample_api_url", + ) + + api_client = config.build_api_client_config() + assert api_client.timeouts.total_timeout_ms == Configuration.DEFAULT_TOTAL_TIMEOUT_MS + assert api_client.timeouts.connect_timeout_ms == Configuration.DEFAULT_CONNECT_TIMEOUT_MS + assert api_client.timeouts.read_timeout_ms == Configuration.DEFAULT_READ_TIMEOUT_MS + assert api_client.rate_limit_retries == Configuration.DEFAULT_RATE_LIMIT_RETRIES + +def test_build_api_client_uses_opts_overrides( + patch_get_access_token, +): + config = ApiConfiguration( + password="password", + username="me", + client_id="test_id", + client_secret="shhhh", + token_url="fake_url", + api_url="sample_api_url", + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000, + rate_limit_retries=5, + ) + + opts = ConfigurationOptions( + total_timeout_ms=6000, + connect_timeout_ms=5000, + read_timeout_ms=4000, + rate_limit_retries=10 + ) + api_client = config.build_api_client_config(opts=opts) + assert api_client.timeouts.total_timeout_ms == 6000 + assert api_client.timeouts.connect_timeout_ms == 5000 + assert api_client.timeouts.read_timeout_ms == 4000 + assert api_client.rate_limit_retries == 10 diff --git a/test_sdk/unit/test_config_overrides.py b/test_sdk/unit/test_config_overrides.py new file mode 100644 index 0000000..c0b2f55 --- /dev/null +++ b/test_sdk/unit/test_config_overrides.py @@ -0,0 +1,897 @@ +from typing import Optional +import aiohttp +import pytest +import urllib3 +import TO_BE_REPLACED +from TO_BE_REPLACED.api import TEST_API +from TO_BE_REPLACED.api_client import ApiClient +from TO_BE_REPLACED.configuration import Configuration, Timeouts +from TO_BE_REPLACED.exceptions import ApiException +from TO_BE_REPLACED.extensions.api_client import SyncApiClient +from TO_BE_REPLACED.extensions.api_client_factory import ApiClientFactory, SyncApiClientFactory +from TO_BE_REPLACED.extensions.configuration_loaders import ArgsConfigurationLoader +from TO_BE_REPLACED.extensions.configuration_options import ConfigurationOptions + +class MockPoolManager: + def __init__(self, return_status_code: int = 200): + self.timeouts = [] + self.callCount = 0 + self.return_status_code = return_status_code + + def request(self, method, url, **kwargs): + self.callCount += 1 + if "timeout" in kwargs: + self.timeouts.append(kwargs["timeout"]) + return MockResponse(self.return_status_code) + + async def close(self): + ... + +class MockResponse: + def __init__(self, status_code: int): + self.status = status_code + self.reason = "test" + self.data = "test".encode('utf-8') + self.headers = {"Retry-After":0} + + def getheader(self, header: str): + if header == "Retry-After": + return 0 + return None + + def getheaders(self): + return self.headers + + async def read(self): + return self.data + + def __await__(self): + async def closure(): + return self + return closure().__await__() + + +# contains tests that are common to both async and sync api clients, whether created manually or with an api client factory +class BaseTests: + + def _override_deserialize(_, __): + return "" + + async def _run_timeout_test( + self, + create_api_client_with_factory: bool, + async_req: Optional[bool], + is_async: bool, + configuration_timeouts: Timeouts, + factory_opts: ConfigurationOptions, + request_timeout, + request_opts: ConfigurationOptions, + expected_total_timeout: Optional[float], + expected_connect_timeout: Optional[float], + expected_read_timeout: Optional[float] + ): + mock_pool_manager = MockPoolManager() + if create_api_client_with_factory: + api = self._create_api_instance_with_factory( + is_async=is_async, + rate_limit_retries_config=None, + configuration_timeouts=configuration_timeouts, + opts=factory_opts, + mock_pool_manager=mock_pool_manager) + else: + api = self._create_api_instance_without_factory(is_async, configuration_timeouts, mock_pool_manager) + + # to avoid having to return data that can be deserialised into real types + # (which will vary from api to api and can change over time) + # override the deserialize method + api.api_client.deserialize = BaseTests._override_deserialize + + try: + result = api.TEST_METHOD + _request_timeout=request_timeout, + async_req=async_req, + opts=request_opts) + finally: + if is_async: + await api.api_client.close() + else: + api.api_client.close() + + if async_req: + result = result.get(timeout=60) + + if is_async: + await result + + assert mock_pool_manager.callCount == 1 + assert len(mock_pool_manager.timeouts) == 1 + timeout = mock_pool_manager.timeouts[0] + + if is_async: + # the timeout is of type aiohttp.ClientTimeout + assert timeout.total == expected_total_timeout + assert timeout.connect == expected_connect_timeout + assert timeout.sock_read == expected_read_timeout + else: + # the timeout is of type urllib3.Timeout + assert timeout.total == expected_total_timeout + assert timeout._connect == expected_connect_timeout + assert timeout._read == expected_read_timeout + + def _create_api_instance_without_factory( + self, + is_async: bool, + configuration_timeouts: Timeouts, + mock_pool_manager: MockPoolManager, + ): + configuration = Configuration(access_token="token", host="http://localhost", timeouts=configuration_timeouts) + api_client_type = ApiClient if is_async else SyncApiClient + api_client = api_client_type(configuration=configuration) + api_client.rest_client.pool_manager = mock_pool_manager + return TEST_API(api_client) + + def _create_api_instance_with_factory( + self, + is_async: bool, + rate_limit_retries_config: Optional[int], + configuration_timeouts: Timeouts, + opts: ConfigurationOptions, + mock_pool_manager: MockPoolManager, + ): + configuration_loader = ArgsConfigurationLoader( + access_token="token", + api_url="http://localhost", + total_timeout_ms=configuration_timeouts.total_timeout_ms, + connect_timeout_ms=configuration_timeouts.connect_timeout_ms, + read_timeout_ms=configuration_timeouts.read_timeout_ms, + rate_limit_retries=rate_limit_retries_config + ) + api_client_factory_type = ApiClientFactory if is_async else SyncApiClientFactory + api_client_factory = api_client_factory_type( + config_loaders=[configuration_loader], + opts=opts + ) + api = api_client_factory.build(TEST_API) + api.api_client.rest_client.rest_object.pool_manager = mock_pool_manager + return api + + async def test_timeout_when_set_on_request_timeout_via_int(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(3000, 2000, 1000), + factory_opts=None, + request_timeout=10, + request_opts=None, + expected_total_timeout=10, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + async def test_timeout_when_set_on_request_timeout_via_float(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(3000, 2000, 1000), + factory_opts=None, + request_timeout=10.1, + request_opts=None, + expected_total_timeout=10.1, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + async def test_timeout_when_set_on_request_opts_only(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=1100, + read_timeout_ms=2100 + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.1, + expected_read_timeout=2.1 + ) + + # some timeouts specified in opts + # no request_timeout + # all timeouts specified in configuration + # expect values from configuration to be used where not specified in opts + async def test_timeout_when_partially_set_on_request_opts(self, create_api_client_with_factory, async_req, is_async): + + # specify total and connect + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=1100, + read_timeout_ms=None + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.1, + expected_read_timeout=3.4 + ) + + # specify connect and read + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=1100, + read_timeout_ms=2100 + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.1, + expected_read_timeout=2.1 + ) + + # specify total and read + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=None, + read_timeout_ms=2100 + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.2, + expected_read_timeout=2.1 + ) + + # specify total only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=None, + read_timeout_ms=None + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.2, + expected_read_timeout=3.4 + ) + + # specify connect only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=1100, + read_timeout_ms=None + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.1, + expected_read_timeout=3.4 + ) + + # specify read only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=10200, + connect_timeout_ms=1200, + read_timeout_ms=3400 + ), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=None, + read_timeout_ms=2100 + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.2, + expected_read_timeout=2.1 + ) + + async def test_timeout_when_set_on_config_only(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000), + factory_opts=None, + request_timeout=None, + request_opts=None, + expected_total_timeout=3, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + async def test_timeout_when_set_on_request_timeout_only(self, create_api_client_with_factory, async_req, is_async): + request_timeout = aiohttp.ClientTimeout(total=10.2, connect=1.2, sock_read=3.4) if is_async else urllib3.Timeout(total=10.2, connect=1.2, read=3.4) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=None, + expected_total_timeout=10.2, + expected_connect_timeout=1.2, + expected_read_timeout=3.4 + ) + + # all timeouts specified in opts + # all timeouts specified in request_timeout + # expect values from opts to be used + async def test_request_opts_override_request_timeout(self, create_api_client_with_factory, async_req, is_async): + request_timeout = aiohttp.ClientTimeout(total=10.2, connect=1.2, sock_read=3.4) if is_async else urllib3.Timeout(total=10.2, connect=1.2, read=3.4) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=1100, + read_timeout_ms=2100 + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.1, + expected_read_timeout=2.1 + ) + + # some timeouts specified in opts + # all timeouts specified in request_timeout + # expect values from request_timeout to be used where not specified in opts + async def test_timeout_when_set_on_request_timeout_and_partially_set_on_request_opts(self, create_api_client_with_factory, async_req, is_async): + + # specify total and connect + request_timeout = aiohttp.ClientTimeout(total=10.2, connect=1.2, sock_read=3.4) if is_async else urllib3.Timeout(total=10.2, connect=1.2, read=3.4) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=1100, + read_timeout_ms=None + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.1, + expected_read_timeout=3.4 + ) + + # specify connect and read + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=1100, + read_timeout_ms=2100 + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.1, + expected_read_timeout=2.1 + ) + + # specify total and read + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=None, + read_timeout_ms=2100 + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.2, + expected_read_timeout=2.1 + ) + + # specify total only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=5100, + connect_timeout_ms=None, + read_timeout_ms=None + ), + expected_total_timeout=5.1, + expected_connect_timeout=1.2, + expected_read_timeout=3.4 + ) + + # specify connect only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=1100, + read_timeout_ms=None + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.1, + expected_read_timeout=3.4 + ) + + # specify read only + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=None, + read_timeout_ms=2100 + ), + expected_total_timeout=10.2, + expected_connect_timeout=1.2, + expected_read_timeout=2.1 + ) + + # some timeouts specified in opts + # some timeouts specified in request_timeout + # some timeouts specified in configuration + # expect precedence to be opts > request_timeout > configuration + async def test_timeout_when_partially_set_on_request_timeout_and_partially_set_on_request_opts(self, create_api_client_with_factory, async_req, is_async): + request_timeout = aiohttp.ClientTimeout(total=6, connect=5, sock_read=None) if is_async else urllib3.Timeout(total=6, connect=5, read=None) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=9000, + connect_timeout_ms=None, + read_timeout_ms=None + ), + expected_total_timeout=9, + expected_connect_timeout=5, + expected_read_timeout=1 + ) + + async def test_timeout_when_both_request_timeout_and_request_opts_all_values_set_to_none(self, create_api_client_with_factory, async_req, is_async): + request_timeout = aiohttp.ClientTimeout(total=None, connect=None, sock_read=None) if is_async else urllib3.Timeout(total=None, connect=None, read=None) + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts( + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000), + factory_opts=None, + request_timeout=request_timeout, + request_opts=ConfigurationOptions( + total_timeout_ms=None, + connect_timeout_ms=None, + read_timeout_ms=None + ), + expected_total_timeout=3, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_timeouts_respected_in_config(self, create_api_client_with_factory, async_req, is_async): + # urllib3 used in sync does not allow the timeout to be set to zero - ensure it's converted to None + expected_timeout = 0 if is_async else None + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(0, 0, 0), + factory_opts=None, + request_timeout=None, + request_opts=None, + expected_total_timeout=expected_timeout, + expected_connect_timeout=expected_timeout, + expected_read_timeout=expected_timeout + ) + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_timeouts_respected_in_request_opts(self, create_api_client_with_factory, async_req, is_async): + # urllib3 used in sync does not allow the timeout to be set to zero - ensure it's converted to None + expected_timeout = 0 if is_async else None + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(5000, 5000, 5000), + factory_opts=None, + request_timeout=None, + request_opts=ConfigurationOptions( + total_timeout_ms=0, + connect_timeout_ms=0, + read_timeout_ms=0 + ), + expected_total_timeout=expected_timeout, + expected_connect_timeout=expected_timeout, + expected_read_timeout=expected_timeout + ) + +# contains tests that are common to async clients created either manually or through the api client factory +class BaseAsyncTests(BaseTests): + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_timeouts_respected_in_request_timeout(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(5000, 5000, 5000), + factory_opts=None, + request_timeout=aiohttp.ClientTimeout(total=0, connect=0, sock_read=0), + request_opts=None, + expected_total_timeout=0, + expected_connect_timeout=0, + expected_read_timeout=0 + ) + +# contains tests that are common to sync clients created either manually or through the api client factory +class BaseSyncTests(BaseTests): + + # this tuple is only valid with urllib3.Timeout + async def test_timeout_when_set_on_request_via_tuple(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=None, + request_timeout=(1,2), + request_opts=None, + expected_total_timeout=Configuration.DEFAULT_TOTAL_TIMEOUT_MS / 1000.0, + expected_connect_timeout=1, + expected_read_timeout=2 + ) + +# contains test that are common to sync and async clients created through the api client factory +# NOTE: that the retries tests are not in BaseTests because the api client created manually does not have the retrying rest client +class BaseApiClientFromFactoryTests(BaseTests): + + async def _run_rate_limit_test( + self, + async_req: Optional[bool], + is_async: bool, + rate_limit_retries_config: int, + factory_opts: ConfigurationOptions, + request_opts: ConfigurationOptions, + return_status_code: int, + expected_times_called: int + ): + mock_pool_manager = MockPoolManager(return_status_code) + api = self._create_api_instance_with_factory( + is_async=is_async, + rate_limit_retries_config=rate_limit_retries_config, + configuration_timeouts=Timeouts.get_default(), + opts=factory_opts, + mock_pool_manager=mock_pool_manager) + + try: + result = api.TEST_METHOD + async_req=async_req, + opts=request_opts) + + if async_req: + result = result.get(timeout=60) + + if is_async: + await result + except(ApiException): + ... + finally: + if is_async: + await api.api_client.close() + else: + api.api_client.close() + + assert mock_pool_manager.callCount == expected_times_called + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_timeouts_respected_in_factory_opts(self, create_api_client_with_factory, async_req, is_async): + # urllib3 used in sync does not allow the timeout to be set to zero - ensure it's converted to None + expected_timeout = 0 if is_async else None + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts(5000, 5000, 5000), + factory_opts=ConfigurationOptions( + total_timeout_ms=0, + connect_timeout_ms=0, + read_timeout_ms=0 + ), + request_timeout=None, + request_opts=None, + expected_total_timeout=expected_timeout, + expected_connect_timeout=expected_timeout, + expected_read_timeout=expected_timeout + ) + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_rate_limit_retries_respected_in_configuration(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=0, + factory_opts=None, + request_opts=None, + return_status_code=429, + expected_times_called=1 + ) + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_rate_limit_retries_respected_in_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=ConfigurationOptions(rate_limit_retries=0), + request_opts=None, + return_status_code=429, + expected_times_called=1 + ) + + # ensure the zeros are not treated as 'falsey' causing us to use the default value + async def test_zero_rate_limit_retries_respected_in_request_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=None, + request_opts=ConfigurationOptions(rate_limit_retries=0), + return_status_code=429, + expected_times_called=1 + ) + + async def test_timeout_when_set_on_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=Timeouts.get_default(), + factory_opts=ConfigurationOptions( + total_timeout_ms=5002, + connect_timeout_ms=4002, + read_timeout_ms=3002 + ), + request_timeout=None, + request_opts=None, + expected_total_timeout=5.002, + expected_connect_timeout=4.002, + expected_read_timeout=3.002 + ) + + async def test_timeout_when_partially_set_on_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_timeout_test( + create_api_client_with_factory=create_api_client_with_factory, + async_req=async_req, + is_async=is_async, + configuration_timeouts=ConfigurationOptions( + total_timeout_ms=3000, + connect_timeout_ms=2000, + read_timeout_ms=1000 + ), + factory_opts=ConfigurationOptions( + total_timeout_ms=5002 + ), + request_timeout=None, + request_opts=None, + expected_total_timeout=5.002, + expected_connect_timeout=2, + expected_read_timeout=1 + ) + + async def test_default_rate_limit_retries_if_none_set(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=None, + request_opts=None, + return_status_code=429, + expected_times_called=Configuration.DEFAULT_RATE_LIMIT_RETRIES + 1 + ) + + async def test_retries_config_used_when_not_429_response(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=10, + factory_opts=None, + request_opts=None, + return_status_code=409, + expected_times_called=Configuration.DEFAULT_RETRIES + 1 + ) + + async def test_rate_limit_retries_set_on_request_when_set_in_config(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=5, + factory_opts=None, + request_opts=None, + return_status_code=429, + expected_times_called=6 + ) + + async def test_rate_limit_retries_set_on_request_when_set_in_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=ConfigurationOptions(rate_limit_retries=6), + request_opts=None, + return_status_code=429, + expected_times_called=7 + ) + + async def test_rate_limit_retries_set_on_request_when_set_in_request_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=None, + request_opts=ConfigurationOptions(rate_limit_retries=7), + return_status_code=429, + expected_times_called=8 + ) + + async def test_rate_limit_retries_set_on_request_when_factory_opts_override_config(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=5, + factory_opts=ConfigurationOptions(rate_limit_retries=6), + request_opts=None, + return_status_code=429, + expected_times_called=7 + ) + + async def test_rate_limit_retries_set_on_request_when_request_opts_override_config(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=5, + factory_opts=None, + request_opts=ConfigurationOptions(rate_limit_retries=7), + return_status_code=429, + expected_times_called=8 + ) + + async def test_rate_limit_retries_set_on_request_when_request_opts_override_factory_opts(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=None, + factory_opts=ConfigurationOptions(rate_limit_retries=6), + request_opts=ConfigurationOptions(rate_limit_retries=7), + return_status_code=429, + expected_times_called=8 + ) + + async def test_rate_limit_retries_set_on_request_when_request_opts_override_factory_opts_override_config(self, create_api_client_with_factory, async_req, is_async): + await self._run_rate_limit_test( + async_req=async_req, + is_async=is_async, + rate_limit_retries_config=5, + factory_opts=ConfigurationOptions(rate_limit_retries=6), + request_opts=ConfigurationOptions(rate_limit_retries=7), + return_status_code=429, + expected_times_called=8 + ) + +@pytest.mark.parametrize("create_api_client_with_factory, async_req, is_async", + [(True, True, True), + (True, False, True), + (True, None, True)]) +@pytest.mark.asyncio +class TestWithAsyncApiClientFromFactory(BaseApiClientFromFactoryTests, BaseAsyncTests): + ... + +@pytest.mark.parametrize("create_api_client_with_factory, async_req, is_async", + [(False, True, True), + (False, False, True), + (False, None, True)]) +@pytest.mark.asyncio +class TestWithAsyncApiClientManualCreation(BaseAsyncTests): + ... + +@pytest.mark.parametrize("create_api_client_with_factory, async_req, is_async", + [(True, True, False), + (True, False, False), + (True, None, False)]) +@pytest.mark.asyncio +class TestWithSyncApiClientFromFactory(BaseApiClientFromFactoryTests, BaseSyncTests): + ... + +@pytest.mark.parametrize("create_api_client_with_factory, async_req, is_async", + [(False, True, False), + (False, False, False), + (False, None, False)]) +@pytest.mark.asyncio +class TestWithSyncApiClientManualCreation(BaseSyncTests): + ... diff --git a/test_sdk/unit/test_configuration.py b/test_sdk/unit/test_configuration.py new file mode 100644 index 0000000..68dab41 --- /dev/null +++ b/test_sdk/unit/test_configuration.py @@ -0,0 +1,24 @@ +import pytest +from TO_BE_REPLACED.configuration import Configuration + + +def test_errors_if_invalid_timeouts_set(): + configuration = Configuration() + assert configuration.timeouts != None + with pytest.raises(TypeError) as e: + configuration.timeouts = None + assert str(e.value) == "timeouts must be type Timeouts but type '' used" + +def test_errors_if_invalid_rate_limit_retries_type_set(): + configuration = Configuration() + assert configuration.rate_limit_retries != None + with pytest.raises(TypeError) as e: + configuration.rate_limit_retries = None + assert str(e.value) == "rate_limit_retries must be type int but type '' used" + +def test_errors_if_invalid_rate_limit_retries_value_set(): + configuration = Configuration() + assert configuration.rate_limit_retries != None + with pytest.raises(ValueError) as e: + configuration.rate_limit_retries = -1 + assert str(e.value) == "rate_limit_retries must be greater than or equal to zero but was '-1'" \ No newline at end of file diff --git a/test_sdk/unit/test_configuration_loaders.py b/test_sdk/unit/test_configuration_loaders.py index 9d7ab0f..8fef1ef 100644 --- a/test_sdk/unit/test_configuration_loaders.py +++ b/test_sdk/unit/test_configuration_loaders.py @@ -1,4 +1,5 @@ import json +import pytest from TO_BE_REPLACED.extensions import ( SecretsFileConfigurationLoader, EnvironmentVariablesConfigurationLoader, @@ -24,6 +25,10 @@ def test_load_config_loads_api_config(self): "proxy_address": None, "proxy_username": None, "proxy_password": None, + "total_timeout_ms": 3000, + "connect_timeout_ms": 2000, + "read_timeout_ms": 1000, + "rate_limit_retries": 5, } secrets_file_contents = { @@ -37,6 +42,10 @@ def test_load_config_loads_api_config(self): "applicationName": "sample_applicationName", "clientCertificate": "sample_clientCertificate", "accessToken": "accessToken", + "totalTimeoutMs": 3000, + "connectTimeoutMs": 2000, + "readTimeoutMs": 1000, + "rateLimitRetries": 5, } } api_secrets_file = "secrets.json" @@ -62,6 +71,10 @@ def test_load_config_loads_proxy_config(self): "proxy_address": "sample_address", "proxy_username": "sample_username", "proxy_password": "sample_password", + "total_timeout_ms": None, + "connect_timeout_ms": None, + "read_timeout_ms": None, + "rate_limit_retries": None, } secrets_file_contents = { @@ -94,6 +107,10 @@ def test_load_config_loads_api_config(self): "FBN_APP_NAME": "sample_applicationName", "FBN_CLIENT_CERTIFICATE": "sample_clientCertificate", "FBN_ACCESS_TOKEN": "accessToken", + "FBN_TOTAL_TIMEOUT_MS": "3000", + "FBN_CONNECT_TIMEOUT_MS": "2000", + "FBN_READ_TIMEOUT_MS": "1000", + "FBN_RATE_LIMIT_RETRIES": "5" } expected_config = { "token_url": "sample_tokenUrl", @@ -108,6 +125,10 @@ def test_load_config_loads_api_config(self): "proxy_address": None, "proxy_username": None, "proxy_password": None, + "total_timeout_ms": 3000, + "connect_timeout_ms": 2000, + "read_timeout_ms": 1000, + "rate_limit_retries": 5, } with mock.patch.dict("os.environ", environment_variables, clear=True): config_loader = EnvironmentVariablesConfigurationLoader() @@ -133,12 +154,30 @@ def test_load_config_loads_proxy_config(self): "proxy_address": "sample_address", "proxy_username": "sample_username", "proxy_password": "sample_password", + "total_timeout_ms": None, + "connect_timeout_ms": None, + "read_timeout_ms": None, + "rate_limit_retries": None, } with mock.patch.dict("os.environ", environment_variables, clear=True): config_loader = EnvironmentVariablesConfigurationLoader() result = config_loader.load_config() assert expected_config == result + @pytest.mark.parametrize("key, config_name", + [("FBN_TOTAL_TIMEOUT_MS", "total_timeout_ms"), + ("FBN_CONNECT_TIMEOUT_MS", "connect_timeout_ms"), + ("FBN_READ_TIMEOUT_MS", "read_timeout_ms"), + ("FBN_RATE_LIMIT_RETRIES", "rate_limit_retries")]) + def test_load_config_throws_if_invalid_int_values(self, key, config_name): + environment_variables = { + key: "not-a-number" + } + with mock.patch.dict("os.environ", environment_variables, clear=True): + with pytest.raises(ValueError) as e: + config_loader = EnvironmentVariablesConfigurationLoader() + config_loader.load_config() + assert str(e.value) == f"invalid value for '{config_name}' - value must be an integer if set" class TestArgsConfigurationLoader: def test_load_config_gets_config_dict(self): @@ -155,6 +194,10 @@ def test_load_config_gets_config_dict(self): "proxy_address": "sample_address", "proxy_username": "sample_username", "proxy_password": "sample_password", + "total_timeout_ms": 3000, + "connect_timeout_ms": 2000, + "read_timeout_ms": 1000, + "rate_limit_retries": 5, } expected_config = { "token_url": "sample_tokenUrl", @@ -169,6 +212,10 @@ def test_load_config_gets_config_dict(self): "proxy_address": "sample_address", "proxy_username": "sample_username", "proxy_password": "sample_password", + "total_timeout_ms": 3000, + "connect_timeout_ms": 2000, + "read_timeout_ms": 1000, + "rate_limit_retries": 5, } config_loader = ArgsConfigurationLoader(**kwargs) result = config_loader.load_config() diff --git a/test_sdk/unit/test_configuration_options.py b/test_sdk/unit/test_configuration_options.py new file mode 100644 index 0000000..ee021cd --- /dev/null +++ b/test_sdk/unit/test_configuration_options.py @@ -0,0 +1,64 @@ +import pytest + +from TO_BE_REPLACED.extensions.configuration_options import ConfigurationOptions + + +def test_errors_if_invalid_total_timeout_type(): + opts = ConfigurationOptions() + with pytest.raises(TypeError) as e: + opts.total_timeout_ms = "not-an-int" + assert str(e.value) == "total_timeout_ms must be type int but type '' used" + +def test_errors_if_invalid_connect_timeout_type(): + opts = ConfigurationOptions() + with pytest.raises(TypeError) as e: + opts.connect_timeout_ms = "not-an-int" + assert str(e.value) == "connect_timeout_ms must be type int but type '' used" + +def test_errors_if_invalid_read_timeout_type(): + opts = ConfigurationOptions() + with pytest.raises(TypeError) as e: + opts.read_timeout_ms = "not-an-int" + assert str(e.value) == "read_timeout_ms must be type int but type '' used" + +def test_errors_if_invalid_rate_limit_retries_type(): + opts = ConfigurationOptions() + with pytest.raises(TypeError) as e: + opts.rate_limit_retries = "not-an-int" + assert str(e.value) == "rate_limit_retries must be type int but type '' used" + +def test_errors_if_invalid_total_timeout_value(): + opts = ConfigurationOptions() + with pytest.raises(ValueError) as e: + opts.total_timeout_ms = -1 + assert str(e.value) == "total_timeout_ms must be an integer greater than or equal to zero" + +def test_errors_if_invalid_connect_timeout_value(): + opts = ConfigurationOptions() + with pytest.raises(ValueError) as e: + opts.connect_timeout_ms = -1 + assert str(e.value) == "connect_timeout_ms must be an integer greater than or equal to zero" + +def test_errors_if_invalid_read_timeout_value(): + opts = ConfigurationOptions() + with pytest.raises(ValueError) as e: + opts.read_timeout_ms = -1 + assert str(e.value) == "read_timeout_ms must be an integer greater than or equal to zero" + +def test_errors_if_invalid_rate_limit_retries_value(): + opts = ConfigurationOptions() + with pytest.raises(ValueError) as e: + opts.rate_limit_retries = -1 + assert str(e.value) == "rate_limit_retries must be an integer greater than or equal to zero" + +def test_correctly_sets_values(): + opts = ConfigurationOptions( + total_timeout_ms=3001, + connect_timeout_ms=2001, + rate_limit_retries=1001, + read_timeout_ms=4 + ) + assert opts.total_timeout_ms == 3001 + assert opts.connect_timeout_ms == 2001 + assert opts.rate_limit_retries == 1001 + assert opts.read_timeout_ms == 4 \ No newline at end of file diff --git a/test_sdk/unit/test_retry.py b/test_sdk/unit/test_retry.py index 1d01f0c..edeaa2b 100644 --- a/test_sdk/unit/test_retry.py +++ b/test_sdk/unit/test_retry.py @@ -9,13 +9,33 @@ class TestSyncRetryRestWrapper(): + def test_errors_if_invalid_retries_type(self): + with pytest.raises(TypeError) as e: + RetryingRestWrapper(None, retries="not-an-int") + assert str(e.value) == "retries should be an int, found " + + def test_errors_if_invalid_rate_limit_retries_type(self): + with pytest.raises(TypeError) as e: + RetryingRestWrapper(None, rate_limit_retries="not-an-int") + assert str(e.value) == "rate_limit_retries should be an int, found " + + def test_errors_if_invalid_retries_value(self): + with pytest.raises(ValueError) as e: + RetryingRestWrapper(None, retries=-1) + assert str(e.value) == "retries should be greater than or equal to zero but was '-1'" + + def test_errors_if_invalid_rate_limit_retries_value(self): + with pytest.raises(ValueError) as e: + RetryingRestWrapper(None, rate_limit_retries=-1) + assert str(e.value) == "rate_limit_retries should be greater than or equal to zero but was '-1'" + def test_on_success_returns_result_and_makes_request_once(self): rest_object_mock = RESTClientObject(Configuration()) rest_object_mock.request = mock.MagicMock(return_value="OK") retry_object = RetryingRestWrapper(rest_object=rest_object_mock) result = retry_object.request("GET", "") assert result == "OK" - rest_object_mock.request.assert_called_once_with('GET', '', None, None, None, None, True, None) + rest_object_mock.request.assert_called_once_with('GET', '', None, None, None, None, True, None, None) def test_on_failure_retries_once(self): rest_object_mock = RESTClientObject(Configuration()) @@ -25,7 +45,7 @@ def test_on_failure_retries_once(self): retry_object = RetryingRestWrapper(rest_object=rest_object_mock, retries=1) result = retry_object.request("GET", "") assert result == "OK" - rest_object_mock.request.assert_called_with('GET', '', None, None, None, None, True, None) + rest_object_mock.request.assert_called_with('GET', '', None, None, None, None, True, None, None) assert rest_object_mock.request.call_count == 2 def test_on_repeated_failures_throws_ApiException(self): @@ -43,13 +63,33 @@ def test_on_repeated_failures_throws_ApiException(self): @pytest.mark.asyncio class TestASyncRetryRestWrapper(): + def test_errors_if_invalid_retries_type(self): + with pytest.raises(TypeError) as e: + RetryingRestWrapperAsync(None, retries="not-an-int") + assert str(e.value) == "retries should be an int, found " + + def test_errors_if_invalid_rate_limit_retries_type(self): + with pytest.raises(TypeError) as e: + RetryingRestWrapperAsync(None, rate_limit_retries="not-an-int") + assert str(e.value) == "rate_limit_retries should be an int, found " + + def test_errors_if_invalid_retries_value(self): + with pytest.raises(ValueError) as e: + RetryingRestWrapperAsync(None, retries=-1) + assert str(e.value) == "retries should be greater than or equal to zero but was '-1'" + + def test_errors_if_invalid_rate_limit_retries_value(self): + with pytest.raises(ValueError) as e: + RetryingRestWrapperAsync(None, rate_limit_retries=-1) + assert str(e.value) == "rate_limit_retries should be greater than or equal to zero but was '-1'" + async def test_on_success_returns_result_and_makes_request_once(self): rest_object_mock = AsyncRestClientObject(Configuration()) rest_object_mock.request = mock.AsyncMock(return_value="OK") retry_object = RetryingRestWrapperAsync(rest_object=rest_object_mock) result = await retry_object.request("GET", "") assert result == "OK" - rest_object_mock.request.assert_called_once_with('GET', '', None, None, None, None, True, None) + rest_object_mock.request.assert_called_once_with('GET', '', None, None, None, None, True, None, None) await rest_object_mock.close() async def test_on_failure_retries_once(self): @@ -60,7 +100,7 @@ async def test_on_failure_retries_once(self): retry_object = RetryingRestWrapperAsync(rest_object=rest_object_mock, retries=1) result = await retry_object.request("GET", "") assert result == "OK" - rest_object_mock.request.assert_called_with('GET', '', None, None, None, None, True, None) + rest_object_mock.request.assert_called_with('GET', '', None, None, None, None, True, None, None) assert rest_object_mock.request.call_count == 2 await rest_object_mock.close() diff --git a/test_sdk/unit/test_timeouts.py b/test_sdk/unit/test_timeouts.py new file mode 100644 index 0000000..5748e23 --- /dev/null +++ b/test_sdk/unit/test_timeouts.py @@ -0,0 +1,39 @@ +from TO_BE_REPLACED.configuration import Timeouts +import pytest + + +def test_timeouts_does_not_allow_incorrect_type_for_total(): + with pytest.raises(TypeError) as e: + Timeouts("not-an-int", 0, 0) + assert str(e.value) == "total_timeout_ms must be type int but type '' used" + +def test_timeouts_does_not_allow_incorrect_type_for_connect(): + with pytest.raises(TypeError) as e: + Timeouts(0, "not-an-int", 0) + assert str(e.value) == "connect_timeout_ms must be type int but type '' used" + +def test_timeouts_does_not_allow_incorrect_type_for_read(): + with pytest.raises(TypeError) as e: + Timeouts(0, 0, "not-an-int") + assert str(e.value) == "read_timeout_ms must be type int but type '' used" + +def test_timeouts_does_not_allow_incorrect_value_for_total(): + with pytest.raises(ValueError) as e: + Timeouts(-1, 0, 0) + assert str(e.value) == "total_timeout_ms must be an integer greater than or equal to zero" + +def test_timeouts_does_not_allow_incorrect_value_for_connect(): + with pytest.raises(ValueError) as e: + Timeouts(0, -1, 0) + assert str(e.value) == "connect_timeout_ms must be an integer greater than or equal to zero" + +def test_timeouts_does_not_allow_incorrect_value_for_read(): + with pytest.raises(ValueError) as e: + Timeouts(0, 0, -1) + assert str(e.value) == "read_timeout_ms must be an integer greater than or equal to zero" + +def test_timeouts_set_correctly(): + timeouts = Timeouts(3000, 2000, 1000) + assert timeouts.total_timeout_ms == 3000 + assert timeouts.connect_timeout_ms == 2000 + assert timeouts.read_timeout_ms == 1000 \ No newline at end of file