Skip to content

Commit

Permalink
CTOOLS-348: allow timeout and rate limit retry configuration
Browse files Browse the repository at this point in the history
  • Loading branch information
@cg-finbourne
cg-finbourne committed Sep 21, 2024
1 parent e82328e commit 97dcf43
Show file tree
Hide file tree
Showing 18 changed files with 1,524 additions and 150 deletions.
3 changes: 3 additions & 0 deletions generate/config-template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@
},
"extensions/file_access_token.mustache": {
"destinationFilename": "${PACKAGE_NAME}/extensions/file_access_token.py"
},
"extensions/configuration_options.mustache": {
"destinationFilename": "${PACKAGE_NAME}/extensions/configuration_options.py"
}
}
}
19 changes: 10 additions & 9 deletions generate/templates/api.mustache
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ from {{packageName}}.exceptions import ( # noqa: F401
ApiTypeError,
ApiValueError
)
from {{packageName}}.extensions.configuration_options import ConfigurationOptions


{{#operations}}
Expand Down Expand Up @@ -66,10 +67,9 @@ class {{classname}}:
{{/allParams}}
:param async_req: Whether to execute the request asynchronously.
:type async_req: bool, optional
:param _request_timeout: timeout setting for this request.
If one number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:param _request_timeout: Timeout setting. Do not use - use the opts parameter instead
:param opts: Configuration options for this request
:type opts: ConfigurationOptions, optional
:return: Returns the result object.
If the method is called asynchronously,
returns the request thread.
Expand Down Expand Up @@ -112,10 +112,9 @@ class {{classname}}:
:param _return_http_data_only: response data instead of ApiResponse
object with status code, headers, etc
:type _return_http_data_only: bool, optional
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:param _request_timeout: Timeout setting. Do not use - use the opts parameter instead
:param opts: Configuration options for this request
:type opts: ConfigurationOptions, optional
:param _request_auth: set to override the auth_settings for an a single
request; this effectively ignores the authentication
in the spec for a single request.
Expand Down Expand Up @@ -162,7 +161,8 @@ class {{classname}}:
'_request_timeout',
'_request_auth',
'_content_type',
'_headers'
'_headers',
'opts'
]
)

Expand Down Expand Up @@ -304,6 +304,7 @@ class {{classname}}:
_return_http_data_only=_params.get('_return_http_data_only'), # noqa: E501
_preload_content=_params.get('_preload_content', True),
_request_timeout=_params.get('_request_timeout'),
opts=_params.get('opts'),
{{#servers.0}}
_host=_host,
{{/servers.0}}
Expand Down
43 changes: 26 additions & 17 deletions generate/templates/api_client.mustache
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ class ApiClient:
files=None, response_types_map=None, auth_settings=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None,
_request_auth=None):
_request_auth=None, opts=None):

config = self.configuration

Expand Down Expand Up @@ -230,7 +230,8 @@ class ApiClient:
headers=header_params,
post_params=post_params, body=body,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
_request_timeout=_request_timeout,
opts=opts)
except ApiException as e:
if e.body:
e.body = e.body.decode('utf-8')
Expand Down Expand Up @@ -390,7 +391,8 @@ class ApiClient:
response_types_map=None, auth_settings=None,
async_req=None, _return_http_data_only=None,
collection_formats=None, _preload_content=True,
_request_timeout=None, _host=None, _request_auth=None):
_request_timeout=None, _host=None, _request_auth=None,
opts=None):
"""Makes the HTTP request (synchronous) and returns deserialized data.

To make an async_req request, set the async_req parameter.
Expand All @@ -417,10 +419,9 @@ class ApiClient:
Default is True.
:param collection_formats: dict of collection formats for path, query,
header, and post parameters.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:param _request_timeout: Timeout setting. Do not use - use the opts parameter instead
:param opts: Configuration options for this request
:type opts: ConfigurationOptions, optional
:param _request_auth: set to override the auth_settings for an a single
request; this effectively ignores the authentication
in the spec for a single request.
Expand All @@ -439,7 +440,7 @@ class ApiClient:
response_types_map, auth_settings,
_return_http_data_only, collection_formats,
_preload_content, _request_timeout, _host,
_request_auth)
_request_auth, opts)

return self.pool.apply_async(self.__call_api, (resource_path,
method, path_params,
Expand All @@ -452,61 +453,69 @@ class ApiClient:
collection_formats,
_preload_content,
_request_timeout,
_host, _request_auth))
_host, _request_auth,
opts))

def request(self, method, url, query_params=None, headers=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
_request_timeout=None, opts=None):
"""Makes the HTTP request using RESTClient."""
if method == "GET":
return self.rest_client.get_request(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
headers=headers,
opts=opts)
elif method == "HEAD":
return self.rest_client.head_request(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
headers=headers,
opts=opts)
elif method == "OPTIONS":
return self.rest_client.options_request(url,
query_params=query_params,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
_request_timeout=_request_timeout,
opts=opts)
elif method == "POST":
return self.rest_client.post_request(url,
query_params=query_params,
headers=headers,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
body=body,
opts=opts)
elif method == "PUT":
return self.rest_client.put_request(url,
query_params=query_params,
headers=headers,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
body=body,
opts=opts)
elif method == "PATCH":
return self.rest_client.patch_request(url,
query_params=query_params,
headers=headers,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
body=body,
opts=opts)
elif method == "DELETE":
return self.rest_client.delete_request(url,
query_params=query_params,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
body=body,
opts=opts)
else:
raise ApiValueError(
"http method must be `GET`, `HEAD`, `OPTIONS`,"
Expand Down
90 changes: 70 additions & 20 deletions generate/templates/asyncio/rest.mustache
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ class RESTClientObject:

self.proxy = configuration.proxy
self.proxy_headers = configuration.proxy_headers
self.timeout = aiohttp.ClientTimeout(
total=configuration.timeouts.total_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.total_timeout_ms != None else None,
connect=configuration.timeouts.connect_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.connect_timeout_ms != None else None,
sock_read=configuration.timeouts.read_timeout_ms / 1000.0 if configuration.timeouts != None and configuration.timeouts.read_timeout_ms != None else None,
)

# https pool manager
self.pool_manager = aiohttp.ClientSession(
Expand All @@ -70,7 +75,7 @@ class RESTClientObject:

async def request(self, method, url, query_params=None, headers=None,
body=None, post_params=None, _preload_content=True,
_request_timeout=None):
_request_timeout=None, opts=None):
"""Execute request

:param method: http request method
Expand All @@ -83,10 +88,9 @@ class RESTClientObject:
and `multipart/form-data`
:param _preload_content: this is a non-applicable field for
the AiohttpClient.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:param _request_timeout: Timeout setting. Do not use - use the opts parameter instead
:param opts: Configuration options for this request
:type opts: ConfigurationOptions, optional
"""
method = method.upper()
assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT',
Expand All @@ -102,7 +106,44 @@ class RESTClientObject:
# url already contains the URL query string
# so reset query_params to empty dict
query_params = {}
timeout = _request_timeout or 5 * 60

# _request_timeout param cannot be removed for backwards compatability
# values from opts override values from _request_timeout
# try to get values from opts first, then _request_timeout, then self.timeout, else set to None
# timeout = _request_timeout or self.timeout
timeout = None
opts_total_timeout = opts.total_timeout_ms / 1000.0 if opts and opts.total_timeout_ms != None else None
opts_connect_timeout = opts.connect_timeout_ms / 1000.0 if opts and opts.connect_timeout_ms != None else None
opts_read_timeout = opts.read_timeout_ms / 1000.0 if opts and opts.read_timeout_ms != None else None
if not _request_timeout:
timeout = aiohttp.ClientTimeout(
total=opts_total_timeout if opts_total_timeout != None
else self.timeout.total,
connect=opts_connect_timeout if opts_connect_timeout != None
else self.timeout.connect,
sock_read=opts_read_timeout if opts_read_timeout != None
else self.timeout.sock_read)
elif isinstance(_request_timeout, aiohttp.ClientTimeout):
timeout = aiohttp.ClientTimeout(
total=opts_total_timeout if opts_total_timeout != None
else _request_timeout.total if _request_timeout.total != None
else self.timeout.total,
connect=opts_connect_timeout if opts_connect_timeout != None
else _request_timeout.connect if _request_timeout.connect != None
else self.timeout.connect,
sock_read=opts_read_timeout if opts_read_timeout != None
else _request_timeout.sock_read if _request_timeout.sock_read != None
else self.timeout.sock_read)
elif isinstance(_request_timeout, (int, float)):
timeout = aiohttp.ClientTimeout(
total=opts_total_timeout if opts_total_timeout != None
else _request_timeout,
connect=opts_connect_timeout if opts_connect_timeout != None
else self.timeout.connect,
sock_read=opts_read_timeout if opts_read_timeout != None
else self.timeout.sock_read)
else:
raise f"unexpected type '{type(_request_timeout)}' for _request_timeout"

if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
Expand Down Expand Up @@ -176,69 +217,78 @@ class RESTClientObject:
return r

async def get_request(self, url, headers=None, query_params=None,
_preload_content=True, _request_timeout=None):
_preload_content=True, _request_timeout=None, opts=None):
return (await self.request("GET", url,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
query_params=query_params))
query_params=query_params,
opts=opts))

async def head_request(self, url, headers=None, query_params=None,
_preload_content=True, _request_timeout=None):
_preload_content=True, _request_timeout=None, opts=None):
return (await self.request("HEAD", url,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
query_params=query_params))
query_params=query_params,
opts=opts))

async def options_request(self, url, headers=None, query_params=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
_request_timeout=None, opts=None):
return (await self.request("OPTIONS", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body))
body=body,
opts=opts))

async def delete_request(self, url, headers=None, query_params=None, body=None,
_preload_content=True, _request_timeout=None):
_preload_content=True, _request_timeout=None,
opts=None):
return (await self.request("DELETE", url,
headers=headers,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body))
body=body,
opts=opts))

async def post_request(self, url, headers=None, query_params=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
_request_timeout=None, opts=None):
return (await self.request("POST", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body))
body=body,
opts=opts))

async def put_request(self, url, headers=None, query_params=None, post_params=None,
body=None, _preload_content=True, _request_timeout=None):
body=None, _preload_content=True, _request_timeout=None,
opts=None):
return (await self.request("PUT", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body))
body=body,
opts=opts))

async def patch_request(self, url, headers=None, query_params=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
_request_timeout=None, opts=None):
return (await self.request("PATCH", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body))
body=body,
opts=opts))
Loading

0 comments on commit 97dcf43

Please sign in to comment.