From 9d1c84313fdfa4baa7fa19834641e6cb539036c5 Mon Sep 17 00:00:00 2001 From: Sono Chhibber Date: Thu, 15 Aug 2019 09:44:29 -0500 Subject: [PATCH 1/9] [Resolves #783] add random jitter to boto retry (#803) --- sceptre/connection_manager.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/sceptre/connection_manager.py b/sceptre/connection_manager.py index c5c35eed1..ffe78fbfe 100644 --- a/sceptre/connection_manager.py +++ b/sceptre/connection_manager.py @@ -9,6 +9,7 @@ import functools import logging +import random import threading import time import boto3 @@ -24,8 +25,11 @@ def _retry_boto_call(func): """ Retries a Boto3 call up to 30 times if request rate limits are hit. - The time waited between retries increases linearly. If rate limits are - hit 30 times, _retry_boto_call raises a + Between each try we wait a random amount with a max of that time being 45 seconds. + Specifically we are picking number between a ceiling (delay_cap) of 45 seconds and the last + delay multiplied by 2.5, rounding to two decimal places. You can read more + here: https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + If rate limits are hit 30 times, _retry_boto_call raises a sceptre.exceptions.RetryLimitExceededException. :param func: A function that uses boto calls @@ -40,13 +44,21 @@ def _retry_boto_call(func): def decorated(*args, **kwargs): max_retries = 30 attempts = 1 + mdelay = 1 + delay_cap = 45 while attempts < max_retries: try: return func(*args, **kwargs) except ClientError as e: if e.response["Error"]["Code"] == "Throttling": - logger.error("Request limit exceeded, pausing...") - time.sleep(attempts) + logger.error("Request limit exceeded, pausing {}...".format(mdelay)) + time.sleep(mdelay) + + # Using De-correlated Jitter Algorithm + # We are picking number between a ceiling (delay_cap) of 45 seconds and the last + # delay multiplied by 2.5, rounding to two decimal places. + mdelay = min(delay_cap, round((random.uniform(1, mdelay * 2.5)), 2)) + attempts += 1 else: raise From 6837e6df23e793721e931e35a95ef05f163c0a92 Mon Sep 17 00:00:00 2001 From: Niall Grant Date: Fri, 16 Aug 2019 10:00:51 +0100 Subject: [PATCH 2/9] Fix lint issue (#806) --- sceptre/connection_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sceptre/connection_manager.py b/sceptre/connection_manager.py index ffe78fbfe..6b143ece4 100644 --- a/sceptre/connection_manager.py +++ b/sceptre/connection_manager.py @@ -55,8 +55,8 @@ def decorated(*args, **kwargs): time.sleep(mdelay) # Using De-correlated Jitter Algorithm - # We are picking number between a ceiling (delay_cap) of 45 seconds and the last - # delay multiplied by 2.5, rounding to two decimal places. + # We are picking number between a ceiling (delay_cap) of 45 seconds and the + # last delay multiplied by 2.5, rounding to two decimal places. mdelay = min(delay_cap, round((random.uniform(1, mdelay * 2.5)), 2)) attempts += 1 From 6d26461648750c83d06b2e9030a49c2fc7547e47 Mon Sep 17 00:00:00 2001 From: Niall Grant Date: Fri, 16 Aug 2019 11:35:23 +0100 Subject: [PATCH 3/9] Update CHANGELOG --- CHANGELOG.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5575a9c57..ad5e09ba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,28 @@ Categories: Added, Removed, Changed, Fixed, Nonfunctional, Deprecated ## Unreleased -- Fix recursion in sceptre_user_data becoming infinite +## 2.2.0 (2019.08.16) + +### Added + +- Meaningful Jinja Template exception output handling + +### Fixed + +- Recursion in `sceptre_user_data` becoming infinite +- AWS `rate_exceeded` error when hitting AWS API too frequently +- Readme links +- StackGraph debugging output + +### Nonfunctional + +- Moved CircleCI Dockerfile to its own repository +- Add SonarQube analysis +- Change CircleCI setup to use context over environment variables +- Removed redundant test code +- Add badges to README +- Adjust Coverage fail-level to 92% +- Update CONTRIBUTING Guide ## 2.1.5 (2019.06.28) From 14e78951d8033cc3912b72a7b92e972d4c8a6291 Mon Sep 17 00:00:00 2001 From: Niall Grant Date: Fri, 16 Aug 2019 11:36:11 +0100 Subject: [PATCH 4/9] =?UTF-8?q?Bump=20version:=202.1.5=20=E2=86=92=202.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sceptre/__init__.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sceptre/__init__.py b/sceptre/__init__.py index caf82efe6..82b07a4d2 100644 --- a/sceptre/__init__.py +++ b/sceptre/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Cloudreach' __email__ = 'sceptre@cloudreach.com' -__version__ = '2.1.5' +__version__ = '2.2.0' # Set up logging to ``/dev/null`` like a library is supposed to. diff --git a/setup.cfg b/setup.cfg index 6750f5f21..d5e250e63 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.5 +current_version = 2.2.0 parse = (?P\d+)\.(?P\d+)\.(?P\d+)|(?P.*) commit = True tag = True From 12fa919a1642095a74f143d7d3601cb9a421f9ab Mon Sep 17 00:00:00 2001 From: Niall Grant Date: Mon, 19 Aug 2019 09:39:54 +0100 Subject: [PATCH 5/9] Add `typing` to prod requirements (#809) [Resolves #808] --- requirements/prod.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/prod.txt b/requirements/prod.txt index 3e2dab356..3e1894eb7 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -4,3 +4,4 @@ colorama==0.3.9 Jinja2>=2.8,<3 networkx==2.1 PyYaml>=5.1,<6.0 +typing>=3.7,<3.8 From dc0b4663b84b62caf6b0e08084bf638112b2524a Mon Sep 17 00:00:00 2001 From: Niall Grant Date: Mon, 19 Aug 2019 12:00:07 +0100 Subject: [PATCH 6/9] Add `typing` lib as a install dependency (#811) --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f1da85ff..57fc7a5d8 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,8 @@ "colorama==0.3.9", "packaging==16.8", "six>=1.11.0,<2.0.0", - "networkx==2.1" + "networkx==2.1", + "typing>=3.7.0,<3.8.0" ] test_requirements = [ From 6cef58b0d729437ad6de8e9abdb619e24fd1191f Mon Sep 17 00:00:00 2001 From: Simon Eilting <52410031+seiltingcr@users.noreply.github.com> Date: Mon, 19 Aug 2019 14:30:40 +0100 Subject: [PATCH 7/9] Make ResolvableProperty somewhat thread-safe. (#812) --- sceptre/resolvers/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index 897ace858..846ee2076 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -2,6 +2,7 @@ import abc import logging from contextlib import contextmanager +from threading import RLock import six from sceptre.helpers import _call_func_on_values @@ -64,6 +65,7 @@ def __init__(self, name): self.name = "_" + name self.logger = logging.getLogger(__name__) self._get_in_progress = False + self._lock = RLock() def __get__(self, instance, type): """ @@ -73,7 +75,7 @@ def __get__(self, instance, type): :return: The attribute stored with the suffix ``name`` in the instance. :rtype: dict or list """ - with self._no_recursive_get(): + with self._lock, self._no_recursive_get(): def resolve(attr, key, value): try: attr[key] = value.resolve() @@ -97,8 +99,9 @@ def setup(attr, key, value): value.stack = instance value.setup() - _call_func_on_values(setup, value, Resolver) - setattr(instance, self.name, value) + with self._lock: + _call_func_on_values(setup, value, Resolver) + setattr(instance, self.name, value) class ResolveLater(object): """Represents a value that could not yet be resolved but can be resolved in the future.""" From 29643e7461aa1e344ad02cedcf702c450d43f41e Mon Sep 17 00:00:00 2001 From: Niall Grant Date: Mon, 19 Aug 2019 16:58:37 +0100 Subject: [PATCH 8/9] Update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad5e09ba1..04bc9605e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ Categories: Added, Removed, Changed, Fixed, Nonfunctional, Deprecated ## Unreleased +## 2.2.1 (2019.08.19) + +### Fixed + +- `typing` install dependency for Python version < 3.5 +- Race condition in stacks causing RecurrsiveGet exception + ## 2.2.0 (2019.08.16) ### Added From b7585fd7bf9607230f9d4324166b49cd59cab3ee Mon Sep 17 00:00:00 2001 From: Niall Grant Date: Mon, 19 Aug 2019 16:59:08 +0100 Subject: [PATCH 9/9] =?UTF-8?q?Bump=20version:=202.2.0=20=E2=86=92=202.2.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sceptre/__init__.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sceptre/__init__.py b/sceptre/__init__.py index 82b07a4d2..99b6b0a81 100644 --- a/sceptre/__init__.py +++ b/sceptre/__init__.py @@ -6,7 +6,7 @@ __author__ = 'Cloudreach' __email__ = 'sceptre@cloudreach.com' -__version__ = '2.2.0' +__version__ = '2.2.1' # Set up logging to ``/dev/null`` like a library is supposed to. diff --git a/setup.cfg b/setup.cfg index d5e250e63..b6f761ad9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.2.0 +current_version = 2.2.1 parse = (?P\d+)\.(?P\d+)\.(?P\d+)|(?P.*) commit = True tag = True