From 78c51606323ac2a0f8dbe483a4be7da48aa534b5 Mon Sep 17 00:00:00 2001 From: Rob Reus Date: Wed, 1 Jul 2020 18:26:53 +0200 Subject: [PATCH 1/4] [Resolves #359] Implements use previous paramater feature --- docs/_source/docs/stack_config.rst | 20 +++++++ docs/_source/docs/templates.rst | 2 +- sceptre/exceptions.py | 7 +++ sceptre/plan/actions.py | 30 +++++++--- tests/test_actions.py | 88 +++++++++++++++++++++++++++++- 5 files changed, 138 insertions(+), 9 deletions(-) diff --git a/docs/_source/docs/stack_config.rst b/docs/_source/docs/stack_config.rst index 518e4390b..65150da67 100644 --- a/docs/_source/docs/stack_config.rst +++ b/docs/_source/docs/stack_config.rst @@ -86,6 +86,13 @@ values/resolvers. Lists of values/resolvers will be formatted into an AWS compatible comma separated string e.g. \ ``value1,value2,value3``. Lists can contain a mixture of values and resolvers. +A parameter can also be configured to use the previous value. You can do so by +making the value a dictionary. The values supported in the dictionary are +``initial_value`` and ``use_previous_value``. When creating stacks, setting +``initial_value`` is required, but can be left out for stack updates. The value +set at ``initial_value`` value will only be used during creation, or when +setting ``use_previous_value`` to false. + Syntax: .. code-block:: yaml @@ -102,6 +109,14 @@ Syntax: : - ! - "value1" + : + initial_value: "value" + use_previous_value: + : + initial_value: + - "value1" + - ! + use_previous_value: Example: @@ -117,6 +132,11 @@ Example: - "sg-12345678" - !stack_output security-groups::BaseSecurityGroupId - !file_contents /file/with/security_group_id.txt + security_group_whitelist: + initial_value: + - "127.0.0.0/24" + - "127.0.1.0/24" + use_previous_value: true protected ~~~~~~~~~ diff --git a/docs/_source/docs/templates.rst b/docs/_source/docs/templates.rst index a53aaa4fd..3dd6db7dc 100644 --- a/docs/_source/docs/templates.rst +++ b/docs/_source/docs/templates.rst @@ -72,7 +72,7 @@ Stack: Template `dns-extras.j2`: -.. code-block:: yaml +.. code-block:: jinja AWSTemplateFormatVersion: '2010-09-09' Description: 'Add Route53 - CNAME and ALIAS records' diff --git a/sceptre/exceptions.py b/sceptre/exceptions.py index 75438d9e0..d6eccc5e3 100644 --- a/sceptre/exceptions.py +++ b/sceptre/exceptions.py @@ -166,3 +166,10 @@ class InvalidAWSCredentialsError(SceptreException): Error raised when AWS credentials are invalid. """ pass + + +class InvalidParameterError(SceptreException): + """ + Error raised when parameters are invalid. + """ + pass diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 09a22dd29..20f90899a 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -27,6 +27,7 @@ from sceptre.exceptions import UnknownStackChangeSetStatusError from sceptre.exceptions import StackDoesNotExistError from sceptre.exceptions import ProtectedStackError +from sceptre.exceptions import InvalidParameterError class StackActions(object): @@ -59,7 +60,7 @@ def create(self): self.logger.info("%s - Creating Stack", self.stack.name) create_stack_kwargs = { "StackName": self.stack.external_name, - "Parameters": self._format_parameters(self.stack.parameters), + "Parameters": self._format_parameters(self.stack.parameters, create=True), "Capabilities": ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND'], "NotificationARNs": self.stack.notifications, "Tags": [ @@ -640,7 +641,7 @@ def get_status(self): except StackDoesNotExistError: return "PENDING" - def _format_parameters(self, parameters): + def _format_parameters(self, parameters, create=False): """ Converts CloudFormation parameters to the format used by Boto3. @@ -653,12 +654,27 @@ def _format_parameters(self, parameters): for name, value in parameters.items(): if value is None: continue + formatted_parameter = dict(ParameterKey=name) if isinstance(value, list): - value = ",".join(value) - formatted_parameters.append({ - "ParameterKey": name, - "ParameterValue": value - }) + formatted_parameter['ParameterValue'] = ",".join(value) + elif isinstance(value, dict): + initial_value = value.get('initial_value') + use_previous_value = value.get('use_previous_value', False) + if not isinstance(use_previous_value, bool): + raise InvalidParameterError("'use_previous_value' must be a boolean") + if (create is True or use_previous_value is False) and initial_value is None: + raise InvalidParameterError("'initial_value' is required when creating a new " + "stack or when 'use_previous_value' is false") + if create is True or use_previous_value is False: + if isinstance(initial_value, list): + formatted_parameter['ParameterValue'] = ",".join(initial_value) + else: + formatted_parameter['ParameterValue'] = value.get('initial_value') + else: + formatted_parameter['UsePreviousValue'] = use_previous_value + else: + formatted_parameter['ParameterValue'] = value + formatted_parameters.append(formatted_parameter) return formatted_parameters diff --git a/tests/test_actions.py b/tests/test_actions.py index c6c689f5b..3e47f47eb 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -18,6 +18,7 @@ from sceptre.exceptions import UnknownStackChangeSetStatusError from sceptre.exceptions import StackDoesNotExistError from sceptre.exceptions import ProtectedStackError +from sceptre.exceptions import InvalidParameterError class TestStackActions(object): @@ -703,7 +704,7 @@ def test_unlock_calls_set_stack_policy_with_policy( "tests/fixtures/stack_policies/unlock.json" ) - def test_format_parameters_with_sting_values(self): + def test_format_parameters_with_string_values(self): parameters = { "key1": "value1", "key2": "value2", @@ -733,6 +734,20 @@ def test_format_parameters_with_none_values(self): ) assert sorted_formatted_parameters == [] + def test_format_parameters_with_empty_dict_value(self): + parameter = { + "key": dict() + } + with pytest.raises(InvalidParameterError): + self.actions._format_parameters(parameter) + + def test_format_parameters_with_non_bool_previous_value(self): + parameter = { + "key": dict(use_previous_value='fosho') + } + with pytest.raises(InvalidParameterError): + self.actions._format_parameters(parameter) + def test_format_parameters_with_none_and_string_values(self): parameters = { "key1": "value1", @@ -815,6 +830,77 @@ def test_format_parameters_with_none_list_and_string_values(self): {"ParameterKey": "key2", "ParameterValue": "value4"}, ] + def test_format_parameters_with_string_and_dict_values(self): + parameters = { + "key1": "value1", + "key2": {"initial_value": "value2"} + } + formatted_parameters = self.actions._format_parameters(parameters) + sorted_formatted_parameters = sorted( + formatted_parameters, + key=lambda x: x["ParameterKey"] + ) + assert sorted_formatted_parameters == [ + {"ParameterKey": "key1", "ParameterValue": "value1"}, + {"ParameterKey": "key2", "ParameterValue": "value2"} + ] + + def test_format_parameters_with_dict_string_and_list_values(self): + parameters = { + "key1": {"initial_value": ["value1", "value2"]}, + "key2": {"initial_value": "value3"} + } + formatted_parameters = self.actions._format_parameters(parameters) + sorted_formatted_parameters = sorted( + formatted_parameters, + key=lambda x: x["ParameterKey"] + ) + assert sorted_formatted_parameters == [ + {"ParameterKey": "key1", "ParameterValue": "value1,value2"}, + {"ParameterKey": "key2", "ParameterValue": "value3"} + ] + + def test_format_parameters_with_string_and_previous_values(self): + parameters = { + "key1": "value1", + "key2": {"use_previous_value": True}, + "key3": {"initial_value": "value3", "use_previous_value": True} + } + formatted_parameters = self.actions._format_parameters(parameters) + sorted_formatted_parameters = sorted( + formatted_parameters, + key=lambda x: x["ParameterKey"] + ) + assert sorted_formatted_parameters == [ + {"ParameterKey": "key1", "ParameterValue": "value1"}, + {"ParameterKey": "key2", "UsePreviousValue": True}, + {"ParameterKey": "key3", "UsePreviousValue": True} + ] + + def test_format_parameters_with_create_and_previous_without_initial_values(self): + parameters = { + "key1": "value1", + "key2": {"use_previous_value": True}, + "key3": {"initial_value": "value3", "use_previous_value": True} + } + with pytest.raises(InvalidParameterError): + self.actions._format_parameters(parameters, create=True) + + def test_format_parameters_with_create_and_previous_values(self): + parameters = { + "key1": "value1", + "key2": {"initial_value": "value2", "use_previous_value": True} + } + formatted_parameters = self.actions._format_parameters(parameters, create=True) + sorted_formatted_parameters = sorted( + formatted_parameters, + key=lambda x: x["ParameterKey"] + ) + assert sorted_formatted_parameters == [ + {"ParameterKey": "key1", "ParameterValue": "value1"}, + {"ParameterKey": "key2", "ParameterValue": "value2"} + ] + @patch("sceptre.plan.actions.StackActions._describe") def test_get_status_with_created_stack(self, mock_describe): mock_describe.return_value = { From c24c976871c693c8b6ddb1fbde7c72baba25afac Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sun, 16 Jun 2024 11:46:38 +1000 Subject: [PATCH 2/4] pre-commit --- sceptre/exceptions.py | 1 + sceptre/plan/actions.py | 33 ++++++++++++++----------------- tests/test_actions.py | 43 +++++++++++++++-------------------------- 3 files changed, 31 insertions(+), 46 deletions(-) diff --git a/sceptre/exceptions.py b/sceptre/exceptions.py index cdcf947e2..6ea076c08 100644 --- a/sceptre/exceptions.py +++ b/sceptre/exceptions.py @@ -195,6 +195,7 @@ class InvalidParameterError(SceptreException): """ Error raised when parameters are invalid. """ + pass diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 5726c2133..93da163ba 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -20,20 +20,13 @@ from sceptre.connection_manager import ConnectionManager -<<<<<<< HEAD -from sceptre.exceptions import CannotUpdateFailedStackError -from sceptre.exceptions import UnknownStackStatusError -from sceptre.exceptions import UnknownStackChangeSetStatusError -from sceptre.exceptions import StackDoesNotExistError -from sceptre.exceptions import ProtectedStackError -from sceptre.exceptions import InvalidParameterError -======= from sceptre.exceptions import ( CannotUpdateFailedStackError, ProtectedStackError, StackDoesNotExistError, UnknownStackChangeSetStatusError, UnknownStackStatusError, + InvalidParameterError, ) from sceptre.helpers import extract_datetime_from_aws_response_headers from sceptre.hooks import add_stack_hooks, add_stack_hooks_with_aliases @@ -44,7 +37,6 @@ if typing.TYPE_CHECKING: from sceptre.diffing.stack_differ import StackDiff, StackDiffer ->>>>>>> master class StackActions: @@ -80,17 +72,12 @@ def create(self): self.logger.info("%s - Creating Stack", self.stack.name) create_stack_kwargs = { "StackName": self.stack.external_name, -<<<<<<< HEAD "Parameters": self._format_parameters(self.stack.parameters, create=True), - "Capabilities": ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND'], -======= - "Parameters": self._format_parameters(self.stack.parameters), "Capabilities": [ "CAPABILITY_IAM", "CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND", ], ->>>>>>> master "NotificationARNs": self.stack.notifications, "Tags": [ {"Key": str(k), "Value": str(v)} for k, v in self.stack.tags.items() @@ -726,15 +713,23 @@ def _format_parameters(self, parameters, create=False): initial_value = value.get("initial_value") use_previous_value = value.get("use_previous_value", False) if not isinstance(use_previous_value, bool): - raise InvalidParameterError("'use_previous_value' must be a boolean") - if (create is True or use_previous_value is False) and initial_value is None: - raise InvalidParameterError("'initial_value' is required when creating a new " - "stack or when 'use_previous_value' is false") + raise InvalidParameterError( + "'use_previous_value' must be a boolean" + ) + if ( + create is True or use_previous_value is False + ) and initial_value is None: + raise InvalidParameterError( + "'initial_value' is required when creating a new " + "stack or when 'use_previous_value' is false" + ) if create is True or use_previous_value is False: if isinstance(initial_value, list): formatted_parameter["ParameterValue"] = ",".join(initial_value) else: - formatted_parameter["ParameterValue"] = value.get("initial_value") + formatted_parameter["ParameterValue"] = value.get( + "initial_value" + ) else: formatted_parameter["UsePreviousValue"] = use_previous_value else: diff --git a/tests/test_actions.py b/tests/test_actions.py index 78ca36554..81caae5e0 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -14,7 +14,7 @@ StackDoesNotExistError, UnknownStackChangeSetStatusError, UnknownStackStatusError, - InvalidParameterError + InvalidParameterError, ) from sceptre.plan.actions import StackActions from sceptre.stack import Stack @@ -821,16 +821,12 @@ def test_format_parameters_with_none_values(self): assert sorted_formatted_parameters == [] def test_format_parameters_with_empty_dict_value(self): - parameter = { - "key": dict() - } + parameter = {"key": dict()} with pytest.raises(InvalidParameterError): self.actions._format_parameters(parameter) def test_format_parameters_with_non_bool_previous_value(self): - parameter = { - "key": dict(use_previous_value='fosho') - } + parameter = {"key": dict(use_previous_value="fosho")} with pytest.raises(InvalidParameterError): self.actions._format_parameters(parameter) @@ -908,57 +904,51 @@ def test_format_parameters_with_none_list_and_string_values(self): ] def test_format_parameters_with_string_and_dict_values(self): - parameters = { - "key1": "value1", - "key2": {"initial_value": "value2"} - } + parameters = {"key1": "value1", "key2": {"initial_value": "value2"}} formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1"}, - {"ParameterKey": "key2", "ParameterValue": "value2"} + {"ParameterKey": "key2", "ParameterValue": "value2"}, ] def test_format_parameters_with_dict_string_and_list_values(self): parameters = { "key1": {"initial_value": ["value1", "value2"]}, - "key2": {"initial_value": "value3"} + "key2": {"initial_value": "value3"}, } formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1,value2"}, - {"ParameterKey": "key2", "ParameterValue": "value3"} + {"ParameterKey": "key2", "ParameterValue": "value3"}, ] def test_format_parameters_with_string_and_previous_values(self): parameters = { "key1": "value1", "key2": {"use_previous_value": True}, - "key3": {"initial_value": "value3", "use_previous_value": True} + "key3": {"initial_value": "value3", "use_previous_value": True}, } formatted_parameters = self.actions._format_parameters(parameters) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1"}, {"ParameterKey": "key2", "UsePreviousValue": True}, - {"ParameterKey": "key3", "UsePreviousValue": True} + {"ParameterKey": "key3", "UsePreviousValue": True}, ] def test_format_parameters_with_create_and_previous_without_initial_values(self): parameters = { "key1": "value1", "key2": {"use_previous_value": True}, - "key3": {"initial_value": "value3", "use_previous_value": True} + "key3": {"initial_value": "value3", "use_previous_value": True}, } with pytest.raises(InvalidParameterError): self.actions._format_parameters(parameters, create=True) @@ -966,16 +956,15 @@ def test_format_parameters_with_create_and_previous_without_initial_values(self) def test_format_parameters_with_create_and_previous_values(self): parameters = { "key1": "value1", - "key2": {"initial_value": "value2", "use_previous_value": True} + "key2": {"initial_value": "value2", "use_previous_value": True}, } formatted_parameters = self.actions._format_parameters(parameters, create=True) sorted_formatted_parameters = sorted( - formatted_parameters, - key=lambda x: x["ParameterKey"] + formatted_parameters, key=lambda x: x["ParameterKey"] ) assert sorted_formatted_parameters == [ {"ParameterKey": "key1", "ParameterValue": "value1"}, - {"ParameterKey": "key2", "ParameterValue": "value2"} + {"ParameterKey": "key2", "ParameterValue": "value2"}, ] @patch("sceptre.plan.actions.StackActions._describe") From d2b9dd55b47dd7d714100e0f1c4ed2bb853fdc1c Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sun, 16 Jun 2024 12:26:18 +1000 Subject: [PATCH 3/4] clean-up --- sceptre/plan/actions.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 93da163ba..3140a6568 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -699,6 +699,9 @@ def _format_parameters(self, parameters, create=False): :param parameters: A dictionary of parameters. :type parameters: dict + :param create: Flags if this is a stack create or update operation. + :type parameters: bool + :returns: A list of the formatted parameters. :rtype: list """ @@ -706,7 +709,9 @@ def _format_parameters(self, parameters, create=False): for name, value in parameters.items(): if value is None: continue + formatted_parameter = dict(ParameterKey=name) + if isinstance(value, list): formatted_parameter["ParameterValue"] = ",".join(value) elif isinstance(value, dict): @@ -716,14 +721,12 @@ def _format_parameters(self, parameters, create=False): raise InvalidParameterError( "'use_previous_value' must be a boolean" ) - if ( - create is True or use_previous_value is False - ) and initial_value is None: + if (create or not use_previous_value) and initial_value is None: raise InvalidParameterError( "'initial_value' is required when creating a new " "stack or when 'use_previous_value' is false" ) - if create is True or use_previous_value is False: + if create or not use_previous_value: if isinstance(initial_value, list): formatted_parameter["ParameterValue"] = ",".join(initial_value) else: @@ -734,6 +737,7 @@ def _format_parameters(self, parameters, create=False): formatted_parameter["UsePreviousValue"] = use_previous_value else: formatted_parameter["ParameterValue"] = value + formatted_parameters.append(formatted_parameter) return formatted_parameters From 4624444beb6a5b9dfc1a39ad4d5540cfb49667c9 Mon Sep 17 00:00:00 2001 From: Alex Harvey Date: Sun, 16 Jun 2024 13:34:42 +1000 Subject: [PATCH 4/4] tweak --- sceptre/plan/actions.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 3140a6568..4ac533bc5 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -721,13 +721,13 @@ def _format_parameters(self, parameters, create=False): raise InvalidParameterError( "'use_previous_value' must be a boolean" ) - if (create or not use_previous_value) and initial_value is None: - raise InvalidParameterError( - "'initial_value' is required when creating a new " - "stack or when 'use_previous_value' is false" - ) if create or not use_previous_value: - if isinstance(initial_value, list): + if initial_value is None: + raise InvalidParameterError( + "'initial_value' is required when creating a new " + "stack or when 'use_previous_value' is false" + ) + elif isinstance(initial_value, list): formatted_parameter["ParameterValue"] = ",".join(initial_value) else: formatted_parameter["ParameterValue"] = value.get(