diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 742178042..0098f30d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v0.52.0 + rev: v0.61.5 hooks: - id: cfn-python-lint args: diff --git a/docs/_source/conf.py b/docs/_source/conf.py index f43721bbe..847535267 100644 --- a/docs/_source/conf.py +++ b/docs/_source/conf.py @@ -68,7 +68,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set 'language' from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -185,10 +185,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'python': ('https://docs.python.org/', None), + 'python': ('https://docs.python.org/3/', None), 'boto3': ( - 'https://boto3.readthedocs.io/en/latest/', - 'https://boto3.readthedocs.io/en/latest/objects.inv', + 'https://boto3.amazonaws.com/v1/documentation/api/latest/', + 'https://boto3.amazonaws.com/v1/documentation/api/latest/objects.inv', ), 'deepdiff': ( 'https://zepworks.com/deepdiff/current/', diff --git a/docs/_source/docs/faq.rst b/docs/_source/docs/faq.rst index f3f886454..55303cdab 100644 --- a/docs/_source/docs/faq.rst +++ b/docs/_source/docs/faq.rst @@ -92,3 +92,29 @@ Sceptre project, see the `sceptre-sam-handler`_ page on PyPI. .. _sceptre-sam-handler: https://pypi.org/project/sceptre-sam-handler/ + +My CI/CD process uses ``sceptre launch``. How do I delete stacks that aren't needed anymore? +--------------------------------------------------------------------------------------------- + +Running the ``launch`` command is a very useful "1-stop-shop" to apply changes from Stack Configs, +creating stacks that don't exist and updating stacks that do exist. This makes it a very useful +command to configure your CI/CD system to invoke. However, sometimes you need to delete a stack that +isn't needed anymore and you want this automatically applied by the same process. + +This "clean up" is complicated by the fact that Sceptre doesn't know anything that isn't in its +Stack and StackGroup Configs; If you delete a Stack Config, Sceptre won't know to clean it up. + +Therefore, the way to accomplish this "clean up" operation is to perform the change in 3 steps: + +1. First, add ``obsolete: True`` to the Stack Config(s) you want to clean up. + For more information on ``obsolete``, see the :ref:`Stack Config entry on it`. +2. Update your CI/CD process to run ``sceptre launch --prune`` instead of ``sceptre launch``. This + will cause all stacks marked as obsolete to be deleted going forward. +3. Once your CI/CD process has cleaned up all the obsolete stacks, delete the local Stack Config files + you marked as obsolete in step 1, since the stacks they create have all been deleted. + +.. note:: + + Using ``obsolete: True`` will not work if any other stacks depend on that stack that are + not themselves obsolete. Attempting to prune any obsolete stacks that are depended on by + non-obsolete stacks will result in Sceptre immediately failing the launch. diff --git a/docs/_source/docs/stack_config.rst b/docs/_source/docs/stack_config.rst index ccaceb14f..48e74694c 100644 --- a/docs/_source/docs/stack_config.rst +++ b/docs/_source/docs/stack_config.rst @@ -16,7 +16,9 @@ particular Stack. The available keys are listed below. - `template_path`_ or `template`_ *(required)* - `dependencies`_ *(optional)* - `hooks`_ *(optional)* +- `ignore`_ *(optional)* - `notifications`_ *(optional)* +- `obsolete`_ *(optional)* - `on_failure`_ *(optional)* - `parameters`_ *(optional)* - `protected`_ *(optional)* @@ -100,6 +102,47 @@ hooks A list of arbitrary shell or Python commands or scripts to run. Find out more in the :doc:`hooks` section. +ignore +~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +This configuration should be set with a boolean value of ``True`` or ``False``. By default, this is +set to ``False`` on all stacks. + +``ignore`` determines how the stack should be handled when running ``sceptre launch``. A stack +marked with ``ignore: True`` will be completely ignored by the launch command. If the stack does NOT +exist on AWS, it won't be created. If it *DOES* exist, it will neither be updated nor deleted. + +You *can* mark a stack with ``ignore: True`` that other non-ignored stacks depend on, but the launch +will fail if dependent stacks require resources or outputs that don't exist because the stack has not been +launched. **Therefore, only ignore dependencies of other stacks if you are aware of the risks of +launch failure.** + +This setting can be especially useful when combined with Jinja logic to exclude certain stacks from +launch based upon conditional Jinja-based template logic. + +For Example: + +.. code-block:: yaml + + template: + path: "my/test/resources.yaml" + + # Configured this way, if the var "use_test_resources" is not true, the stack will not be launched + # and instead excluded from the launch. But if "use_test_resources" is true, the stack will be + # deployed along with the rest of the resources being deployed. + {% if not var.use_test_resources %} + ignore: True + {% endif %} + + +.. note:: + The ``ignore`` configuration **only** applies to the **launch** command. You can still run + ``create``, ``update``, or ``delete`` commands on a stack marked with ``ignore: True``; + these commands will ignore the ``ignore`` setting and act upon the stack the same as any other. + notifications ~~~~~~~~~~~~~ * Resolvable: Yes @@ -112,6 +155,39 @@ can be specified per Stack. This configuration will be used by the ``create``, can found under the relevant section in the `AWS CloudFormation API documentation`_. +.. _`obsolete`: + +obsolete +~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set + +This configuration should be set with a boolean value of ``True`` or ``False``. By default, this is +set to ``False`` on all stacks. + +The ``obsolete`` configuration should be used to mark stacks to be deleted via ``prune`` actions, +if they currently exist on AWS. (If they don't exist on AWS, pruning does nothing). + +There are two ways to prune obsolete stacks: + +1. ``sceptre prune`` will delete *all* obsolete stacks in the **project**. +2. ``sceptre launch --prune [command path]`` will delete all obsolete stacks in the command path + before continuing with the launch. + +In practice, the ``obsolete`` configuration operates identically to ``ignore`` with the extra prune +effects. When the ``launch`` command is invoked without the ``--prune`` flag, obsolete stacks will +be ignored and not launched, just as if ``ignore: True`` was on the Stack Config. + +**Important**: You cannot have non-obsolete stacks dependent upon obsolete stacks. Both the +``prune`` and ``launch --prune`` will reject such configurations and will not continue if this sort +of dependency structure is detected. Only obsolete stacks can depend on obsolete stacks. + +.. note:: + The ``obsolete`` configuration **only** applies to the **launch** and **prune** commands. You can + still run ``create``, ``update``, or ``delete`` commands on a stack marked with ``obsolete: True``; + these commands will ignore the ``obsolete`` setting and act upon the stack the same as any other. + on_failure ~~~~~~~~~~ * Resolvable: No @@ -245,7 +321,7 @@ For more information on this configuration, its implications, and its uses, see :ref:`Sceptre and IAM: iam_role `. iam_role_session_duration -~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~ * Resolvable: No * Can be inherited from StackGroup: Yes * Inheritance strategy: Overrides parent if set diff --git a/integration-tests/features/launch-stack-group.feature b/integration-tests/features/launch-stack-group.feature index 36009e245..c78c6b2f1 100644 --- a/integration-tests/features/launch-stack-group.feature +++ b/integration-tests/features/launch-stack-group.feature @@ -22,19 +22,19 @@ Feature: Launch stack_group Scenario: launch a stack_group with updates that partially exists Given stack "2/A" exists in "CREATE_COMPLETE" state - and stack "2/B" does not exist - and stack "2/C" does not exist - and the template for stack "2/A" is "updated_template.json" + And stack "2/B" does not exist + And stack "2/C" does not exist + And the template for stack "2/A" is "updated_template.json" When the user launches stack_group "2" Then stack "2/A" exists in "UPDATE_COMPLETE" state - and stack "2/B" exists in "CREATE_COMPLETE" state - and stack "2/C" exists in "CREATE_COMPLETE" state + And stack "2/B" exists in "CREATE_COMPLETE" state + And stack "2/C" exists in "CREATE_COMPLETE" state Scenario: launch a stack_group with updates that already exists Given all the stacks in stack_group "2" are in "CREATE_COMPLETE" - and the template for stack "2/A" is "updated_template.json" - and the template for stack "2/B" is "updated_template.json" - and the template for stack "2/C" is "updated_template.json" + And the template for stack "2/A" is "updated_template.json" + And the template for stack "2/B" is "updated_template.json" + And the template for stack "2/C" is "updated_template.json" When the user launches stack_group "2" Then all the stacks in stack_group "2" are in "UPDATE_COMPLETE" @@ -47,3 +47,31 @@ Feature: Launch stack_group Given stack_group "2" does not exist When the user launches stack_group "2" with ignore dependencies Then all the stacks in stack_group "2" are in "CREATE_COMPLETE" + + Scenario: launch a StackGroup with ignored and obsolete stacks that have not been launched + Given stack_group "launch-actions" does not exist + When the user launches stack_group "launch-actions" + Then stack "launch-actions/obsolete" does not exist + And stack "launch-actions/ignore" does not exist + And stack "launch-actions/deploy" exists in "CREATE_COMPLETE" state + + Scenario: launch a StackGroup without --prune with obsolete stacks that currently exist + Given stack "launch-actions/obsolete" exists using "valid_template.json" + And stack "launch-actions/deploy" exists using "valid_template.json" + When the user launches stack_group "launch-actions" + Then stack "launch-actions/obsolete" exists in "CREATE_COMPLETE" state + And stack "launch-actions/deploy" exists in "CREATE_COMPLETE" state + + Scenario: launch a StackGroup with --prune with obsolete stacks that currently exist + Given stack "launch-actions/obsolete" exists using "valid_template.json" + And stack "launch-actions/deploy" exists using "valid_template.json" + When the user launches stack_group "launch-actions" with --prune + Then stack "launch-actions/obsolete" does not exist + And stack "launch-actions/deploy" exists in "CREATE_COMPLETE" state + + Scenario: launch a StackGroup with ignored stacks that currently exist + Given stack "launch-actions/ignore" exists using "valid_template.json" + And stack "launch-actions/deploy" exists using "valid_template.json" + When the user launches stack_group "launch-actions" + Then stack "launch-actions/ignore" exists in "CREATE_COMPLETE" state + And stack "launch-actions/deploy" exists in "CREATE_COMPLETE" state diff --git a/integration-tests/features/launch-stack.feature b/integration-tests/features/launch-stack.feature index f546c0bb6..73f41ab53 100644 --- a/integration-tests/features/launch-stack.feature +++ b/integration-tests/features/launch-stack.feature @@ -2,24 +2,49 @@ Feature: Launch stack Scenario: launch a new stack Given stack "1/A" does not exist - and the template for stack "1/A" is "valid_template.json" + And the template for stack "1/A" is "valid_template.json" When the user launches stack "1/A" Then stack "1/A" exists in "CREATE_COMPLETE" state Scenario: launch a stack that was newly created Given stack "1/A" exists in "CREATE_COMPLETE" state - and the template for stack "1/A" is "updated_template.json" + And the template for stack "1/A" is "updated_template.json" When the user launches stack "1/A" Then stack "1/A" exists in "UPDATE_COMPLETE" state Scenario: launch a stack that has been previously updated Given stack "1/A" exists in "UPDATE_COMPLETE" state - and the template for stack "1/A" is "valid_template.json" + And the template for stack "1/A" is "valid_template.json" When the user launches stack "1/A" Then stack "1/A" exists in "UPDATE_COMPLETE" state Scenario: launch a new stack with ignore dependencies Given stack "1/A" does not exist - and the template for stack "1/A" is "valid_template.json" + And the template for stack "1/A" is "valid_template.json" When the user launches stack "1/A" with ignore dependencies Then stack "1/A" exists in "CREATE_COMPLETE" state + + Scenario: launch an obsolete stack that doesn't exist + Given stack "launch-actions/obsolete" does not exist + When the user launches stack "launch-actions/obsolete" + Then stack "launch-actions/obsolete" does not exist + + Scenario: launch an obsolete stack that does exist without --prune + Given stack "launch-actions/obsolete" exists using "valid_template.json" + When the user launches stack "launch-actions/obsolete" + Then stack "launch-actions/obsolete" exists in "CREATE_COMPLETE" state + + Scenario: launch an obsolete stack that does exist with --prune + Given stack "launch-actions/obsolete" exists using "valid_template.json" + When the user launches stack "launch-actions/obsolete" with --prune + Then stack "launch-actions/obsolete" does not exist + + Scenario: launch an ignored stack that doesn't exist + Given stack "launch-actions/ignore" does not exist + When the user launches stack "launch-actions/ignore" + Then stack "launch-actions/ignore" does not exist + + Scenario: launch an ignored stack that does exist + Given stack "launch-actions/ignore" exists using "valid_template.json" + When the user launches stack "launch-actions/ignore" + Then stack "launch-actions/ignore" exists in "CREATE_COMPLETE" state diff --git a/integration-tests/features/prune.feature b/integration-tests/features/prune.feature new file mode 100644 index 000000000..fd6baa402 --- /dev/null +++ b/integration-tests/features/prune.feature @@ -0,0 +1,22 @@ +Feature: Prune + + Scenario: Prune with no stacks marked obsolete does nothing + Given stack "pruning/not-obsolete" exists using "valid_template.json" + When command path "pruning/not-obsolete" is pruned + Then stack "pruning/not-obsolete" exists in "CREATE_COMPLETE" state + + Scenario: Prune whole project deletes all obsolete stacks that exist + Given all the stacks in stack_group "pruning" are in "CREATE_COMPLETE" + And stack "launch-actions/obsolete" exists using "valid_template.json" + When the whole project is pruned + Then stack "pruning/obsolete-1" does not exist + And stack "pruning/obsolete-2" does not exist + And stack "launch-actions/obsolete" does not exist + And stack "pruning/not-obsolete" exists in "CREATE_COMPLETE" state + + Scenario: Prune command path only deletes stacks on command path + Given stack "pruning/obsolete-1" exists using "valid_template.json" + And stack "pruning/obsolete-2" exists using "valid_template.json" + When command path "pruning/obsolete-1.yaml" is pruned + Then stack "pruning/obsolete-1" does not exist + And stack "pruning/obsolete-2" exists in "CREATE_COMPLETE" state diff --git a/integration-tests/sceptre-project/config/launch-actions/deploy.yaml b/integration-tests/sceptre-project/config/launch-actions/deploy.yaml new file mode 100644 index 000000000..4d3e36dc5 --- /dev/null +++ b/integration-tests/sceptre-project/config/launch-actions/deploy.yaml @@ -0,0 +1,2 @@ +template: + path: valid_template.yaml diff --git a/integration-tests/sceptre-project/config/launch-actions/ignore.yaml b/integration-tests/sceptre-project/config/launch-actions/ignore.yaml new file mode 100644 index 000000000..4123a82ac --- /dev/null +++ b/integration-tests/sceptre-project/config/launch-actions/ignore.yaml @@ -0,0 +1,4 @@ +ignore: True + +template: + path: valid_template.yaml diff --git a/integration-tests/sceptre-project/config/launch-actions/obsolete.yaml b/integration-tests/sceptre-project/config/launch-actions/obsolete.yaml new file mode 100644 index 000000000..afe30525f --- /dev/null +++ b/integration-tests/sceptre-project/config/launch-actions/obsolete.yaml @@ -0,0 +1,4 @@ +obsolete: True + +template: + path: valid_template.yaml diff --git a/integration-tests/sceptre-project/config/pruning/not-obsolete.yaml b/integration-tests/sceptre-project/config/pruning/not-obsolete.yaml new file mode 100644 index 000000000..b195ba2a7 --- /dev/null +++ b/integration-tests/sceptre-project/config/pruning/not-obsolete.yaml @@ -0,0 +1,3 @@ +template: + path: /Users/jfalkenstein/sceptre/integration-tests/sceptre-project/templates/valid_template.json + type: file diff --git a/integration-tests/sceptre-project/config/pruning/obsolete-1.yaml b/integration-tests/sceptre-project/config/pruning/obsolete-1.yaml new file mode 100644 index 000000000..ba377c5ea --- /dev/null +++ b/integration-tests/sceptre-project/config/pruning/obsolete-1.yaml @@ -0,0 +1,4 @@ +obsolete: true +template: + path: /Users/jfalkenstein/sceptre/integration-tests/sceptre-project/templates/valid_template.json + type: file diff --git a/integration-tests/sceptre-project/config/pruning/obsolete-2.yaml b/integration-tests/sceptre-project/config/pruning/obsolete-2.yaml new file mode 100644 index 000000000..ba377c5ea --- /dev/null +++ b/integration-tests/sceptre-project/config/pruning/obsolete-2.yaml @@ -0,0 +1,4 @@ +obsolete: true +template: + path: /Users/jfalkenstein/sceptre/integration-tests/sceptre-project/templates/valid_template.json + type: file diff --git a/integration-tests/sceptre-project/templates/sam_template.yaml b/integration-tests/sceptre-project/templates/sam_template.yaml index 713361359..2985d99a4 100644 --- a/integration-tests/sceptre-project/templates/sam_template.yaml +++ b/integration-tests/sceptre-project/templates/sam_template.yaml @@ -5,5 +5,5 @@ Resources: Type: 'AWS::Serverless::Function' Properties: Handler: index.handler - Runtime: python3.6 + Runtime: python3.9 InlineCode: "print('Hello World!')" diff --git a/integration-tests/sceptre-project/templates/sam_updated_template.yaml b/integration-tests/sceptre-project/templates/sam_updated_template.yaml index e47bf94ff..c0630953c 100644 --- a/integration-tests/sceptre-project/templates/sam_updated_template.yaml +++ b/integration-tests/sceptre-project/templates/sam_updated_template.yaml @@ -5,5 +5,5 @@ Resources: Type: 'AWS::Serverless::Function' Properties: Handler: index.handler - Runtime: python3.6 + Runtime: python3.9 InlineCode: "print('Hello Again World!')" diff --git a/integration-tests/steps/stack_groups.py b/integration-tests/steps/stack_groups.py index f6debb426..5b31d391f 100644 --- a/integration-tests/steps/stack_groups.py +++ b/integration-tests/steps/stack_groups.py @@ -5,6 +5,8 @@ from botocore.exceptions import ClientError from helpers import read_template_file, get_cloudformation_stack_name, retry_boto_call +from sceptre.cli.launch import Launcher +from sceptre.cli.prune import PATH_FOR_WHOLE_PROJECT, Pruner from sceptre.context import SceptreContext from sceptre.diffing.diff_writer import DeepDiffWriter from sceptre.diffing.stack_differ import DeepDiffStackDiffer, DifflibStackDiffer @@ -47,25 +49,28 @@ def step_impl(context, stack_group_name, status): @when('the user launches stack_group "{stack_group_name}"') def step_impl(context, stack_group_name): - sceptre_context = SceptreContext( - command_path=stack_group_name, - project_path=context.sceptre_dir - ) + launch_stack_group(context, stack_group_name) - sceptre_plan = SceptrePlan(sceptre_context) - sceptre_plan.launch() + +@when('the user launches stack_group "{stack_group_name}" with --prune') +def step_impl(context, stack_group_name): + launch_stack_group(context, stack_group_name, True) @when('the user launches stack_group "{stack_group_name}" with ignore dependencies') def step_impl(context, stack_group_name): + launch_stack_group(context, stack_group_name, False, True) + + +def launch_stack_group(context, stack_group_name, prune=False, ignore_dependencies=False): sceptre_context = SceptreContext( command_path=stack_group_name, project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=ignore_dependencies ) - sceptre_plan = SceptrePlan(sceptre_context) - sceptre_plan.launch() + launcher = Launcher(sceptre_context) + launcher.launch(prune) @when('the user deletes stack_group "{stack_group_name}"') @@ -291,6 +296,17 @@ def step_impl(context, group_name, diff_type): context.output = list(sceptre_plan.diff(differ).values()) +@when("the whole project is pruned") +def step_impl(context): + sceptre_context = SceptreContext( + command_path=PATH_FOR_WHOLE_PROJECT, + project_path=context.sceptre_dir, + ) + + pruner = Pruner(sceptre_context) + pruner.prune() + + def get_stack_creation_times(context, stacks): creation_times = {} response = retry_boto_call(context.client.describe_stacks) diff --git a/integration-tests/steps/stacks.py b/integration-tests/steps/stacks.py index eddcd810c..b35579919 100644 --- a/integration-tests/steps/stacks.py +++ b/integration-tests/steps/stacks.py @@ -13,6 +13,8 @@ from botocore.exceptions import ClientError from helpers import read_template_file, get_cloudformation_stack_name from helpers import retry_boto_call +from sceptre.cli.launch import Launcher +from sceptre.cli.prune import Pruner from sceptre.diffing.diff_writer import DeepDiffWriter, DiffWriter from sceptre.diffing.stack_differ import DeepDiffStackDiffer, DifflibStackDiffer, StackDiff @@ -266,35 +268,45 @@ def step_impl(context, stack_name): @when('the user launches stack "{stack_name}"') def step_impl(context, stack_name): + launch_stack(context, stack_name) + + +@when('the user launches stack "{stack_name}" with --prune') +def step_impl(context, stack_name): + launch_stack(context, stack_name, True) + + +@when('command path "{path}" is pruned') +def step_impl(context, path): sceptre_context = SceptreContext( - command_path=stack_name + '.yaml', - project_path=context.sceptre_dir + command_path=path, + project_path=context.sceptre_dir, ) - sceptre_plan = SceptrePlan(sceptre_context) - - try: - sceptre_plan.launch() - except Exception as e: - context.error = e + pruner = Pruner(sceptre_context) + pruner.prune() -@when('the user launches stack "{stack_name}" with ignore dependencies') -def step_impl(context, stack_name): +def launch_stack(context, stack_name, prune=False, ignore_dependencies=False): sceptre_context = SceptreContext( command_path=stack_name + '.yaml', project_path=context.sceptre_dir, - ignore_dependencies=True + ignore_dependencies=ignore_dependencies ) - sceptre_plan = SceptrePlan(sceptre_context) + launcher = Launcher(sceptre_context) try: - sceptre_plan.launch() + launcher.launch(prune) except Exception as e: context.error = e +@when('the user launches stack "{stack_name}" with ignore dependencies') +def step_impl(context, stack_name): + launch_stack(context, stack_name, False, True) + + @when('the user describes the resources of stack "{stack_name}"') def step_impl(context, stack_name): sceptre_context = SceptreContext( diff --git a/integration-tests/steps/templates.py b/integration-tests/steps/templates.py index 52187dd11..a2cdc0ca5 100644 --- a/integration-tests/steps/templates.py +++ b/integration-tests/steps/templates.py @@ -29,6 +29,7 @@ def set_template_path(context, stack_name, template_name): if "template_path" in stack_config: stack_config["template_path"] = template_path if "template" in stack_config: + stack_config["template"]["type"] = stack_config["template"].get("type", "file") template_handler_type = stack_config["template"]["type"] if template_handler_type.lower() == 's3': segments = stack_config["template"]["path"].split('/') diff --git a/requirements/dev.txt b/requirements/dev.txt index 014265824..e8b319ef5 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -12,10 +12,10 @@ pytest-sugar>=0.9.4,<1.0.0 readme-renderer>=24.0,<25.0 requests-mock>=1.9.3,<2.0 setuptools==63.2.0 -Sphinx>=1.6.5,<4.3 +Sphinx>=1.6.5,<=5.1.1 sphinx-click>=2.0.1,<4.0.0 sphinx-rtd-theme==0.5.2 -sphinx-autodoc-typehints==1.12.0 +sphinx-autodoc-typehints==1.19.2 docutils<0.17 # temporary fix for sphinx-rtd-theme==0.5.2, it depends on docutils<0.17 tox>=3.23.0,<4.0.0 twine>=1.12.1,<2.0.0 diff --git a/sceptre/cli/__init__.py b/sceptre/cli/__init__.py index a3c2bed28..00da04d0d 100644 --- a/sceptre/cli/__init__.py +++ b/sceptre/cli/__init__.py @@ -16,6 +16,7 @@ from sceptre import __version__ from sceptre.cli.new import new_group from sceptre.cli.create import create_command +from sceptre.cli.prune import prune_command from sceptre.cli.update import update_command from sceptre.cli.delete import delete_command from sceptre.cli.launch import launch_command @@ -92,3 +93,4 @@ def cli( cli.add_command(fetch_remote_template_command) cli.add_command(diff_command) cli.add_command(drift_group) +cli.add_command(prune_command) diff --git a/sceptre/cli/delete.py b/sceptre/cli/delete.py index 5510c875a..6ceaeea72 100644 --- a/sceptre/cli/delete.py +++ b/sceptre/cli/delete.py @@ -48,9 +48,8 @@ def delete_command(ctx, path, change_set_name, yes): delete_msg = "The following stacks, in the following order, will be deleted:\n" dependencies = '' - for stacks in plan.launch_order: - for stack in stacks: - dependencies += "{}{}{}\n".format(Fore.YELLOW, stack.name, Style.RESET_ALL) + for stack in plan: + dependencies += "{}{}{}\n".format(Fore.YELLOW, stack.name, Style.RESET_ALL) print(delete_msg + "{}".format(dependencies)) diff --git a/sceptre/cli/diff.py b/sceptre/cli/diff.py index b63897277..a7f2697cd 100644 --- a/sceptre/cli/diff.py +++ b/sceptre/cli/diff.py @@ -41,15 +41,34 @@ '-n', '--no-placeholders', is_flag=True, - help="If True, no placeholder values will be supplied for resolvers that cannot be resolved." + help="If set, no placeholder values will be supplied for resolvers that cannot be resolved." +) +@click.option( + '-a', + '--all', + 'all_', + is_flag=True, + help=( + "If set, will perform diffing on ALL stacks, including ignored and obsolete ones; Otherwise, " + "it will diff only stacks that would be created or updated when running the launch command." + ) ) @click.argument('path') @click.pass_context @catch_exceptions -def diff_command(ctx: Context, differ: str, show_no_echo: bool, no_placeholders: bool, path: str): +def diff_command( + ctx: Context, + differ: str, + show_no_echo: bool, + no_placeholders: bool, + all_: bool, + path: str +): """Indicates the difference between the currently DEPLOYED stacks in the command path and the stacks configured in Sceptre right now. This command will compare both the templates as well - as the subset of stack configurations that can be compared. + as the subset of stack configurations that can be compared. By default, only stacks that would + be launched via the launch command will be diffed, but you can diff ALL stacks relevant to the + passed command path if you pass the --all flag. Some settings (such as sceptre_user_data) are not available in a CloudFormation stack description, so the diff will not be indicated. Currently compared stack configurations are: @@ -60,7 +79,7 @@ def diff_command(ctx: Context, differ: str, show_no_echo: bool, no_placeholders: * role_arn * stack_tags - Important: There are resolvers (notably !stack_output, among others) that rely on other stacks + Important: There are resolvers (notably !stack_output) that rely on other stacks to be already deployed when they are resolved. When producing a diff on Stack Configs that have such resolvers that point to non-deployed stacks, this presents a challenge, since this means those resolvers cannot be resolved. This particularly applies to stack parameters and when a @@ -87,6 +106,8 @@ def diff_command(ctx: Context, differ: str, show_no_echo: bool, no_placeholders: ) output_format = context.output_format plan = SceptrePlan(context) + if not all_: + filter_plan_for_launchable(plan) if differ == "deepdiff": stack_differ = DeepDiffStackDiffer(show_no_echo) @@ -156,3 +177,8 @@ def output_buffer_with_normalized_bar_lengths(buffer: io.StringIO, output_stream if DiffWriter.LINE_BAR in line: line = line.replace(DiffWriter.LINE_BAR, full_length_line_bar) output_stream.write(line) + + +def filter_plan_for_launchable(plan: SceptrePlan): + plan.resolve(plan.diff.__name__) + plan.filter(lambda stack: not stack.ignore and not stack.obsolete) diff --git a/sceptre/cli/launch.py b/sceptre/cli/launch.py index 09ed462df..da9c2f140 100644 --- a/sceptre/cli/launch.py +++ b/sceptre/cli/launch.py @@ -1,28 +1,43 @@ +import logging +from typing import List + import click +from click import Context +from colorama import Fore, Style +from sceptre.cli.helpers import catch_exceptions, confirmation, stack_status_exit_code +from sceptre.cli.prune import Pruner from sceptre.context import SceptreContext -from sceptre.cli.helpers import catch_exceptions -from sceptre.cli.helpers import confirmation -from sceptre.cli.helpers import stack_status_exit_code +from sceptre.exceptions import DependencyDoesNotExistError from sceptre.plan.plan import SceptrePlan +from sceptre.stack import Stack + +logger = logging.getLogger(__name__) @click.command(name="launch", short_help="Launch a Stack or StackGroup.") @click.argument("path") +@click.option("-y", "--yes", is_flag=True, help="Assume yes to all questions.") @click.option( - "-y", "--yes", is_flag=True, help="Assume yes to all questions." + "-p", + "--prune", + is_flag=True, + help="If set, will delete all stacks in the command path marked as obsolete." ) @click.pass_context @catch_exceptions -def launch_command(ctx, path, yes): +def launch_command(ctx: Context, path: str, yes: bool, prune: bool): """ - Launch a Stack or StackGroup for a given config PATH. - \f + Launch a Stack or StackGroup for a given config PATH. This command is intended as a catch-all + command that will apply any changes from Stack Configs indicated via the path. - :param path: The path to launch. Can be a Stack or StackGroup. - :type path: str - :param yes: A flag to answer 'yes' to all CLI questions. - :type yes: bool + \b + * Any Stacks that do not exist will be created + * Any stacks that already exist will be updated (if there are any changes) + * If any stacks are marked with "ignore: True", those stacks will neither be created nor updated + * If any stacks are marked with "obsolete: True", those stacks will neither be created nor updated. + Furthermore, if the "-p"/"--prune" flag is used, these stacks will be deleted prior to any + other launch commands """ context = SceptreContext( command_path=path, @@ -31,9 +46,135 @@ def launch_command(ctx, path, yes): options=ctx.obj.get("options"), ignore_dependencies=ctx.obj.get("ignore_dependencies") ) + launcher = Launcher(context) + launcher.print_operations(prune) + if not yes: + launcher.confirm(prune) + + exit_code = launcher.launch(prune) + exit(exit_code) + + +class Launcher: + """Launcher is a utility to coordinate the flow of launching. + + :param context: The Sceptre context to use for launching + :param plan_factory: A callable with the signature of (SceptreContext) -> SceptrePlan + """ + def __init__(self, context: SceptreContext, plan_factory=SceptrePlan, pruner_factory=Pruner): + self._context = context + self._make_plan = plan_factory + self._make_pruner = pruner_factory + + self._plan = None + + def confirm(self, prune: bool): + self._confirm_launch(prune) + + def print_operations(self, prune: bool): + deploy_plan = self._create_deploy_plan() + stacks_to_skip = self._get_stacks_to_skip(deploy_plan, prune) + self._print_skips(stacks_to_skip) + if prune: + pruner = self._make_pruner(self._context, self._make_plan) + pruner.print_operations() + + def launch(self, prune: bool) -> int: + deploy_plan = self._create_deploy_plan() + stacks_to_skip = self._get_stacks_to_skip(deploy_plan, prune) + stacks_to_prune = self._get_stacks_to_prune(deploy_plan, prune) + + self._exclude_stacks_from_plan(deploy_plan, *stacks_to_skip, *stacks_to_prune) + self._validate_launch_for_missing_dependencies(deploy_plan, prune) + + code = 0 + if prune: + code = self._prune() + + code = code or self._deploy(deploy_plan) + return code + + def _create_deploy_plan(self) -> SceptrePlan: + if not self._plan: + plan = self._make_plan(self._context) + # The plan must be resolved so we can modify launch order and items before executing it + plan.resolve(plan.launch.__name__) + self._plan = plan + return self._plan + + def _get_stacks_to_skip(self, deploy_plan: SceptrePlan, prune: bool) -> List[Stack]: + return [stack for stack in deploy_plan if stack.ignore or (stack.obsolete and not prune)] + + def _get_stacks_to_prune(self, deploy_plan: SceptrePlan, prune: bool) -> List[Stack]: + return [stack for stack in deploy_plan if prune and stack.obsolete] + + def _exclude_stacks_from_plan(self, deployment_plan: SceptrePlan, *stacks: Stack): + for stack in stacks: + deployment_plan.remove_stack_from_plan(stack) + + def _validate_launch_for_missing_dependencies(self, deploy_plan: SceptrePlan, prune: bool): + validated_stacks = set() + skipped_dependencies = set() + + def validate_stack_dependencies(stack: Stack): + if stack in validated_stacks: + # In order to avoid unnecessary recursions on stacks already evaluated, we'll return + # early if we've already evaluated the stack without issue. + return + if prune and stack.obsolete: + raise DependencyDoesNotExistError( + f"Launch plan with --prune option depends on stack '{stack.name}' that is marked " + f"as obsolete. Only obsolete stacks can depend upon obsolete stacks when pruning." + ) + for dependency in stack.dependencies: + if dependency.ignore or dependency.obsolete: + skipped_dependencies.add(dependency) + if not self._context.ignore_dependencies: + validate_stack_dependencies(dependency) + validated_stacks.add(stack) + + for stack in deploy_plan: + validate_stack_dependencies(stack) + + message = ( + "WARNING: Launch plan depends on the following ignored and/or obsolete stacks.\n" + " Sceptre will attempt to continue with launch, but it may fail if any Stack Configs \n" + " require certain resources or outputs that don't currently exist." + ) + self._print_stacks_with_message(list(skipped_dependencies), message) + + def _print_skips(self, stacks_to_skip: List[Stack]): + skip_message = "During launch, the following stacks will be skipped, neither created nor updated:" + self._print_stacks_with_message(stacks_to_skip, skip_message) + + def _print_stacks_with_message(self, stacks: List[Stack], message: str): + if not len(stacks): + return + + message = f'* {message}\n' + for stack in stacks: + message += f"{Fore.YELLOW}{stack.name}{Style.RESET_ALL}\n" + + click.echo(message) + + def _print_deletions(self, stacks_to_prune: List[Stack]): + delete_message = "During launch, the following stacks will be will be deleted, if they exist:" + self._print_stacks_with_message(stacks_to_prune, delete_message) + + def _confirm_launch(self, prune: bool): + operation_name = "launch" + if prune: + operation_name += " --prune" + confirmation(operation_name, False, command_path=self._context.command_path) - plan = SceptrePlan(context) + def _prune(self) -> int: + pruner = self._make_pruner(self._context, self._make_plan) + exit_code = pruner.prune() + if exit_code != 0: + click.echo("Stack deletion failed, so could not proceed with launch.") + return exit_code - confirmation(plan.launch.__name__, yes, command_path=path) - responses = plan.launch() - exit(stack_status_exit_code(responses.values())) + def _deploy(self, deploy_plan: SceptrePlan) -> int: + result = deploy_plan.launch() + exit_code = stack_status_exit_code(result.values()) + return exit_code diff --git a/sceptre/cli/prune.py b/sceptre/cli/prune.py new file mode 100644 index 000000000..e24ac38e6 --- /dev/null +++ b/sceptre/cli/prune.py @@ -0,0 +1,170 @@ +import click +from colorama import Fore, Style + +from sceptre.cli.helpers import catch_exceptions, stack_status_exit_code +from sceptre.context import SceptreContext +from sceptre.exceptions import CannotPruneStackError +from sceptre.plan.plan import SceptrePlan +from sceptre.stack import Stack + +PATH_FOR_WHOLE_PROJECT = "." + + +@click.command(name="prune", short_help="Deletes all obsolete stacks in the project") +@click.option( + "-y", "--yes", is_flag=True, help="Assume yes to all questions." +) +@click.argument("path", default=PATH_FOR_WHOLE_PROJECT) +@click.pass_context +@catch_exceptions +def prune_command(ctx, yes: bool, path): + """ + This command deletes all obsolete stacks in the project. Only obsolete stacks can be deleted + via prune; If any non-obsolete stacks depend on obsolete stacks, an error will be + raised and this command will fail. + """ + context = SceptreContext( + command_path=path, + project_path=ctx.obj.get("project_path"), + user_variables=ctx.obj.get("user_variables"), + options=ctx.obj.get("options"), + ignore_dependencies=ctx.obj.get("ignore_dependencies"), + full_scan=True, + ) + pruner = Pruner(context) + pruner.print_operations() + if not yes and pruner.prune_count > 0: + pruner.confirm() + + code = pruner.prune() + exit(code) + + +class Pruner: + """Pruner is a utility to coordinate the flow of deleting all stacks in the project that + are marked "obsolete". + + Note: The command_path on the passed context will be ignored; This command operates on the + entire project rather than on any particular command path. + + :param context: The Sceptre context to use for pruning + :param plan_factory: A callable with the signature of (SceptreContext) -> SceptrePlan + """ + def __init__(self, context: SceptreContext, plan_factory=SceptrePlan): + self._context = context + self._make_plan = plan_factory + + self._plan = None + + def confirm(self): + self._confirm_prune() + + def print_operations(self): + plan = self._create_plan() + + if not self._plan_has_obsolete_stacks(plan): + self._print_no_obsolete_stacks() + return + + self._print_stacks_to_be_deleted(plan) + + @property + def prune_count(self) -> 0: + plan = self._create_plan() + if self._plan_has_obsolete_stacks(plan): + return len(list(plan)) + return 0 + + def prune(self) -> int: + plan = self._create_plan() + + if not self._plan_has_obsolete_stacks(plan): + return 0 + + if not self._context.ignore_dependencies: + self._validate_plan_for_dependencies_on_obsolete_stacks(plan) + + code = self._prune(plan) + return code + + def _create_plan(self): + if not self._plan: + context = self._context.clone() + context.full_scan = True + plan = self._make_plan(self._context) + if context.command_path == PATH_FOR_WHOLE_PROJECT: + stacks = plan.graph + else: + stacks = plan.command_stacks + + plan.command_stacks = { + stack for stack in stacks if stack.obsolete + } + self._resolve_plan(plan) + self._plan = plan + return self._plan + + def _plan_has_obsolete_stacks(self, plan: SceptrePlan): + return len(plan.command_stacks) > 0 + + def _print_no_obsolete_stacks(self): + click.echo("* There are no stacks marked obsolete, so there is nothing to prune.") + + def _resolve_plan(self, plan: SceptrePlan): + if len(plan.command_stacks) > 0: + # Prune is actually a particular kind of filtered deletion, so we use delete as the actual + # resolved command. + plan.resolve(plan.delete.__name__, reverse=True) + + def _validate_plan_for_dependencies_on_obsolete_stacks(self, plan: SceptrePlan): + def check_for_non_obsolete_dependencies(stack: Stack): + # If we've already established it as an obsolete stack to delete, we're good. + if stack in plan.command_stacks: + return + + # This check shouldn't be necessary, but we're just double-checking that it is indeed + # not obsolete. + if stack.obsolete: + return + + # Theoretically, we've already gathered up ALL obsolete stacks as command stacks. If + # we've hit this line, there's a problem. Now we just need to know what caused it. This + # block climbs down the dependency graph to see which obsolete stack caused this stack + # to be included in the plan. + for dependency in stack.dependencies: + if dependency.obsolete: + raise CannotPruneStackError( + f"Cannot prune obsolete stack {dependency.name} because stack {stack.name} " + f"depends on it but is not obsolete." + ) + + # If we get to this point, it means this stack isn't obsolete and none of its dependencies + # are either. That only happens it depends on another non-obsolete stack that depends on + # an obsolete stack. As a result, we're not going to blow up here and instead will + # continue iterating on the plan and will raise the error on a stack that directly + # depends on the obsolete stack. + return + + for stack in plan: + check_for_non_obsolete_dependencies(stack) + + def _print_stacks_to_be_deleted(self, plan: SceptrePlan): + delete_msg = "* The following obsolete stacks will be deleted (if they exist on AWS):\n" + + stacks_list = '' + for stack in plan: + # It's possible there could be stacks in the plan that aren't obsolete because those + # stacks depend on obsolete stacks. They won't pass validation, but that's not the + # point of this method. We'll just skip those here and fail validation later. + if not stack.obsolete: + continue + stacks_list += "{}{}{}\n".format(Fore.YELLOW, stack.name, Style.RESET_ALL) + + click.echo(delete_msg + stacks_list) + + def _confirm_prune(self): + click.confirm("Do you want to delete these stacks?", abort=True) + + def _prune(self, plan: SceptrePlan): + responses = plan.delete() + return stack_status_exit_code(responses.values()) diff --git a/sceptre/config/graph.py b/sceptre/config/graph.py index 1f52b6a9f..b77ba0194 100644 --- a/sceptre/config/graph.py +++ b/sceptre/config/graph.py @@ -8,8 +8,11 @@ """ import logging +from typing import List + import networkx as nx from sceptre.exceptions import CircularDependenciesError +from sceptre.stack import Stack class StackGraph(object): @@ -77,27 +80,25 @@ def _generate_graph(self, stacks): self._generate_edges(stack, stack.dependencies) self.graph.remove_edges_from(nx.selfloop_edges(self.graph)) - def _generate_edges(self, stack, dependencies): + def _generate_edges(self, stack: Stack, dependencies: List[Stack]): """ Adds edges to the graph based on a list of dependencies that are - generated from the inital stack config. Each of the paths + generated from the initial stack config. Each of the paths in the inital_dependency_paths list are a depency that the inital Stack config depends on. :param stack: A Sceptre Stack - :type stack: sceptre.stack.Stack :param dependencies: a collection of dependency paths - :type dependencies: list """ self.logger.debug( "Generate dependencies for stack {0}".format(stack) ) - for dependency in dependencies: + for dependency in set(dependencies): self.graph.add_edge(dependency, stack) if not nx.is_directed_acyclic_graph(self.graph): raise CircularDependenciesError( - "Dependency cycle detected: {} {}".format(stack, - dependency)) + f"Dependency cycle detected: {stack} {dependency}" + ) self.logger.debug(" Added dependency: {}".format(dependency)) if not dependencies: diff --git a/sceptre/config/reader.py b/sceptre/config/reader.py index 30ec1f957..b7eb8e3a4 100644 --- a/sceptre/config/reader.py +++ b/sceptre/config/reader.py @@ -16,7 +16,7 @@ import yaml from os import environ, path, walk -from typing import Set +from typing import Set, Tuple from pathlib import Path from jinja2 import Environment from jinja2 import StrictUndefined @@ -58,7 +58,9 @@ "template_bucket_name": strategies.child_wins, "template_key_value": strategies.child_wins, "template_path": strategies.child_wins, - "template": strategies.child_wins + "template": strategies.child_wins, + "ignore": strategies.child_wins, + "obsolete": strategies.child_wins } STACK_GROUP_CONFIG_ATTRIBUTES = ConfigAttributes( @@ -203,7 +205,7 @@ def resolve_node_tag(self, loader, node): node.tag = loader.resolve(type(node), node.value, (True, False)) return node - def construct_stacks(self) -> Set[Stack]: + def construct_stacks(self) -> Tuple[Set[Stack], Set[Stack]]: """ Traverses the files under the command path. For each file encountered, a Stack is constructed @@ -272,7 +274,7 @@ def construct_stacks(self) -> Set[Stack]: return stacks, command_stacks - def resolve_stacks(self, stack_map): + def resolve_stacks(self, stack_map) -> Set[Stack]: """ Transforms map of Stacks into a set of Stacks, transforms dependencies from a list of Strings (stack names) to a list of Stacks. @@ -301,7 +303,9 @@ def resolve_stacks(self, stack_map): "have their full path from `config` defined." .format(stackname=stack.name, dep=dep, stackkeys=", ".join(stack_map.keys()))) - + # We deduplicate the dependencies using a set here, since it's possible that a given + # dependency ends up in the list multiple times. + stack.dependencies = list(set(stack.dependencies)) else: stack.dependencies = [] stacks.add(stack) @@ -542,6 +546,7 @@ def _construct_stack(self, rel_path, stack_group_config=None): s3_details = self._collect_s3_details( stack_name, config ) + stack = Stack( name=stack_name, project_code=config["project_code"], @@ -566,6 +571,8 @@ def _construct_stack(self, rel_path, stack_group_config=None): notifications=config.get("notifications"), on_failure=config.get("on_failure"), stack_timeout=config.get("stack_timeout", 0), + ignore=config.get("ignore", False), + obsolete=config.get("obsolete", False), stack_group_config=parsed_stack_group_config ) diff --git a/sceptre/context.py b/sceptre/context.py index 9508dbc5d..6376b7c07 100644 --- a/sceptre/context.py +++ b/sceptre/context.py @@ -6,7 +6,7 @@ This module implements the SceptreContext class which holds details about the paths used in a Sceptre project. """ - +from copy import deepcopy from os import path from sceptre.helpers import normalise_path @@ -123,3 +123,9 @@ def command_path_is_stack(self): self.command_path ) ) + + def clone(self) -> "SceptreContext": + """Creates a new, deep clone of the context with all the same values.""" + new = type(self).__new__(type(self)) + new.__dict__.update(deepcopy(self.__dict__)) + return new diff --git a/sceptre/exceptions.py b/sceptre/exceptions.py index 9429bb676..6bf73bd20 100644 --- a/sceptre/exceptions.py +++ b/sceptre/exceptions.py @@ -187,3 +187,10 @@ class TemplateNotFoundError(SceptreException): Error raised when a Template file is not found """ pass + + +class CannotPruneStackError(SceptreException): + """ + Error raised when an obsolete stack cannot be pruned because another stack depends on it that is + not itself obsolete. + """ diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py index 992a6e9bb..1576e1d7c 100644 --- a/sceptre/plan/actions.py +++ b/sceptre/plan/actions.py @@ -10,6 +10,7 @@ import json import logging import time +import typing import urllib from datetime import datetime, timedelta from os import path @@ -18,17 +19,22 @@ import botocore from dateutil.tz import tzutc -from sceptre.connection_manager import ConnectionManager -from sceptre.exceptions import CannotUpdateFailedStackError -from sceptre.exceptions import ProtectedStackError -from sceptre.exceptions import StackDoesNotExistError -from sceptre.exceptions import UnknownStackChangeSetStatusError -from sceptre.exceptions import UnknownStackStatusError -from sceptre.hooks import add_stack_hooks -from sceptre.stack_status import StackChangeSetStatus -from sceptre.stack_status import StackStatus from sceptre.config.reader import ConfigReader +from sceptre.connection_manager import ConnectionManager +from sceptre.exceptions import ( + CannotUpdateFailedStackError, + ProtectedStackError, + StackDoesNotExistError, + UnknownStackChangeSetStatusError, + UnknownStackStatusError +) from sceptre.helpers import normalise_path +from sceptre.hooks import add_stack_hooks +from sceptre.stack import Stack +from sceptre.stack_status import StackChangeSetStatus, StackStatus + +if typing.TYPE_CHECKING: + from sceptre.diffing.stack_differ import StackDiff, StackDiffer class StackActions(object): @@ -40,7 +46,7 @@ class StackActions(object): :type stack: sceptre.stack.Stack """ - def __init__(self, stack): + def __init__(self, stack: Stack): self.stack = stack self.name = self.stack.name self.logger = logging.getLogger(__name__) @@ -177,7 +183,7 @@ def cancel_stack_update(self): return self._wait_for_completion() @add_stack_hooks - def launch(self): + def launch(self) -> StackStatus: """ Launches the Stack. @@ -187,10 +193,10 @@ def launch(self): performed, launch exits gracefully. :returns: The Stack's status. - :rtype: sceptre.stack_status.StackStatus """ self._protect_execution() - self.logger.info("%s - Launching Stack", self.stack.name) + self.logger.info(f"{self.stack.name} - Launching Stack") + try: existing_status = self._get_status() except StackDoesNotExistError: @@ -214,7 +220,6 @@ def launch(self): ) status = StackStatus.IN_PROGRESS elif existing_status.endswith("FAILED"): - status = StackStatus.FAILED raise CannotUpdateFailedStackError( "'{0}' is in a the state '{1}' and cannot be updated".format( self.stack.name, existing_status @@ -991,16 +996,13 @@ def _get_template_summary(self, **kwargs) -> Optional[dict]: raise @add_stack_hooks - def diff(self, stack_differ): + def diff(self, stack_differ: "StackDiffer") -> "StackDiff": """ - Returns a diff of Template and Remote Template - using a specific diff library. - - :param stack_differ: The diff lib to use, default difflib. - :type: sceptre.diffing.stack_differ.StackDiffer + Returns a diff of local and deployed template and stack configuration using a specific diff + library. - :returns: A StackDiff object. - :rtype: sceptre.diffing.stack_differ.StackDiff + :param stack_differ: The differ to use + :returns: A StackDiff object with the full, computed diff """ return stack_differ.diff(self) @@ -1010,10 +1012,10 @@ def drift_detect(self) -> Dict[str, str]: Show stack drift for a running stack. :returns: The stack drift detection status. - If the stack does not exist, we return a detection and - stack drift status of STACK_DOES_NOT_EXIST. - If drift detection times out after 5 minutes, we return - TIMED_OUT. + If the stack does not exist, we return a detection and + stack drift status of STACK_DOES_NOT_EXIST. + If drift detection times out after 5 minutes, we return + TIMED_OUT. """ try: self._get_status() diff --git a/sceptre/plan/executor.py b/sceptre/plan/executor.py index 8d121fbe6..0789213a4 100644 --- a/sceptre/plan/executor.py +++ b/sceptre/plan/executor.py @@ -7,41 +7,38 @@ executing the command specified in a SceptrePlan. """ import logging - from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Set + from sceptre.plan.actions import StackActions -from sceptre.stack_status import StackStatus +from sceptre.stack import Stack class SceptrePlanExecutor(object): - def __init__(self, command, launch_order): + def __init__(self, command: str, launch_order: List[Set[Stack]]): """ - Initalises a SceptrePlanExecutor, generates the launch order, threads + Initialises a SceptrePlanExecutor, generates the launch order, threads and intial Stack Statuses. :param command: The command to execute on the Stack. - :type command: str - :param launch_order: A list containing sets of Stacks that can be\ - executed concurrently. - :type launch_order: list + :param launch_order: A list containing sets of Stacks that can be executed concurrently. """ self.logger = logging.getLogger(__name__) self.command = command self.launch_order = launch_order - - self.num_threads = len(max(launch_order, key=len)) - self.stack_statuses = {stack: StackStatus.PENDING - for batch in launch_order for stack in batch} + # Select the number of threads based upon the max batch size, + # or use 1 if all batches are empty + self.num_threads = len(max(launch_order, key=len)) or 1 def execute(self, *args): """ - Execute is responsible executing the sets of Stacks in `launch_order` + Execute is responsible executing the sets of Stacks in launch_order concurrently, in the correct order. - :param *args: Any arguments that should be passed through to the\ + :param args: Any arguments that should be passed through to the StackAction being called. """ responses = {} diff --git a/sceptre/plan/plan.py b/sceptre/plan/plan.py index 1168dbdbe..7fafbaa4a 100644 --- a/sceptre/plan/plan.py +++ b/sceptre/plan/plan.py @@ -6,49 +6,60 @@ This module implements a SceptrePlan, which is responsible for holding all nessessary information for a command to execute. """ +import functools +import itertools from os import path, walk -from typing import Dict +from typing import Dict, List, Set, Callable, Iterable, Optional -from sceptre.diffing.stack_differ import StackDiff -from sceptre.exceptions import ConfigFileNotFoundError from sceptre.config.graph import StackGraph from sceptre.config.reader import ConfigReader -from sceptre.plan.executor import SceptrePlanExecutor +from sceptre.context import SceptreContext +from sceptre.diffing.stack_differ import StackDiff +from sceptre.exceptions import ConfigFileNotFoundError from sceptre.helpers import sceptreise_path +from sceptre.plan.executor import SceptrePlanExecutor from sceptre.stack import Stack +def require_resolved(func) -> Callable: + @functools.wraps(func) + def wrapped(self: "SceptrePlan", *args, **kwargs): + if self.launch_order is None: + raise RuntimeError(f"You cannot call {func.__name__}() before resolve().") + return func(self, *args, **kwargs) + + return wrapped + + class SceptrePlan(object): - def __init__(self, context): + def __init__(self, context: SceptreContext): """ Intialises a SceptrePlan and generates the Stacks, StackGraph and launch order of required. :param context: A SceptreContext - :type sceptre.context.SceptreContext: """ self.context = context self.command = None self.reverse = None - self.launch_order = None + self.launch_order: Optional[List[Set[Stack]]] = None self.config_reader = ConfigReader(context) all_stacks, command_stacks = self.config_reader.construct_stacks() self.graph = StackGraph(all_stacks) self.command_stacks = command_stacks + @require_resolved def _execute(self, *args): executor = SceptrePlanExecutor(self.command, self.launch_order) return executor.execute(*args) - def _generate_launch_order(self, reverse=False): + def _generate_launch_order(self, reverse=False) -> List[Set[Stack]]: if self.context.ignore_dependencies: return [self.command_stacks] graph = self.graph.filtered(self.command_stacks, reverse) - if self.context.ignore_dependencies: - return [self.command_stacks] launch_order = [] while graph.graph: @@ -69,6 +80,31 @@ def _generate_launch_order(self, reverse=False): return launch_order + @require_resolved + def __iter__(self) -> Iterable[Stack]: + """Iterates the stacks in the launch_order""" + # We cast it to list so it's "frozen" in time, in case the launch order is modified + # while iterating. + yield from list(itertools.chain.from_iterable(self.launch_order)) + + @require_resolved + def remove_stack_from_plan(self, stack: Stack): + for batch in self.launch_order: + if stack in batch: + batch.remove(stack) + return + + @require_resolved + def filter(self, predicate: Callable[[Stack], bool]): + """Filters the plan's resolved launch_order to remove specific stacks. + + :param predicate: This callable should take a single Stack and return True if it should stay + in the launch_order or False if it should be filtered out. + """ + for stack in self: + if not predicate(stack): + self.remove_stack_from_plan(stack) + def resolve(self, command, reverse=False): if command == self.command and reverse == self.reverse: return diff --git a/sceptre/stack.py b/sceptre/stack.py index 97d74ff80..9b2688a2d 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -8,6 +8,7 @@ """ import logging +from typing import List, Any from sceptre.connection_manager import ConnectionManager from sceptre.exceptions import InvalidConfigFileError @@ -27,98 +28,81 @@ class Stack(object): Stack stores information about a particular CloudFormation Stack. :param name: The name of the Stack. - :type project: str :param project_code: A code which is prepended to the Stack names\ of all Stacks built by Sceptre. - :type project_code: str :param template_path: The relative path to the CloudFormation, Jinja2\ or Python template to build the Stack from. If this is filled, `template_handler_config` should not be filled. - :type template_path: str :param template_handler_config: Configuration for a Template Handler that can resolve its arguments to a template string. Should contain the `type` property to specify the type of template handler to load. Conflicts with `template_path`. - :type template_handler_config: dict :param region: The AWS region to build Stacks in. - :type region: str :param template_bucket_name: The name of the S3 bucket the Template is uploaded to. - :type template_bucket_name: str :param template_key_prefix: A prefix to the key used to store templates uploaded to S3 - :type template_key_prefix: str :param required_version: A PEP 440 compatible version specifier. If the Sceptre version does\ not fall within the given version requirement it will abort. - :type required_version: str :param parameters: The keys must match up with the name of the parameter.\ The value must be of the type as defined in the template. - :type parameters: dict :param sceptre_user_data: Data passed into\ `sceptre_handler(sceptre_user_data)` function in Python templates\ or accessible under `sceptre_user_data` variable within Jinja2\ templates. - :type sceptre_user_data: dict :param hooks: A list of arbitrary shell or python commands or scripts to\ run. - :type hooks: sceptre.hooks.Hook :param s3_details: - :type s3_details: dict :param dependencies: The relative path to the Stack, including the file\ extension of the Stack. - :type dependencies: list :param role_arn: The ARN of a CloudFormation Service Role that is assumed\ by CloudFormation to create, update or delete resources. - :type role_arn: str :param protected: Stack protection against execution. - :type protected: bool :param tags: CloudFormation Tags to be applied to the Stack. - :type tags: dict :param external_name: - :type external_name: str :param notifications: SNS topic ARNs to publish Stack related events to.\ A maximum of 5 ARNs can be specified per Stack. - :type notifications: list :param on_failure: This parameter describes the action taken by\ CloudFormation when a Stack fails to create. - :type on_failure: str :param iam_role: The ARN of a role for Sceptre to assume before interacting\ with the environment. If not supplied, Sceptre uses the user's AWS CLI\ credentials. - :type iam_role: str :param profile: The name of the profile as defined in ~/.aws/config and\ ~/.aws/credentials. - :type profile: str :param stack_timeout: A timeout in minutes before considering the Stack\ deployment as failed. After the specified timeout, the Stack will\ - be rolled back. Specifiyng zero, as well as ommiting the field,\ + be rolled back. Specifying zero, as well as omitting the field,\ will result in no timeout. Supports only positive integer value. - :type stack_timeout: int - :param stack_group_config: The StackGroup config for the Stack - :type stack_group_config: dict + :param ignore: If True, this stack will be ignored during launches (but it can be explicitly + deployed with create, update, and delete commands. + + :param obsolete: If True, this stack will operate the same as if ignore was set, but it will + also be deleted if the prune command is invoked or the --prune option is used with the + launch command. :param iam_role_session_duration: The session duration when Scetre assumes a role.\ If not supplied, Sceptre uses default value (3600 seconds) - :type iam_role_session_duration: int + + :param stack_group_config: The StackGroup config for the Stack """ parameters = ResolvableContainerProperty("parameters") @@ -160,9 +144,10 @@ def __init__( self, name: str, project_code: str, region: str, template_path: str = None, template_handler_config: dict = None, template_bucket_name: str = None, template_key_prefix: str = None, required_version: str = None, parameters: dict = None, sceptre_user_data: dict = None, hooks: Hook = None, - s3_details: dict = None, iam_role: str = None, dependencies=None, role_arn: str = None, protected: bool = False, - tags: dict = None, external_name: str = None, notifications=None, on_failure: str = None, profile: str = None, - stack_timeout: int = 0, iam_role_session_duration: int = 0, stack_group_config: dict = {} + s3_details: dict = None, iam_role: str = None, dependencies: List["Stack"] = None, role_arn: str = None, + protected: bool = False, tags: dict = None, external_name: str = None, notifications: List[str] = None, + on_failure: str = None, profile: str = None, stack_timeout: int = 0, iam_role_session_duration: int = 0, + ignore=False, obsolete=False, stack_group_config: dict = {} ): self.logger = logging.getLogger(__name__) @@ -186,6 +171,8 @@ def __init__( self.profile = profile self.template_key_prefix = template_key_prefix self.iam_role_session_duration = iam_role_session_duration + self.ignore = self._ensure_boolean("ignore", ignore) + self.obsolete = self._ensure_boolean("obsolete", obsolete) self._template = None self._connection_manager = None @@ -205,90 +192,68 @@ def __init__( self.hooks = hooks or {} + def _ensure_boolean(self, config_name: str, value: Any) -> bool: + if not isinstance(value, bool): + raise InvalidConfigFileError( + f"{self.name}: Value for {config_name} must be a boolean, not a {type(value).__name__}" + ) + return value + def __repr__(self): return ( "sceptre.stack.Stack(" - "name='{name}', " - "project_code={project_code}, " - "template_path={template_path}, " - "template_handler_config={template_handler_config}, " - "region={region}, " - "template_bucket_name={template_bucket_name}, " - "template_key_prefix={template_key_prefix}, " - "required_version={required_version}, " - "iam_role={iam_role}, " - "iam_role_session_duration={iam_role_session_duration}, " - "profile={profile}, " - "sceptre_user_data={sceptre_user_data}, " - "parameters={parameters}, " - "hooks={hooks}, " - "s3_details={s3_details}, " - "dependencies={dependencies}, " - "role_arn={role_arn}, " - "protected={protected}, " - "tags={tags}, " - "external_name={external_name}, " - "notifications={notifications}, " - "on_failure={on_failure}, " - "stack_timeout={stack_timeout}, " - "stack_group_config={stack_group_config}" - ")".format( - name=self.name, - project_code=self.project_code, - template_path=self.template_path, - template_handler_config=self.template_handler_config, - region=self.region, - template_bucket_name=self.template_bucket_name, - template_key_prefix=self.template_key_prefix, - required_version=self.required_version, - iam_role=self.iam_role, - iam_role_session_duration=self.iam_role_session_duration, - profile=self.profile, - sceptre_user_data=self.sceptre_user_data, - parameters=self.parameters, - hooks=self.hooks, - s3_details=self.s3_details, - dependencies=self.dependencies, - role_arn=self.role_arn, - protected=self.protected, - tags=self.tags, - external_name=self.external_name, - notifications=self.notifications, - on_failure=self.on_failure, - stack_timeout=self.stack_timeout, - stack_group_config=self.stack_group_config - ) + f"name='{self.name}', " + f"project_code={self.project_code}, " + f"template_path={self.template_path}, " + f"template_handler_config={self.template_handler_config}, " + f"region={self.region}, " + f"template_bucket_name={self.template_bucket_name}, " + f"template_key_prefix={self.template_key_prefix}, " + f"required_version={self.required_version}, " + f"iam_role={self.iam_role}, " + f"iam_role_session_duration={self.iam_role_session_duration}, " + f"profile={self.profile}, " + f"sceptre_user_data={self.sceptre_user_data}, " + f"parameters={self.parameters}, " + f"hooks={self.hooks}, " + f"s3_details={self.s3_details}, " + f"dependencies={self.dependencies}, " + f"role_arn={self.role_arn}, " + f"protected={self.protected}, " + f"tags={self.tags}, " + f"external_name={self.external_name}, " + f"notifications={self.notifications}, " + f"on_failure={self.on_failure}, " + f"stack_timeout={self.stack_timeout}, " + f"stack_group_config={self.stack_group_config}, " + f"ignore={self.ignore}, " + f"obsolete={self.obsolete}" + ")" ) def __str__(self): return self.name def __eq__(self, stack): + # We should not use any resolvable properties in __eq__, since it is used when adding the + # Stack to a set, which is done very early in plan resolution. Trying to reference resolvers + # before the plan is fully resolved can potentially blow up. return ( self.name == stack.name and + self.external_name == stack.external_name and self.project_code == stack.project_code and self.template_path == stack.template_path and - self.template_handler_config == stack.template_handler_config and self.region == stack.region and - self.template_bucket_name == stack.template_bucket_name and self.template_key_prefix == stack.template_key_prefix and self.required_version == stack.required_version and - self.iam_role == stack.iam_role and self.iam_role_session_duration == stack.iam_role_session_duration and self.profile == stack.profile and - self.sceptre_user_data == stack.sceptre_user_data and - self.parameters == stack.parameters and - self.hooks == stack.hooks and - self.s3_details == stack.s3_details and self.dependencies == stack.dependencies and - self.role_arn == stack.role_arn and self.protected == stack.protected and - self.tags == stack.tags and - self.external_name == stack.external_name and - self.notifications == stack.notifications and self.on_failure == stack.on_failure and self.stack_timeout == stack.stack_timeout and - self.stack_group_config == stack.stack_group_config + self.ignore == stack.ignore and + self.obsolete == stack.obsolete ) def __hash__(self): diff --git a/tests/test_actions.py b/tests/test_actions.py index 62d76fd1a..3b3b5d129 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -1,21 +1,22 @@ # -*- coding: utf-8 -*- import datetime import json +from unittest.mock import patch, sentinel, Mock, call import pytest from botocore.exceptions import ClientError from dateutil.tz import tzutc -from unittest.mock import patch, sentinel, Mock, call -from sceptre.exceptions import CannotUpdateFailedStackError -from sceptre.exceptions import ProtectedStackError -from sceptre.exceptions import StackDoesNotExistError -from sceptre.exceptions import UnknownStackChangeSetStatusError -from sceptre.exceptions import UnknownStackStatusError +from sceptre.exceptions import ( + CannotUpdateFailedStackError, + ProtectedStackError, + StackDoesNotExistError, + UnknownStackChangeSetStatusError, + UnknownStackStatusError +) from sceptre.plan.actions import StackActions from sceptre.stack import Stack -from sceptre.stack_status import StackChangeSetStatus -from sceptre.stack_status import StackStatus +from sceptre.stack_status import StackChangeSetStatus, StackStatus from sceptre.template import Template diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_cli.py b/tests/test_cli/test_cli_commands.py similarity index 99% rename from tests/test_cli.py rename to tests/test_cli/test_cli_commands.py index 864482bc3..a7cccb093 100644 --- a/tests/test_cli.py +++ b/tests/test_cli/test_cli_commands.py @@ -30,7 +30,7 @@ from sceptre.stack_status import StackStatus -class TestCli(object): +class TestCli: def setup_method(self, test_method): self.patcher_ConfigReader = patch("sceptre.plan.plan.ConfigReader") @@ -42,13 +42,17 @@ def setup_method(self, test_method): self.mock_config_reader = MagicMock(spec=ConfigReader) self.mock_stack_actions = MagicMock(spec=StackActions) - self.mock_stack = MagicMock(spec=Stack) + self.mock_stack = MagicMock( + spec=Stack, + region=None, + profile=None, + external_name='mock-stack-external', + dependencies=[], + ignore=False, + obsolete=False, + ) self.mock_stack.name = 'mock-stack' - self.mock_stack.region = None - self.mock_stack.profile = None - self.mock_stack.external_name = 'mock-stack-external' - self.mock_stack.dependencies = [] self.mock_config_reader.construct_stacks.return_value = \ set([self.mock_stack]), set([self.mock_stack]) diff --git a/tests/test_cli/test_launch.py b/tests/test_cli/test_launch.py new file mode 100644 index 000000000..e4bba240f --- /dev/null +++ b/tests/test_cli/test_launch.py @@ -0,0 +1,219 @@ +import functools +import itertools +from collections import defaultdict +from typing import Optional, List, Set +from unittest.mock import create_autospec, Mock + +import pytest + +from sceptre.cli.launch import Launcher +from sceptre.cli.prune import Pruner +from sceptre.context import SceptreContext +from sceptre.exceptions import DependencyDoesNotExistError +from sceptre.plan.plan import SceptrePlan +from sceptre.stack import Stack +from sceptre.stack_status import StackStatus + + +class FakePlan(SceptrePlan): + def __init__( + self, + context: SceptreContext, + command_stacks: Set[Stack], + all_stacks: Set[Stack], + statuses_to_return: dict, + ): + self.context = context + self.command = None + self.reverse = None + self.launch_order: Optional[List[Set[Stack]]] = None + + self.all_stacks = all_stacks + self.command_stacks = command_stacks + self.statuses_to_return = statuses_to_return + + self.executions = [] + + def _execute(self, *args): + self.executions.append( + (self.command, self.launch_order.copy(), args) + ) + return { + stack: self.statuses_to_return[stack] + for stack in self + } + + def _generate_launch_order(self, reverse=False) -> List[Set[Stack]]: + launch_order = [self.command_stacks] + if self.context.ignore_dependencies: + return launch_order + + all_stacks = list(self.all_stacks) + + for start_index in range(0, len(all_stacks), 2): + chunk = { + stack for stack in all_stacks[start_index: start_index + 2] + if stack not in self.command_stacks + and self._has_dependency_on_a_command_stack(stack) + } + if len(chunk): + launch_order.append(chunk) + + return launch_order + + @functools.lru_cache() + def _has_dependency_on_a_command_stack(self, stack): + if len(self.command_stacks.intersection(stack.dependencies)): + return True + + for dependency in stack.dependencies: + if self._has_dependency_on_a_command_stack(dependency): + return True + + return False + + +class TestLauncher: + + def setup_method(self, test_method): + self.plans: List[FakePlan] = [] + + self.context = SceptreContext( + project_path="project", + command_path="my-test-group", + ) + self.cloned_context = self.context.clone() + # Since contexts don't have a __eq__ method, you can't assert easily off the result of + # clone without some hijinks. + self.context = Mock(wraps=self.context, **{ + 'clone.return_value': self.cloned_context, + 'ignore_dependencies': False + }) + + self.all_stacks = [ + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]) + ] + for index, stack in enumerate(self.all_stacks): + stack.name = f'stacks/stack-{index}.yaml' + + self.command_stacks = list(self.all_stacks) + + self.statuses_to_return = defaultdict(lambda: StackStatus.COMPLETE) + + self.fake_pruner = Mock(spec=Pruner, **{ + 'prune.return_value': 0 + }) + + self.plan_factory = create_autospec(SceptrePlan) + self.plan_factory.side_effect = self.fake_plan_factory + self.pruner_factory = create_autospec(Pruner) + self.pruner_factory.return_value = self.fake_pruner + + self.launcher = Launcher(self.context, self.plan_factory, self.pruner_factory) + + def fake_plan_factory(self, sceptre_context): + fake_plan = FakePlan( + sceptre_context, + set(self.command_stacks), + set(self.all_stacks), + self.statuses_to_return + ) + self.plans.append(fake_plan) + return fake_plan + + def get_executed_stacks(self, plan_number: int): + launch_order = self.plans[plan_number].executions[0][1] + return list(itertools.chain.from_iterable(launch_order)) + + def test_launch__launches_stacks_that_are_neither_ignored_nor_obsolete(self): + assert all(not s.ignore and not s.obsolete for s in self.all_stacks) + self.command_stacks = self.all_stacks + self.launcher.launch(True) + launched_stacks = set(self.get_executed_stacks(0)) + expected_stacks = set(self.all_stacks) + assert expected_stacks == launched_stacks + assert self.plans[0].executions[0][0] == "launch" + assert len(self.plans[0].executions) == 1 + + def test_launch__prune__no_obsolete_stacks__does_not_delete_any_stacks(self): + assert all(not s.obsolete for s in self.all_stacks) + self.launcher.launch(True) + assert len(self.plans) == 1 + assert self.plans[0].executions[0][0] == "launch" + + def test_launch__prune__instantiates_and_invokes_pruner(self): + self.launcher.launch(True) + self.fake_pruner.prune.assert_any_call() + + def test_launch__no_prune__obsolete_stacks__does_not_delete_any_stacks(self): + self.all_stacks[4].obsolete = True + self.all_stacks[5].obsolete = True + + self.launcher.launch(False) + assert len(self.plans) == 1 + assert self.plans[0].executions[0][0] == "launch" + + @pytest.mark.parametrize("prune", [ + pytest.param(True, id="prune"), + pytest.param(False, id="no prune") + ]) + def test_launch__returns_0(self, prune): + assert all(not s.ignore and not s.obsolete for s in self.all_stacks) + result = self.launcher.launch(prune) + + assert result == 0 + + def test_launch__does_not_launch_stacks_that_should_be_excluded(self): + self.all_stacks[4].ignore = True + self.all_stacks[5].obsolete = True + + self.launcher.launch(True) + + launched_stacks = set(self.get_executed_stacks(0)) + expected_stacks = {s for i, s in enumerate(self.all_stacks) if i not in (4, 5)} + assert expected_stacks == launched_stacks + assert self.plans[0].executions[0][0] == "launch" + + def test_launch__prune__stack_with_dependency_marked_obsolete__raises_dependency_does_not_exist_error(self): + self.all_stacks[0].obsolete = True + self.all_stacks[1].dependencies.append(self.all_stacks[0]) + + self.command_stacks = [self.all_stacks[0]] + + with pytest.raises(DependencyDoesNotExistError): + self.launcher.launch(True) + + def test_launch__prune__ignore_dependencies__stack_with_dependency_marked_obsolete__raises_no_error(self): + self.all_stacks[0].obsolete = True + self.all_stacks[1].dependencies.append(self.all_stacks[0]) + + self.command_stacks = [self.all_stacks[0]] + self.context.ignore_dependencies = True + self.launcher.launch(True) + + def test_launch__no_prune__does_not_raise_error(self): + self.all_stacks[0].obsolete = True + self.all_stacks[1].dependencies.append(self.all_stacks[0]) + self.launcher.launch(False) + + def test_launch__stacks_are_pruned__delete_and_deploy_actions_succeed__returns_0(self): + self.all_stacks[0].obsolete = True + + code = self.launcher.launch(True) + assert code == 0 + + def test_launch__pruner_returns_nonzero__returns_nonzero(self): + self.fake_pruner.prune.return_value = 99 + + code = self.launcher.launch(True) + assert code == 99 + + def test_launch__deploy_action_fails__returns_nonzero(self): + self.statuses_to_return[self.all_stacks[3]] = StackStatus.FAILED + code = self.launcher.launch(False) + assert code != 0 diff --git a/tests/test_cli/test_prune.py b/tests/test_cli/test_prune.py new file mode 100644 index 000000000..70cf390b0 --- /dev/null +++ b/tests/test_cli/test_prune.py @@ -0,0 +1,209 @@ +import functools +import itertools +from collections import defaultdict +from typing import Set, Optional, List +from unittest.mock import Mock, create_autospec + +import pytest + +from sceptre.cli.prune import Pruner, PATH_FOR_WHOLE_PROJECT +from sceptre.context import SceptreContext +from sceptre.exceptions import CannotPruneStackError +from sceptre.plan.plan import SceptrePlan +from sceptre.stack import Stack +from sceptre.stack_status import StackStatus + + +class FakePlan(SceptrePlan): + def __init__( + self, + context: SceptreContext, + command_stacks: Set[Stack], + all_stacks: Set[Stack], + statuses_to_return: dict, + ): + self.context = context + self.command = None + self.reverse = None + self.launch_order: Optional[List[Set[Stack]]] = None + + self.all_stacks = self.graph = all_stacks + self.command_stacks = command_stacks + self.statuses_to_return = statuses_to_return + + self.executions = [] + + def _execute(self, *args): + self.executions.append( + (self.command, self.launch_order.copy(), args) + ) + return { + stack: self.statuses_to_return[stack] + for stack in self + } + + def _generate_launch_order(self, reverse=False) -> List[Set[Stack]]: + launch_order = [self.command_stacks] + if self.context.ignore_dependencies: + return launch_order + + all_stacks = list(self.all_stacks) + + for start_index in range(0, len(all_stacks), 2): + chunk = { + stack for stack in all_stacks[start_index: start_index + 2] + if stack not in self.command_stacks + and self._has_dependency_on_a_command_stack(stack) + } + if len(chunk): + launch_order.append(chunk) + + return launch_order + + @functools.lru_cache() + def _has_dependency_on_a_command_stack(self, stack): + if len(self.command_stacks.intersection(stack.dependencies)): + return True + + for dependency in stack.dependencies: + if self._has_dependency_on_a_command_stack(dependency): + return True + + return False + + +class TestPruner: + + def setup_method(self, test_method): + self.plans: List[FakePlan] = [] + + self.context = SceptreContext( + project_path="project", + command_path=PATH_FOR_WHOLE_PROJECT, + ) + + self.all_stacks = [ + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]), + Mock(spec=Stack, ignore=False, obsolete=False, dependencies=[]) + ] + for index, stack in enumerate(self.all_stacks): + stack.name = f'stacks/stack-{index}.yaml' + + self.command_stacks = [ + self.all_stacks[2], + self.all_stacks[4] + ] + + self.statuses_to_return = defaultdict(lambda: StackStatus.COMPLETE) + + self.plan_factory = create_autospec(SceptrePlan) + self.plan_factory.side_effect = self.fake_plan_factory + + self.pruner = Pruner(self.context, self.plan_factory) + + def fake_plan_factory(self, sceptre_context): + fake_plan = FakePlan( + sceptre_context, + set(self.command_stacks), + set(self.all_stacks), + self.statuses_to_return + ) + self.plans.append(fake_plan) + return fake_plan + + @property + def executed_stacks(self): + assert len(self.plans) == 1 + launch_order = self.plans[0].executions[0][1] + return list(itertools.chain.from_iterable(launch_order)) + + def test_prune__no_obsolete_stacks__returns_zero(self): + code = self.pruner.prune() + assert code == 0 + + def test_prune__no_obsolete_stacks__does_not_call_command_on_plan(self): + self.pruner.prune() + assert len(self.plans[0].executions) == 0 + + def test_prune__whole_project__obsolete_stacks__deletes_all_obsolete_stacks(self): + self.all_stacks[4].obsolete = True + self.all_stacks[5].obsolete = True + + self.pruner.prune() + + assert self.plans[0].executions[0][0] == 'delete' + assert set(self.executed_stacks) == {self.all_stacks[4], self.all_stacks[5]} + + def test_prune__command_path__obsolete_stacks__deletes_only_obsolete_stacks_on_path(self): + self.all_stacks[4].obsolete = True # On command path + self.all_stacks[5].obsolete = True # not on command path + self.context.command_path = "my/command/path" + self.pruner.prune() + + assert self.plans[0].executions[0][0] == 'delete' + assert set(self.executed_stacks) == {self.all_stacks[4]} + + def test_prune__obsolete_stacks__returns_zero(self): + self.all_stacks[4].obsolete = True + self.all_stacks[5].obsolete = True + + code = self.pruner.prune() + assert code == 0 + + def test_prune__obsolete_stacks_depend_on_other_obsolete_stacks__deletes_only_obsolete_stacks(self): + self.all_stacks[1].obsolete = True + self.all_stacks[3].obsolete = True + self.all_stacks[4].obsolete = True + self.all_stacks[5].obsolete = True + self.all_stacks[3].dependencies.append(self.all_stacks[1]) + self.all_stacks[4].dependencies.append(self.all_stacks[3]) + self.all_stacks[5].dependencies.append(self.all_stacks[3]) + + self.pruner.prune() + + assert self.plans[0].executions[0][0] == 'delete' + assert set(self.executed_stacks) == { + self.all_stacks[1], + self.all_stacks[3], + self.all_stacks[4], + self.all_stacks[5], + } + + def test_prune__non_obsolete_stacks_depend_on_obsolete_stacks__raises_cannot_prune_stack_error(self): + self.all_stacks[1].obsolete = True + self.all_stacks[3].obsolete = False + self.all_stacks[4].obsolete = False + self.all_stacks[5].obsolete = False + self.all_stacks[3].dependencies.append(self.all_stacks[1]) + self.all_stacks[4].dependencies.append(self.all_stacks[3]) + self.all_stacks[5].dependencies.append(self.all_stacks[3]) + + with pytest.raises(CannotPruneStackError): + self.pruner.prune() + + def test_prune__non_obsolete_stacks_depend_on_obsolete_stacks__ignore_dependencies__deletes_obsolete_stacks(self): + self.all_stacks[1].obsolete = True + self.all_stacks[3].obsolete = False + self.all_stacks[4].obsolete = False + self.all_stacks[5].obsolete = False + self.all_stacks[3].dependencies.append(self.all_stacks[1]) + self.all_stacks[4].dependencies.append(self.all_stacks[3]) + self.all_stacks[5].dependencies.append(self.all_stacks[3]) + self.context.ignore_dependencies = True + self.pruner.prune() + + assert self.plans[0].executions[0][0] == 'delete' + assert set(self.executed_stacks) == { + self.all_stacks[1], + } + + def test_prune__delete_action_fails__returns_nonzero(self): + self.all_stacks[1].obsolete = True + self.statuses_to_return[self.all_stacks[1]] = StackStatus.FAILED + + code = self.pruner.prune() + assert code != 0 diff --git a/tests/test_config_reader.py b/tests/test_config_reader.py index fce25732e..b592afccc 100644 --- a/tests/test_config_reader.py +++ b/tests/test_config_reader.py @@ -1,21 +1,23 @@ # -*- coding: utf-8 -*- +import errno import os from unittest.mock import patch, sentinel, MagicMock + import pytest import yaml -import errno - -from sceptre.context import SceptreContext -from sceptre.exceptions import DependencyDoesNotExistError -from sceptre.exceptions import VersionIncompatibleError -from sceptre.exceptions import ConfigFileNotFoundError -from sceptre.exceptions import InvalidSceptreDirectoryError -from sceptre.exceptions import InvalidConfigFileError - -from freezegun import freeze_time from click.testing import CliRunner +from freezegun import freeze_time + from sceptre.config.reader import ConfigReader +from sceptre.context import SceptreContext +from sceptre.exceptions import ( + DependencyDoesNotExistError, + VersionIncompatibleError, + ConfigFileNotFoundError, + InvalidSceptreDirectoryError, + InvalidConfigFileError +) class TestConfigReader(object): @@ -324,6 +326,8 @@ def test_construct_stacks_constructs_stack( required_version='>1.0', template_bucket_name='stack_group_template_bucket_name', template_key_prefix=None, + ignore=False, + obsolete=False, stack_group_config={ "project_path": self.context.project_path, "custom_key": "custom_value" diff --git a/tests/test_context.py b/tests/test_context.py index 62a523358..fc7ca694c 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -66,3 +66,25 @@ def test_full_templates_path_returns_correct_path(self): ) full_templates_path = path.join("project_path", self.templates_path) assert context.full_templates_path() == full_templates_path + + def test_clone__returns_full_clone_of_context(self): + context = SceptreContext( + project_path="project_path", + command_path="command", + user_variables={"user": "variables"}, + options={"hello": "there"}, + output_format=sentinel.output_format, + no_colour=sentinel.no_colour, + ignore_dependencies=sentinel.ignore_dependencies + ) + clone = context.clone() + assert clone is not context + assert clone.project_path == context.project_path + assert clone.command_path == context.command_path + assert clone.user_variables == context.user_variables + assert clone.user_variables is not context.user_variables + assert clone.options == context.options + assert clone.options is not context.options + assert clone.output_format == context.output_format + assert clone.no_colour == context.no_colour + assert clone.ignore_dependencies == context.ignore_dependencies diff --git a/tests/test_stack.py b/tests/test_stack.py index 5bb9b6ed9..d8a7d8ba9 100644 --- a/tests/test_stack.py +++ b/tests/test_stack.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- -import importlib +from unittest.mock import MagicMock, sentinel import pytest -from unittest.mock import MagicMock, sentinel from sceptre.exceptions import InvalidConfigFileError from sceptre.resolvers import Resolver @@ -62,7 +61,7 @@ def setup_method(self, test_method): ) self.stack._template = MagicMock(spec=Template) - def test_initiate_stack_with_template_path(self): + def test_initialize_stack_with_template_path(self): stack = Stack( name='dev/stack/app', project_code=sentinel.project_code, template_path=sentinel.template_path, @@ -93,7 +92,7 @@ def test_initiate_stack_with_template_path(self): assert stack.on_failure is None assert stack.stack_group_config == {} - def test_initiate_stack_with_template_handler(self): + def test_initialize_stack_with_template_handler(self): stack = Stack( name='dev/stack/app', project_code=sentinel.project_code, template_handler_config=sentinel.template_handler_config, @@ -132,6 +131,30 @@ def test_raises_exception_if_path_and_handler_configured(self): region="region" ) + def test_init__non_boolean_ignore_value__raises_invalid_config_file_error(self): + with pytest.raises(InvalidConfigFileError): + Stack( + name='dev/stack/app', project_code=sentinel.project_code, + template_handler_config=sentinel.template_handler_config, + template_bucket_name=sentinel.template_bucket_name, + template_key_prefix=sentinel.template_key_prefix, + required_version=sentinel.required_version, + region=sentinel.region, external_name=sentinel.external_name, + ignore="true" + ) + + def test_init__non_boolean_obsolete_value__raises_invalid_config_file_error(self): + with pytest.raises(InvalidConfigFileError): + Stack( + name='dev/stack/app', project_code=sentinel.project_code, + template_handler_config=sentinel.template_handler_config, + template_bucket_name=sentinel.template_bucket_name, + template_key_prefix=sentinel.template_key_prefix, + required_version=sentinel.required_version, + region=sentinel.region, external_name=sentinel.external_name, + obsolete="true" + ) + def test_stack_repr(self): assert self.stack.__repr__() == \ "sceptre.stack.Stack(" \ @@ -158,21 +181,11 @@ def test_stack_repr(self): "notifications=[sentinel.notification], " \ "on_failure=sentinel.on_failure, " \ "stack_timeout=sentinel.stack_timeout, " \ - "stack_group_config={}" \ + "stack_group_config={}, " \ + "ignore=False, " \ + "obsolete=False" \ ")" - def test_repr_can_eval_correctly(self): - sceptre = importlib.import_module('sceptre') - evaluated_stack = eval( - repr(self.stack), - { - 'sceptre': sceptre, - 'sentinel': sentinel - } - ) - assert isinstance(evaluated_stack, Stack) - assert evaluated_stack.__eq__(self.stack) - def test_configuration_manager__iam_role_raises_recursive_resolve__returns_connection_manager_with_no_role(self): class FakeResolver(Resolver): def resolve(self):