From 4580b5156f9d8e69debb5a5d48b50e7576b373b6 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Mon, 4 Nov 2019 19:32:16 +0000 Subject: [PATCH 01/14] initial failing test case for SSM backup --- .gitignore | 34 ++++++++++ .pre-commit-config.yaml | 30 +++++++++ Makefile | 10 +++ README.md | 1 + backup_cloud/__init__.py | 0 backup_cloud/aws_ssm_dict.py | 100 ++++++++++++++++++++++++++++ backup_cloud/backup_aws_ssm.py | 6 ++ features/backup-restore-ssm.feature | 11 +++ features/environment.py | 40 +++++++++++ features/steps/ssm-backup.py | 30 +++++++++ 10 files changed, 262 insertions(+) create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 Makefile create mode 100644 README.md create mode 100644 backup_cloud/__init__.py create mode 100644 backup_cloud/aws_ssm_dict.py create mode 100644 backup_cloud/backup_aws_ssm.py create mode 100644 features/backup-restore-ssm.feature create mode 100644 features/environment.py create mode 100644 features/steps/ssm-backup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7c1ed6b --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +*.retry +*~ +*venv +aws_credentials* +*password* +\#* +*environment +*postfix +.#* +*.dat +*.pub +gpgtemp +keys/ +build +*.log +dump.sql* +*.pri +mockstmp +.anslk_* +.prepare* +test-config* +__pycache__ +.mypy_cache +*.gpg +bin/backup_encrypt +*.pyc +dist +MANIFEST +deploy_key +deploy_key.bck +encrypted_build_files +encrypted_build_files.tjz +*.egg-info +*.makestamp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..8b61c31 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +exclude: '^$' +fail_fast: false +repos: +- repo: https://github.com/ambv/black + rev: stable + hooks: + - id: black + language_version: python3.7 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v1.2.3 + hooks: + - id: check-added-large-files + - id: check-json + - id: detect-private-key + - id: end-of-file-fixer + - id: flake8 + exclude: '^features/.*_steps/*' + args: + - --ignore=W503,E402,E501 + - --max-line-length=131 + - id: forbid-new-submodules + - id: check-yaml +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'master' # Use the sha / tag you want to point at + hooks: + - id: mypy +# - repo: https://github.com/Lucas-C/pre-commit-hooks-go +# sha: v1.0.0 +# hooks: +# - id: checkmake diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9d36442 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +test: + behave + +wip: + behave --wip + + +lint: + pre-commit install --install-hooks + pre-commit run -a diff --git a/README.md b/README.md new file mode 100644 index 0000000..6308460 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +backup SSM parameter store to a diff --git a/backup_cloud/__init__.py b/backup_cloud/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backup_cloud/aws_ssm_dict.py b/backup_cloud/aws_ssm_dict.py new file mode 100644 index 0000000..f73e474 --- /dev/null +++ b/backup_cloud/aws_ssm_dict.py @@ -0,0 +1,100 @@ +import boto3 +from hamcrest import assert_that, equal_to +from typing import Dict, Tuple, Union +from collections.abc import MutableMapping + + +class aws_ssm_dict(MutableMapping): + """provide a flat dictionary with access to AWS SSM parameters + + aws_ssm_dict provides a flat dictionary which allows access to AWS + SSM parameters. The key is the complete path to the parameter + name, the value is the value of the parameter. + + When the dictionary is created it can either deliver the raw + parmeter contents or it can decrypt them. The default is to decryptb + + """ + + def __init__(self, decrypt=True, return_type="value"): + self.ssm = boto3.client("ssm") + self.decrypt = decrypt + self.return_type = return_type + + def __setitem__(self, key: str, value: Union[str, Tuple]): + """set the SSM parameter, to match a value + + given a string we use that by default as a securestring + given a tuple we treat the first parameter as a type, the second as a value and optional third as a description + """ + + description = None + if isinstance(value, tuple): + param_type = value[0] + val_string = value[1] + try: + description = value[2] + except IndexError: + pass + else: + param_type = "SecureString" + val_string = value + + request_params = dict(Name=key, Type=param_type, Value=val_string) + if description: + request_params.update(dict(Description=description)) + + self.ssm.put_parameter(**request_params) + + def __delitem__(self, key): + request_params = dict(Name=key) + self.ssm.delete_parameter(**request_params) + + def __iter__(self): + pass + + def __len__(self): + pass + + def _return_as_tuple(self, key: str): + raise + pass + + def _return_as_value(self, key: str): + try: + response = self.ssm.get_parameter(Name=key, WithDecryption=self.decrypt) + except self.ssm.exceptions.ParameterNotFound as e: + raise KeyError(e) + assert response["Parameter"]["Name"] == key + return response["Parameter"]["Value"] + + def __getitem__(self, key: str): + if self.return_type == "tuple": + return self._return_as_tuple(key) + else: + return self._return_as_value(key) + + def upload_dictionary(self, my_dict: Dict[str, str]): + for i in my_dict.keys(): + self[i] = my_dict[i] + + def remove_dictionary(self, my_dict: Dict[str, str]): + keys = [x for x in my_dict.keys()] + for i in keys: + self.pop(i, None) + + def verify_dictionary(self, my_dict: Dict[str, str]): + for i in my_dict.keys(): + value = self[i] + assert_that(value, equal_to(my_dict[i])) + + def verify_deleted_dictionary(self, my_dict: Dict[str, str]): + except_count = 0 + for i in my_dict.keys(): + try: + self[i] + except KeyError: + except_count += 1 + continue + assert False, "still found key: " + i + " in dictionary" + assert except_count == len(my_dict.keys()) diff --git a/backup_cloud/backup_aws_ssm.py b/backup_cloud/backup_aws_ssm.py new file mode 100644 index 0000000..b34ac0e --- /dev/null +++ b/backup_cloud/backup_aws_ssm.py @@ -0,0 +1,6 @@ +def backup_to_file(filename): + pass + + +def restore_from_file(filename): + pass diff --git a/features/backup-restore-ssm.feature b/features/backup-restore-ssm.feature new file mode 100644 index 0000000..956ca79 --- /dev/null +++ b/features/backup-restore-ssm.feature @@ -0,0 +1,11 @@ +Feature: backup SSM parameter store +In order to ensure that I can recover my SSM parameters even + + @wip + @fixture.ssm_params + Scenario: backup SSM to a plaintext file + Given that I have some parameters in SSM parameter store + And that I have backed up those parmameters + When I delete those parameters from SSM parameter store + And I run my restore script + Then those parameters should be in SSM parameter store diff --git a/features/environment.py b/features/environment.py new file mode 100644 index 0000000..c3cfa2c --- /dev/null +++ b/features/environment.py @@ -0,0 +1,40 @@ +import random +import string +from behave import fixture +from behave.fixture import use_fixture_by_tag +from backup_cloud.aws_ssm_dict import aws_ssm_dict + + +@fixture +def setup_ssm_parameters(context): + ssm_dict = context.ssm_dict = aws_ssm_dict() + params = context.test_params = { + "".join( + [random.choice(string.ascii_letters + string.digits) for n in range(16)] + ): "".join( + [random.choice(string.ascii_letters + string.digits) for n in range(16)] + ), + "".join( + [random.choice(string.ascii_letters + string.digits) for n in range(16)] + ): "".join( + [random.choice(string.ascii_letters + string.digits) for n in range(16)] + ), + "".join( + [random.choice(string.ascii_letters + string.digits) for n in range(16)] + ): "".join( + [random.choice(string.ascii_letters + string.digits) for n in range(16)] + ), + } + aws_ssm_dict.upload_dictionary(ssm_dict, params) + yield (True) + aws_ssm_dict.remove_dictionary(ssm_dict, params) + aws_ssm_dict.verify_deleted_dictionary(ssm_dict, params) + + +# -- REGISTRY DATA SCHEMA 1: fixture_func +fixture_registry1 = {"fixture.ssm_params": setup_ssm_parameters} + + +def before_tag(context, tag): + if tag.startswith("fixture."): + return use_fixture_by_tag(tag, context, fixture_registry1) diff --git a/features/steps/ssm-backup.py b/features/steps/ssm-backup.py new file mode 100644 index 0000000..df29258 --- /dev/null +++ b/features/steps/ssm-backup.py @@ -0,0 +1,30 @@ +from backup_cloud import backup_aws_ssm +from tempfile import NamedTemporaryFile +from behave import given, when, then + + +@given(u"that I have some parameters in SSM parameter store") +def step_impl_1(context): + context.ssm_dict.verify_dictionary(context.test_params) + + +@given(u"that I have backed up those parmameters") +def step_impl_2(context): + t = context.backup_temp_file = NamedTemporaryFile() + backup_aws_ssm.backup_to_file(t.name) + + +@when(u"I delete those parameters from SSM parameter store") +def step_impl_3(context): + context.ssm_dict.remove_dictionary(context.test_params) + context.ssm_dict.verify_deleted_dictionary(context.test_params) + + +@when(u"I run my restore script") +def step_impl_4(context): + backup_aws_ssm.restore_from_file(context.backup_temp_file.name) + + +@then(u"those parameters should be in SSM parameter store") +def step_impl_5(context): + context.ssm_dict.verify_dictionary(context.test_params) From e63677ea325e519c79f4b59a3e0589a047e155b7 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Wed, 6 Nov 2019 00:04:06 +0000 Subject: [PATCH 02/14] passing backup/restore test as python function --- backup_cloud/aws_ssm_dict.py | 48 +++++++++++++++++++++++++++++++--- backup_cloud/backup_aws_ssm.py | 17 ++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/backup_cloud/aws_ssm_dict.py b/backup_cloud/aws_ssm_dict.py index f73e474..8b26b7a 100644 --- a/backup_cloud/aws_ssm_dict.py +++ b/backup_cloud/aws_ssm_dict.py @@ -16,8 +16,8 @@ class aws_ssm_dict(MutableMapping): """ - def __init__(self, decrypt=True, return_type="value"): - self.ssm = boto3.client("ssm") + def __init__(self, decrypt=True, return_type="value", region_name="eu-west-1"): + self.ssm = boto3.client("ssm", region_name=region_name) self.decrypt = decrypt self.return_type = return_type @@ -50,8 +50,50 @@ def __delitem__(self, key): request_params = dict(Name=key) self.ssm.delete_parameter(**request_params) + def get_tuple_for_param(self, name): + try: + get_response = self.ssm.get_parameter( + Name=name, WithDecryption=self.decrypt + ) + except self.ssm.exceptions.ParameterNotFound as e: + raise KeyError(e) + try: + describe_response = self.ssm.describe_parameter(Name=name) + except self.ssm.exceptions.ParameterNotFound as e: + raise KeyError(e) + return ( + get_response["Parameter"]["Type"], + get_response["Parameter"]["Value"], + describe_response["Parameter"]["Description"], + ) + + def iterate_parameter_list(self): + paginator = self.ssm.get_paginator("get_parameters_by_path") + page_iterator = paginator.paginate( + Path="/", Recursive=True, WithDecryption=True + ) + for page in page_iterator: + for i in page["Parameters"]: + yield i + + def iterate_parameter_descriptions(self): + paginator = self.ssm.get_paginator("describe_parameters") + page_iterator = paginator.paginate() + for page in page_iterator: + for i in page["Parameters"]: + yield i["Name"] + + def iterate_for_tuples(self): + for i in self.iterate_parameter_descriptions(): + name = i["Name"] + yield (name, self.get_tuple_for_param(name)) + + def iterate_for_values(self): + for i in self.iterate_parameter_list(): + yield (i["Name"], i["Value"]) + def __iter__(self): - pass + yield from self.iterate_parameter_descriptions() def __len__(self): pass diff --git a/backup_cloud/backup_aws_ssm.py b/backup_cloud/backup_aws_ssm.py index b34ac0e..f12de3f 100644 --- a/backup_cloud/backup_aws_ssm.py +++ b/backup_cloud/backup_aws_ssm.py @@ -1,6 +1,19 @@ +from backup_cloud.aws_ssm_dict import aws_ssm_dict +import json + + def backup_to_file(filename): - pass + ssm_dict = aws_ssm_dict() + contents = {} + for i in ssm_dict.keys(): + contents[i] = ssm_dict[i] + with open(filename, "w") as f: + json.dump(contents, f) def restore_from_file(filename): - pass + ssm_dict = aws_ssm_dict() + with open(filename) as f: + contents = json.load(f) + for i in contents.keys(): + ssm_dict[i] = contents[i] From e58ccef5f6bf43055c88d5bee347b50761619a41 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Thu, 7 Nov 2019 16:20:13 +0000 Subject: [PATCH 03/14] first running version of CLI - very basic --- Makefile | 23 +++++++++-- README.md | 3 +- backup_cloud/__init__.py | 0 backup_cloud/backup_aws_ssm.py | 19 --------- backup_cloud_ssm/__init__.py | 4 ++ backup_cloud_ssm/aws_ssm_cli.py | 17 ++++++++ .../aws_ssm_dict.py | 1 + backup_cloud_ssm/backup_aws_ssm.py | 40 +++++++++++++++++++ features/backup-restore-ssm.feature | 26 ++++++++++-- features/environment.py | 36 +++++++++++++---- features/steps/ssm-backup.py | 30 ++++++++++++-- setup.py | 26 ++++++++++++ 12 files changed, 189 insertions(+), 36 deletions(-) delete mode 100644 backup_cloud/__init__.py delete mode 100644 backup_cloud/backup_aws_ssm.py create mode 100644 backup_cloud_ssm/__init__.py create mode 100644 backup_cloud_ssm/aws_ssm_cli.py rename {backup_cloud => backup_cloud_ssm}/aws_ssm_dict.py (99%) create mode 100644 backup_cloud_ssm/backup_aws_ssm.py create mode 100644 setup.py diff --git a/Makefile b/Makefile index 9d36442..06a5a23 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,27 @@ -test: - behave +AWS_ACCOUNT_NAME ?= michael +AWS_DEFAULT_REGION ?= eu-west-1 +PYTHON ?= python3 +BEHAVE ?= behave +KEYFILE ?=.anslk_random_testkey -wip: +LIBFILES := $(shell find backup_cloud_ssm -name '*.py') + +test: develop + behave --tags ~@future + +wip: develop behave --wip lint: pre-commit install --install-hooks pre-commit run -a + + +# develop is needed to install scripts that are called during testing +develop: .develop.makestamp + +.develop.makestamp: setup.py backup_cloud_ssm/aws_ssm_cli.py $(LIBFILES) + $(PYTHON) setup.py install --force + $(PYTHON) setup.py develop + touch $@ diff --git a/README.md b/README.md index 6308460..53c7ecc 100644 --- a/README.md +++ b/README.md @@ -1 +1,2 @@ -backup SSM parameter store to a +Backup SSM parameter store to a file. Optional (but default) +encryption to be added. diff --git a/backup_cloud/__init__.py b/backup_cloud/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backup_cloud/backup_aws_ssm.py b/backup_cloud/backup_aws_ssm.py deleted file mode 100644 index f12de3f..0000000 --- a/backup_cloud/backup_aws_ssm.py +++ /dev/null @@ -1,19 +0,0 @@ -from backup_cloud.aws_ssm_dict import aws_ssm_dict -import json - - -def backup_to_file(filename): - ssm_dict = aws_ssm_dict() - contents = {} - for i in ssm_dict.keys(): - contents[i] = ssm_dict[i] - with open(filename, "w") as f: - json.dump(contents, f) - - -def restore_from_file(filename): - ssm_dict = aws_ssm_dict() - with open(filename) as f: - contents = json.load(f) - for i in contents.keys(): - ssm_dict[i] = contents[i] diff --git a/backup_cloud_ssm/__init__.py b/backup_cloud_ssm/__init__.py new file mode 100644 index 0000000..6bbec1f --- /dev/null +++ b/backup_cloud_ssm/__init__.py @@ -0,0 +1,4 @@ +from backup_cloud_ssm.backup_aws_ssm import ( # noqa: F401 + backup_to_file, + restore_from_file, +) diff --git a/backup_cloud_ssm/aws_ssm_cli.py b/backup_cloud_ssm/aws_ssm_cli.py new file mode 100644 index 0000000..c2dafc1 --- /dev/null +++ b/backup_cloud_ssm/aws_ssm_cli.py @@ -0,0 +1,17 @@ +import argparse +import sys +import backup_cloud_ssm + + +def main(): + parser = argparse.ArgumentParser(description="Backup AWS SSM Parameter Store") + parser.add_argument("--restore", help="restore from stdin", action="store_true") + args = parser.parse_args() + if args.restore: + backup_cloud_ssm.restore_from_file(sys.stdin) + else: + backup_cloud_ssm.backup_to_file(sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/backup_cloud/aws_ssm_dict.py b/backup_cloud_ssm/aws_ssm_dict.py similarity index 99% rename from backup_cloud/aws_ssm_dict.py rename to backup_cloud_ssm/aws_ssm_dict.py index 8b26b7a..d2ae02d 100644 --- a/backup_cloud/aws_ssm_dict.py +++ b/backup_cloud_ssm/aws_ssm_dict.py @@ -18,6 +18,7 @@ class aws_ssm_dict(MutableMapping): def __init__(self, decrypt=True, return_type="value", region_name="eu-west-1"): self.ssm = boto3.client("ssm", region_name=region_name) + self.exceptions = self.ssm.exceptions self.decrypt = decrypt self.return_type = return_type diff --git a/backup_cloud_ssm/backup_aws_ssm.py b/backup_cloud_ssm/backup_aws_ssm.py new file mode 100644 index 0000000..ef46b6a --- /dev/null +++ b/backup_cloud_ssm/backup_aws_ssm.py @@ -0,0 +1,40 @@ +from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict +import json +import logging +import sys + +logger = logging.getLogger() + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def backup_to_file(file): + ssm_dict = aws_ssm_dict() + contents = {} + for i in ssm_dict.keys(): + try: + contents[i] = ssm_dict[i] + except KeyError: + eprint("failed to find expected parameter:", i) + raise + try: + with open(file, "w") as f: + json.dump(contents, f) + except TypeError: + json.dump(contents, file) + + +def restore_from_file(file): + ssm_dict = aws_ssm_dict() + try: + with open(file) as f: + contents = json.load(f) + except TypeError: + contents = json.load(file) + for i in contents.keys(): + try: + ssm_dict[i] = contents[i] + except ssm_dict.exceptions.ParameterAlreadyExists: + logger.warning("Parameter " + i + " already exists!") diff --git a/features/backup-restore-ssm.feature b/features/backup-restore-ssm.feature index 956ca79..a2b9b36 100644 --- a/features/backup-restore-ssm.feature +++ b/features/backup-restore-ssm.feature @@ -1,11 +1,31 @@ Feature: backup SSM parameter store In order to ensure that I can recover my SSM parameters even - @wip + @fixture.preexist_params @fixture.ssm_params Scenario: backup SSM to a plaintext file - Given that I have some parameters in SSM parameter store + Given I have some parameters in SSM parameter store And that I have backed up those parmameters When I delete those parameters from SSM parameter store - And I run my restore script + And I restore those parameters Then those parameters should be in SSM parameter store + + + @wip + @fixture.preexist_params + @fixture.ssm_params + Scenario: provide backup from command line + Given I have some parameters in SSM parameter store + And I run the aws-ssm-backup command + When I delete those parameters from SSM parameter store + And I run the aws-ssm-backup command with the restore argument + Then those parameters should be in SSM parameter store + + + @future + Scenario: only warn for preexisting parameters if there is a mismatch + Given I have an existing parameter with the same value as in my backup + And I have an existing parameter with the a differnt value from my backup + When I run my restore without overwriting parameters + Then I should get a debug message about the matching parameter + And I should get a warning message about the other parameter diff --git a/features/environment.py b/features/environment.py index c3cfa2c..90c75f2 100644 --- a/features/environment.py +++ b/features/environment.py @@ -2,13 +2,11 @@ import string from behave import fixture from behave.fixture import use_fixture_by_tag -from backup_cloud.aws_ssm_dict import aws_ssm_dict +from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict -@fixture -def setup_ssm_parameters(context): - ssm_dict = context.ssm_dict = aws_ssm_dict() - params = context.test_params = { +def create_random_params(ssm_dict): + params = { "".join( [random.choice(string.ascii_letters + string.digits) for n in range(16)] ): "".join( @@ -25,14 +23,38 @@ def setup_ssm_parameters(context): [random.choice(string.ascii_letters + string.digits) for n in range(16)] ), } - aws_ssm_dict.upload_dictionary(ssm_dict, params) + ssm_dict.upload_dictionary(params) + return params + + +@fixture +def setup_ssm_parameters(context): + """parameters that will be deleted during the simulated disaster""" + ssm_dict = context.ssm_dict = aws_ssm_dict() + context.test_params = params = create_random_params(ssm_dict) + yield (True) + aws_ssm_dict.remove_dictionary(ssm_dict, params) + aws_ssm_dict.verify_deleted_dictionary(ssm_dict, params) + + +@fixture +def preexisting_ssm_parameters(context): + """\ + parameters that don't get deleted during disaster (or perhaps were + recreated manually after disaster but before restore) + """ + ssm_dict = context.ssm_dict = aws_ssm_dict() + context.preexist_params = params = create_random_params(ssm_dict) yield (True) aws_ssm_dict.remove_dictionary(ssm_dict, params) aws_ssm_dict.verify_deleted_dictionary(ssm_dict, params) # -- REGISTRY DATA SCHEMA 1: fixture_func -fixture_registry1 = {"fixture.ssm_params": setup_ssm_parameters} +fixture_registry1 = { + "fixture.ssm_params": setup_ssm_parameters, + "fixture.preexist_params": preexisting_ssm_parameters, +} def before_tag(context, tag): diff --git a/features/steps/ssm-backup.py b/features/steps/ssm-backup.py index df29258..601d87d 100644 --- a/features/steps/ssm-backup.py +++ b/features/steps/ssm-backup.py @@ -1,9 +1,10 @@ -from backup_cloud import backup_aws_ssm +from backup_cloud_ssm import backup_aws_ssm from tempfile import NamedTemporaryFile from behave import given, when, then +from subprocess import run, CalledProcessError -@given(u"that I have some parameters in SSM parameter store") +@given(u"I have some parameters in SSM parameter store") def step_impl_1(context): context.ssm_dict.verify_dictionary(context.test_params) @@ -20,7 +21,7 @@ def step_impl_3(context): context.ssm_dict.verify_deleted_dictionary(context.test_params) -@when(u"I run my restore script") +@when(u"I restore those parameters") def step_impl_4(context): backup_aws_ssm.restore_from_file(context.backup_temp_file.name) @@ -28,3 +29,26 @@ def step_impl_4(context): @then(u"those parameters should be in SSM parameter store") def step_impl_5(context): context.ssm_dict.verify_dictionary(context.test_params) + + +@given(u"I run the aws-ssm-backup command") +def step_impl_6(context): + call_args = ["aws-ssm-backup"] + try: + proc_res = run(call_args, capture_output=True, check=True) + context.backup_contents = proc_res.stdout + except CalledProcessError as e: + print("Failed backup stderr:", e.stderr, "stdout:", e.stdout) + raise + + +@when(u"I run the aws-ssm-backup command with the restore argument") +def step_impl_7(context): + call_args = ["aws-ssm-backup", "--restore"] + try: + context.restore_res = run( + call_args, capture_output=True, input=context.backup_contents, check=True + ) + except CalledProcessError as e: + print("Failed backup stderr:", e.stderr, "stdout:", e.stdout) + raise diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d4c290e --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages + +with open("README.md", "r") as fh: + long_description = fh.read() + +setup( + name="backup_ssm", + version="0.1", + author="Michael De La Rue", + author_email="michael-paddle@fake.github.com", + description="Backup your AWS SSM parameters - part of backup-cloud", + long_description=long_description, + url="https://github.com/backup-cloud/backup-ssm", + packages=find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Operating System :: OS Independent", + ], + entry_points={ + "console_scripts": ["aws-ssm-backup = backup_cloud_ssm.aws_ssm_cli:main"] + }, + install_requires=["boto3"], +) From ace38e46b44746bbc1a49a441c2b5ef9c3628b5f Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Fri, 8 Nov 2019 16:09:17 +0000 Subject: [PATCH 04/14] start adding type handling --- backup_cloud_ssm/test-parameter-storage.py | 2 + features/backup-restore-ssm.feature | 11 +++- features/environment.py | 61 ++++++++++++++++------ 3 files changed, 58 insertions(+), 16 deletions(-) create mode 100644 backup_cloud_ssm/test-parameter-storage.py diff --git a/backup_cloud_ssm/test-parameter-storage.py b/backup_cloud_ssm/test-parameter-storage.py new file mode 100644 index 0000000..9428dd4 --- /dev/null +++ b/backup_cloud_ssm/test-parameter-storage.py @@ -0,0 +1,2 @@ +def test_store_returns_input(): + pass diff --git a/features/backup-restore-ssm.feature b/features/backup-restore-ssm.feature index a2b9b36..8fa3e71 100644 --- a/features/backup-restore-ssm.feature +++ b/features/backup-restore-ssm.feature @@ -10,8 +10,17 @@ In order to ensure that I can recover my SSM parameters even And I restore those parameters Then those parameters should be in SSM parameter store - @wip + @fixture.preexist_params + @fixture.ssm_typed_params + Scenario: backup SSM to a plaintext file + Given I have some parameters in SSM parameter store + And that I have backed up those parmameters + When I delete those parameters from SSM parameter store + And I restore those parameters + Then those parameters should be in SSM parameter store + And the types of those parameters should be preserved + @fixture.preexist_params @fixture.ssm_params Scenario: provide backup from command line diff --git a/features/environment.py b/features/environment.py index 90c75f2..ac6637b 100644 --- a/features/environment.py +++ b/features/environment.py @@ -5,23 +5,43 @@ from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict +def randomword(length=16): + return "".join( + [random.choice(string.ascii_letters + string.digits) for n in range(16)] + ) + + +def randomlist(): + return ",".join([randomword(4), randomword(4), randomword(4), randomword(4)]) + + def create_random_params(ssm_dict): params = { - "".join( - [random.choice(string.ascii_letters + string.digits) for n in range(16)] - ): "".join( - [random.choice(string.ascii_letters + string.digits) for n in range(16)] - ), - "".join( - [random.choice(string.ascii_letters + string.digits) for n in range(16)] - ): "".join( - [random.choice(string.ascii_letters + string.digits) for n in range(16)] - ), - "".join( - [random.choice(string.ascii_letters + string.digits) for n in range(16)] - ): "".join( - [random.choice(string.ascii_letters + string.digits) for n in range(16)] - ), + randomword(): randomword(), + randomword(): randomword(), + randomword(): randomword(), + } + ssm_dict.upload_dictionary(params) + return params + + +def create_random_typed_params(ssm_dict): + params = { + randomword(): { + "Value": randomword(), + "Type": "String", + "Description": "aws-ssm-backup testing String parameter", + }, + randomword(): { + "Value": randomword(), + "Type": "SecureString", + "Description": "aws-ssm-backup testing String parameter", + }, + randomword(): { + "Value": randomlist(), + "Type": "StringList", + "Description": "aws-ssm-backup testing String parameter", + }, } ssm_dict.upload_dictionary(params) return params @@ -37,6 +57,16 @@ def setup_ssm_parameters(context): aws_ssm_dict.verify_deleted_dictionary(ssm_dict, params) +@fixture +def setup_typed_ssm_parameters(context): + """parameters that will be deleted during the simulated disaster""" + ssm_dict = context.ssm_dict = aws_ssm_dict() + context.test_params = params = create_random_typed_params(ssm_dict) + yield (True) + aws_ssm_dict.remove_dictionary(ssm_dict, params) + aws_ssm_dict.verify_deleted_dictionary(ssm_dict, params) + + @fixture def preexisting_ssm_parameters(context): """\ @@ -53,6 +83,7 @@ def preexisting_ssm_parameters(context): # -- REGISTRY DATA SCHEMA 1: fixture_func fixture_registry1 = { "fixture.ssm_params": setup_ssm_parameters, + "fixture.ssm_typed_params": setup_typed_ssm_parameters, "fixture.preexist_params": preexisting_ssm_parameters, } From fa01ae04b9e36e7503f6688e34c8b96d707833cd Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Tue, 12 Nov 2019 13:21:09 +0000 Subject: [PATCH 05/14] add type backup and restore --- .gitignore | 1 + Makefile | 12 ++- README.md | 52 ++++++++++ backup_cloud_ssm/aws_ssm_dict.py | 115 +++++++++++++++------ backup_cloud_ssm/backup_aws_ssm.py | 9 +- backup_cloud_ssm/test-parameter-storage.py | 2 - features/environment.py | 18 ++-- features/steps/ssm-backup.py | 41 +++++--- testing/test_parameter_storage.py | 90 ++++++++++++++++ 9 files changed, 278 insertions(+), 62 deletions(-) delete mode 100644 backup_cloud_ssm/test-parameter-storage.py create mode 100644 testing/test_parameter_storage.py diff --git a/.gitignore b/.gitignore index 7c1ed6b..f4e1661 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ encrypted_build_files encrypted_build_files.tjz *.egg-info *.makestamp +.hypothesis diff --git a/Makefile b/Makefile index 06a5a23..fcd9238 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,17 @@ KEYFILE ?=.anslk_random_testkey LIBFILES := $(shell find backup_cloud_ssm -name '*.py') -test: develop +all: lint test + +test: develop pytest behave + +behave: behave --tags ~@future +pytest: + pytest + + wip: develop behave --wip @@ -25,3 +33,5 @@ develop: .develop.makestamp $(PYTHON) setup.py install --force $(PYTHON) setup.py develop touch $@ + +.PHONY: all test behave pytest wip lint develop diff --git a/README.md b/README.md index 53c7ecc..1e8305e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,54 @@ Backup SSM parameter store to a file. Optional (but default) encryption to be added. + + +## Using CLI tools + +1) set up the appropriate environment including AWS variables + +2) to backup run + + aws-ssm-backup > + +3) to restore run + + aws-ssm-backup --restore > + +## Using python interface + + import backup_aws_ssm + backup_aws_ssm.backup_to_file("myfile") + + + import backup_aws_ssm + backup_aws_ssm.restore_from_file("myfile") + + +## Using python ssm library + +Included in the package is a library which provides a dict object +which accesses SSM parameter store. This will likely, later, be split out into a separate packge. In the meantime it can be used in Alpha testing mode. + + from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict + ssm_dict = aws_ssm_dict() + ssm_dict["parameter"] = "value" + print(ssm_dict["parameter"]) + + +## Development + +We aim to use Behavior Driven Development to encourage reasonable feature descriptions and a level of tests appropriate for the business functionality included here. Test Driven Development and to some extent Test Driven Design are encouraged in order to improve testability and eas of modification of the code. + +Some of the tests are designed to run against either the Moto library or a real AWS instance. By defining the variable:\ + + We aim to use Behavior Driven Development to encourage reasonable feature descriptions and a level of tests appropriate for the business functionality included here. Test Driven Development and to some extent Test Driven Design are encouraged in order to improve testability and eas of modification of the code. + +Some of the tests are designed to run against either the Moto library or a real AWS instance. By defining the shell variable MOCK_AWS as "true" all of the tests which can be run in mocked form will be. + + export MOCK_AWS=true + +This considerably speeds up testing but slightly increases risk since Moto's model of SSM is missing a number of features. + +## Defined functionality + +See the features directory for the supported features of the software. This is considered part of the documentation. diff --git a/backup_cloud_ssm/aws_ssm_dict.py b/backup_cloud_ssm/aws_ssm_dict.py index d2ae02d..adbb21c 100644 --- a/backup_cloud_ssm/aws_ssm_dict.py +++ b/backup_cloud_ssm/aws_ssm_dict.py @@ -2,6 +2,7 @@ from hamcrest import assert_that, equal_to from typing import Dict, Tuple, Union from collections.abc import MutableMapping +from botocore.exceptions import ParamValidationError class aws_ssm_dict(MutableMapping): @@ -30,7 +31,14 @@ def __setitem__(self, key: str, value: Union[str, Tuple]): """ description = None - if isinstance(value, tuple): + if isinstance(value, dict): + param_type = value["type"] + val_string = value["value"] + try: + description = value["description"] + except KeyError: + pass + elif isinstance(value, tuple): param_type = value[0] val_string = value[1] try: @@ -42,31 +50,27 @@ def __setitem__(self, key: str, value: Union[str, Tuple]): val_string = value request_params = dict(Name=key, Type=param_type, Value=val_string) - if description: + if description is not None: request_params.update(dict(Description=description)) - - self.ssm.put_parameter(**request_params) + try: + self.ssm.put_parameter(**request_params) + except ( + self.ssm.exceptions.ParameterNotFound, + ParamValidationError, # import from botocore.exceptions since ssm doesn't (currently?) export this + self.ssm.exceptions.ParameterAlreadyExists, + ) as e: + raise AttributeError(e) + except self.ssm.exceptions.ClientError as e: + if "ValidationException" not in str(e): + raise e + raise AttributeError(e) def __delitem__(self, key): request_params = dict(Name=key) - self.ssm.delete_parameter(**request_params) - - def get_tuple_for_param(self, name): - try: - get_response = self.ssm.get_parameter( - Name=name, WithDecryption=self.decrypt - ) - except self.ssm.exceptions.ParameterNotFound as e: - raise KeyError(e) try: - describe_response = self.ssm.describe_parameter(Name=name) + self.ssm.delete_parameter(**request_params) except self.ssm.exceptions.ParameterNotFound as e: raise KeyError(e) - return ( - get_response["Parameter"]["Type"], - get_response["Parameter"]["Value"], - describe_response["Parameter"]["Description"], - ) def iterate_parameter_list(self): paginator = self.ssm.get_paginator("get_parameters_by_path") @@ -87,7 +91,7 @@ def iterate_parameter_descriptions(self): def iterate_for_tuples(self): for i in self.iterate_parameter_descriptions(): name = i["Name"] - yield (name, self.get_tuple_for_param(name)) + yield (name, self.get_param_as_tuple(name)) def iterate_for_values(self): for i in self.iterate_parameter_list(): @@ -99,23 +103,64 @@ def __iter__(self): def __len__(self): pass - def _return_as_tuple(self, key: str): - raise - pass - - def _return_as_value(self, key: str): + def get_param(self, key: str): try: response = self.ssm.get_parameter(Name=key, WithDecryption=self.decrypt) except self.ssm.exceptions.ParameterNotFound as e: raise KeyError(e) assert response["Parameter"]["Name"] == key + return response + + def get_param_as_dict(self, key: str): + try: + get_response = self.ssm.get_parameter(Name=key, WithDecryption=self.decrypt) + except self.ssm.exceptions.ParameterNotFound as e: + raise KeyError(e) + try: + describe_response = self.ssm.describe_parameters( + Filters=[{"Key": "Name", "Values": [key]}] + ) + except self.ssm.exceptions.ParameterNotFound as e: + raise KeyError(e) + retval = { + "value": get_response["Parameter"]["Value"], + "type": get_response["Parameter"]["Type"], + } + try: + retval["description"] = describe_response["Parameters"][0]["Description"] + except KeyError: + pass + return retval + + def get_param_as_tuple(self, key: str): + try: + get_response = self.ssm.get_parameter(Name=key, WithDecryption=self.decrypt) + except self.ssm.exceptions.ParameterNotFound as e: + raise KeyError(e) + try: + describe_response = self.ssm.describe_parameters( + Filters=[{"Key": "Name", "Values": [key]}] + ) + except self.ssm.exceptions.ParameterNotFound as e: + raise KeyError(e) + return ( + get_response["Parameter"]["Type"], + get_response["Parameter"]["Value"], + describe_response["Parameters"][0]["Description"], + ) + + def get_param_as_value(self, key: str): + response = self.get_param(key) return response["Parameter"]["Value"] def __getitem__(self, key: str): - if self.return_type == "tuple": - return self._return_as_tuple(key) - else: - return self._return_as_value(key) + if self.return_type == "dict": + return self.get_param_as_dict(key) + elif self.return_type == "tuple": + return self.get_param_as_tuple(key) + elif self.return_type == "value": + return self.get_param_as_value(key) + raise Exception("unknown return type:" + self.return_type) def upload_dictionary(self, my_dict: Dict[str, str]): for i in my_dict.keys(): @@ -128,8 +173,16 @@ def remove_dictionary(self, my_dict: Dict[str, str]): def verify_dictionary(self, my_dict: Dict[str, str]): for i in my_dict.keys(): - value = self[i] - assert_that(value, equal_to(my_dict[i])) + test_value = my_dict[i] + if isinstance(test_value, dict): + ssm_param = self.get_param_as_dict(i) + assert_that(ssm_param["type"], equal_to(test_value["type"])) + assert_that(ssm_param["value"], equal_to(test_value["value"])) + assert_that( + ssm_param["description"], equal_to(test_value["description"]) + ) + if isinstance(test_value, str): + assert_that(self.get_param_as_value(i), equal_to(test_value)) def verify_deleted_dictionary(self, my_dict: Dict[str, str]): except_count = 0 diff --git a/backup_cloud_ssm/backup_aws_ssm.py b/backup_cloud_ssm/backup_aws_ssm.py index ef46b6a..4cd1166 100644 --- a/backup_cloud_ssm/backup_aws_ssm.py +++ b/backup_cloud_ssm/backup_aws_ssm.py @@ -11,7 +11,7 @@ def eprint(*args, **kwargs): def backup_to_file(file): - ssm_dict = aws_ssm_dict() + ssm_dict = aws_ssm_dict(return_type="dict") contents = {} for i in ssm_dict.keys(): try: @@ -36,5 +36,8 @@ def restore_from_file(file): for i in contents.keys(): try: ssm_dict[i] = contents[i] - except ssm_dict.exceptions.ParameterAlreadyExists: - logger.warning("Parameter " + i + " already exists!") + except AttributeError as e: + if "ParameterAlreadyExists" in str(e): + logger.warning("Parameter " + i + " already exists!") + else: + raise diff --git a/backup_cloud_ssm/test-parameter-storage.py b/backup_cloud_ssm/test-parameter-storage.py deleted file mode 100644 index 9428dd4..0000000 --- a/backup_cloud_ssm/test-parameter-storage.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_store_returns_input(): - pass diff --git a/features/environment.py b/features/environment.py index ac6637b..734876e 100644 --- a/features/environment.py +++ b/features/environment.py @@ -28,19 +28,19 @@ def create_random_params(ssm_dict): def create_random_typed_params(ssm_dict): params = { randomword(): { - "Value": randomword(), - "Type": "String", - "Description": "aws-ssm-backup testing String parameter", + "value": randomword(), + "type": "String", + "description": "aws-ssm-backup testing String parameter", }, randomword(): { - "Value": randomword(), - "Type": "SecureString", - "Description": "aws-ssm-backup testing String parameter", + "value": randomword(), + "type": "SecureString", + "description": "aws-ssm-backup testing String parameter", }, randomword(): { - "Value": randomlist(), - "Type": "StringList", - "Description": "aws-ssm-backup testing String parameter", + "value": randomlist(), + "type": "StringList", + "description": "aws-ssm-backup testing String parameter", }, } ssm_dict.upload_dictionary(params) diff --git a/features/steps/ssm-backup.py b/features/steps/ssm-backup.py index 601d87d..c481cc9 100644 --- a/features/steps/ssm-backup.py +++ b/features/steps/ssm-backup.py @@ -1,3 +1,4 @@ +from hamcrest import assert_that, instance_of from backup_cloud_ssm import backup_aws_ssm from tempfile import NamedTemporaryFile from behave import given, when, then @@ -15,22 +16,6 @@ def step_impl_2(context): backup_aws_ssm.backup_to_file(t.name) -@when(u"I delete those parameters from SSM parameter store") -def step_impl_3(context): - context.ssm_dict.remove_dictionary(context.test_params) - context.ssm_dict.verify_deleted_dictionary(context.test_params) - - -@when(u"I restore those parameters") -def step_impl_4(context): - backup_aws_ssm.restore_from_file(context.backup_temp_file.name) - - -@then(u"those parameters should be in SSM parameter store") -def step_impl_5(context): - context.ssm_dict.verify_dictionary(context.test_params) - - @given(u"I run the aws-ssm-backup command") def step_impl_6(context): call_args = ["aws-ssm-backup"] @@ -42,6 +27,17 @@ def step_impl_6(context): raise +@when(u"I delete those parameters from SSM parameter store") +def step_impl_3(context): + context.ssm_dict.remove_dictionary(context.test_params) + context.ssm_dict.verify_deleted_dictionary(context.test_params) + + +@when(u"I restore those parameters") +def step_impl_4(context): + backup_aws_ssm.restore_from_file(context.backup_temp_file.name) + + @when(u"I run the aws-ssm-backup command with the restore argument") def step_impl_7(context): call_args = ["aws-ssm-backup", "--restore"] @@ -52,3 +48,16 @@ def step_impl_7(context): except CalledProcessError as e: print("Failed backup stderr:", e.stderr, "stdout:", e.stdout) raise + + +@then(u"those parameters should be in SSM parameter store") +def step_impl_5(context): + context.ssm_dict.verify_dictionary(context.test_params) + + +@then(u"the types of those parameters should be preserved") +def step_impl(context): + test_params = context.test_params + for i in test_params.keys(): + assert_that(test_params[i], instance_of(dict)) + context.ssm_dict.verify_dictionary(context.test_params) diff --git a/testing/test_parameter_storage.py b/testing/test_parameter_storage.py new file mode 100644 index 0000000..127d356 --- /dev/null +++ b/testing/test_parameter_storage.py @@ -0,0 +1,90 @@ +fixed_key = "/backup_cloud_aws_ssm_test/test_key" +from hypothesis import given, settings +import hypothesis.strategies as strategies +from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict +from moto import mock_ssm +from os import environ as env +import pytest + + +def identity(ob): + return ob + + +if env.get("MOCK_AWS", "false") == "true": + sometimes_mock_ssm = mock_ssm +else: + sometimes_mock_ssm = identity + +badkey_examples = ("", "/", None) + + +# this cannot be mocked via moto because moto accepts these keys! +@given(strategies.sampled_from(badkey_examples)) +def test_bad_key_raises_exception(key): + ssm_dict = aws_ssm_dict() + exception_thrown = False + try: + ssm_dict[key] = "the_value" + except AttributeError: + exception_thrown = True + assert exception_thrown, "key: " + key + " failed to throw expected assertion" + + +def store_restore_key(ssm_dict, value): + try: + del ssm_dict[fixed_key] + except KeyError: + pass + ssm_dict[fixed_key] = value + return ssm_dict[fixed_key] + + +def store_restore_key_with_types(ssm_dict, value, type_name, description): + assert ssm_dict.return_type == "dict" + try: + del ssm_dict[fixed_key] + except KeyError: + pass + ssm_dict[fixed_key] = { + "value": value, + "type": type_name, + "description": description, + } + return ssm_dict[fixed_key] + + +@sometimes_mock_ssm +@settings(deadline=3000) +@given(strategies.text(min_size=1, max_size=400)) +def test_store_returns_input(value): + ssm_dict = aws_ssm_dict() + assert store_restore_key(ssm_dict, value) == value + + +type_names = ("SecureString", "String", "StringList") + + +@pytest.mark.wip +@sometimes_mock_ssm +@settings(deadline=3000) +@given( + strategies.text(min_size=1, max_size=400), + strategies.sampled_from(type_names), + strategies.text(min_size=0, max_size=400), +) +def test_store_returns_input_with_type(value, type_name, description): + ssm_dict = aws_ssm_dict(return_type="dict") + returned_param = store_restore_key_with_types( + ssm_dict, value, type_name, description + ) + assert isinstance(returned_param, dict) + assert ( + value == returned_param["value"] + and type_name == returned_param["type"] + and description == returned_param.get("description", "") + ), "return parameter mismatch" + + +def test_true(): + return True From 987a1209e39d551d7c55c6a7931251d675a2b896 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Thu, 14 Nov 2019 09:56:38 +0000 Subject: [PATCH 06/14] fixes for timeout & sync edge cases found in testing - see e.g. Case ID - 6590189121 --- README.md | 11 +++ backup_cloud_ssm/aws_ssm_dict.py | 121 +++++++++++++++++++++------- features/backup-restore-ssm.feature | 1 - features/encrypt-backup.feature | 21 +++++ features/steps/encrypt-backup.py | 46 +++++++++++ testing/test_parameter_access.py | 58 +++++++++++++ testing/test_parameter_storage.py | 46 ++++++++--- 7 files changed, 266 insertions(+), 38 deletions(-) create mode 100644 features/encrypt-backup.feature create mode 100644 features/steps/encrypt-backup.py create mode 100644 testing/test_parameter_access.py diff --git a/README.md b/README.md index 1e8305e..efe8b0d 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,17 @@ which accesses SSM parameter store. This will likely, later, be split out into print(ssm_dict["parameter"]) +SSM parameter store treats storing no description and storing the +empty description ("") as the same thing and will not return any +description. For simplicity we have now chosen to represent this as +the empty string. This decision may change in future and feedback is +appreciated. + +When parameters are deleted the parameter description sometimes seems +to persist for some time, possibly only when it was '0'. Do not rely +on the deescription to be empty or ee testing/test_parameter_storage +for how to handle this. + ## Development We aim to use Behavior Driven Development to encourage reasonable feature descriptions and a level of tests appropriate for the business functionality included here. Test Driven Development and to some extent Test Driven Design are encouraged in order to improve testability and eas of modification of the code. diff --git a/backup_cloud_ssm/aws_ssm_dict.py b/backup_cloud_ssm/aws_ssm_dict.py index adbb21c..a39568e 100644 --- a/backup_cloud_ssm/aws_ssm_dict.py +++ b/backup_cloud_ssm/aws_ssm_dict.py @@ -3,6 +3,8 @@ from typing import Dict, Tuple, Union from collections.abc import MutableMapping from botocore.exceptions import ParamValidationError +from time import sleep +import logging class aws_ssm_dict(MutableMapping): @@ -17,8 +19,16 @@ class aws_ssm_dict(MutableMapping): """ - def __init__(self, decrypt=True, return_type="value", region_name="eu-west-1"): - self.ssm = boto3.client("ssm", region_name=region_name) + def __init__( + self, decrypt=True, return_type="value", region_name=None, ssm_client=None + ): + if ssm_client is None: + if region_name is not None: + self.ssm = boto3.client("ssm", region_name=region_name) + else: + self.ssm = boto3.client("ssm", region_name=region_name) + else: + self.ssm = ssm_client self.exceptions = self.ssm.exceptions self.decrypt = decrypt self.return_type = return_type @@ -37,14 +47,14 @@ def __setitem__(self, key: str, value: Union[str, Tuple]): try: description = value["description"] except KeyError: - pass + description = None elif isinstance(value, tuple): param_type = value[0] val_string = value[1] try: description = value[2] except IndexError: - pass + description = None else: param_type = "SecureString" val_string = value @@ -65,12 +75,41 @@ def __setitem__(self, key: str, value: Union[str, Tuple]): raise e raise AttributeError(e) - def __delitem__(self, key): + def __delitem__(self, key, max_retries=15): + """delete a parameter and wait for it to be deleted + + We send the delete_parameter call to AWS which requests + parameter deletion. Unfortunately it seems that this takes + some time after the call returns to complete. This means we + means we wait to retry by default. + """ + request_params = dict(Name=key) try: self.ssm.delete_parameter(**request_params) except self.ssm.exceptions.ParameterNotFound as e: raise KeyError(e) + count = 0 + sleep_secs = 100 / 1000 + sleep_mult = 1.2 + while True: + describe_response = self.ssm.describe_parameters( + Filters=[{"Key": "Name", "Values": [key]}] + ) + try: + if "Description" not in describe_response["Parameters"][0]: + break + except IndexError: + break + if count < max_retries: + logging.debug("sleeping " + str(sleep_secs) + " to give ssm time") + sleep(sleep_secs) + count += 1 + sleep_secs = sleep_secs * sleep_mult + else: + raise Exception( + "Description failed to clear after delete - nasty AWS!!!" + ) def iterate_parameter_list(self): paginator = self.ssm.get_paginator("get_parameters_by_path") @@ -111,42 +150,70 @@ def get_param(self, key: str): assert response["Parameter"]["Name"] == key return response - def get_param_as_dict(self, key: str): - try: - get_response = self.ssm.get_parameter(Name=key, WithDecryption=self.decrypt) - except self.ssm.exceptions.ParameterNotFound as e: - raise KeyError(e) - try: - describe_response = self.ssm.describe_parameters( - Filters=[{"Key": "Name", "Values": [key]}] + def desc_param(self, key: str, max_retries=15): + """get the description of a parameter + + this tries to get the describe_parameters response for a + parameter with the parameter as the first (and only) parameter + in the parameter list. In the case that the initial call + fails it retries (max_retries times) in case a parameter has + been created but the data about it is not yet in sync. + """ + + count = 0 + sleep_secs = 100 / 1000 + sleep_mult = 1.2 + while True: + try: + + describe_response = self.ssm.describe_parameters( + Filters=[{"Key": "Name", "Values": [key]}] + ) + logging.debug( + "Desc param - parameters returned: " + + str(describe_response["Parameters"]) + ) + if describe_response["Parameters"][0]["Name"] == key: + break + except (self.ssm.exceptions.ParameterNotFound, IndexError) as e: + if count == max_retries: + raise KeyError("description retry count exceeded", e) + count += 1 + + logging.debug( + "sleeping " + + str(sleep_secs) + + " to give ssm time to retrieve description" ) - except self.ssm.exceptions.ParameterNotFound as e: - raise KeyError(e) + sleep(sleep_secs) + sleep_secs = sleep_secs * sleep_mult + + return describe_response + + def get_param_as_dict(self, key: str): + get_response = self.get_param(key) retval = { "value": get_response["Parameter"]["Value"], "type": get_response["Parameter"]["Type"], } + describe_response = self.desc_param(key) try: retval["description"] = describe_response["Parameters"][0]["Description"] except KeyError: - pass + retval["description"] = "" return retval def get_param_as_tuple(self, key: str): + get_response = self.ssm.get_parameter(Name=key, WithDecryption=self.decrypt) + describe_response = self.desc_param(key) try: - get_response = self.ssm.get_parameter(Name=key, WithDecryption=self.decrypt) - except self.ssm.exceptions.ParameterNotFound as e: - raise KeyError(e) - try: - describe_response = self.ssm.describe_parameters( - Filters=[{"Key": "Name", "Values": [key]}] - ) - except self.ssm.exceptions.ParameterNotFound as e: - raise KeyError(e) + description = describe_response["Parameters"][0]["Description"] + except KeyError: + description = "" return ( get_response["Parameter"]["Type"], get_response["Parameter"]["Value"], - describe_response["Parameters"][0]["Description"], + description, ) def get_param_as_value(self, key: str): @@ -160,7 +227,7 @@ def __getitem__(self, key: str): return self.get_param_as_tuple(key) elif self.return_type == "value": return self.get_param_as_value(key) - raise Exception("unknown return type:" + self.return_type) + raise Exception("unknown return type: " + self.return_type) def upload_dictionary(self, my_dict: Dict[str, str]): for i in my_dict.keys(): diff --git a/features/backup-restore-ssm.feature b/features/backup-restore-ssm.feature index 8fa3e71..7699a16 100644 --- a/features/backup-restore-ssm.feature +++ b/features/backup-restore-ssm.feature @@ -10,7 +10,6 @@ In order to ensure that I can recover my SSM parameters even And I restore those parameters Then those parameters should be in SSM parameter store - @wip @fixture.preexist_params @fixture.ssm_typed_params Scenario: backup SSM to a plaintext file diff --git a/features/encrypt-backup.feature b/features/encrypt-backup.feature new file mode 100644 index 0000000..ecf4dcf --- /dev/null +++ b/features/encrypt-backup.feature @@ -0,0 +1,21 @@ +Feature: encrypt backups + +In order to reduce the risk of leakage of credentials from SSM +parameter store Michael would like to use public key encryption from +backup_cloud to ensure that backups are encrypted and protected but can be recovered when needed. + + + Background: we have prepared to run encrypted backups + Given I have access to an account for doing backups + and I have a private public key pair + and the public key from that key pair is stored in an s3 bucket + and I have configured my settings in SSM + + + @future + @wip + Scenario: default encryption when ssm is backed up to S3 + Given I have some parameters in SSM parameter store + When I run the aws-ssm-backup command + Then a backup object should be created in the S3 destination bucket + and if I decrypt that file the content with the private key it should match the original diff --git a/features/steps/encrypt-backup.py b/features/steps/encrypt-backup.py new file mode 100644 index 0000000..c12494c --- /dev/null +++ b/features/steps/encrypt-backup.py @@ -0,0 +1,46 @@ +from behave import given, when, then + + +@given(u"I have access to an account for doing backups") +def step_impl_0(context): + raise NotImplementedError( + u"STEP: Given I have access to an account for doing backups" + ) + + +@given(u"I have a private public key pair") +def step_impl_1(context): + raise NotImplementedError(u"STEP: Given I have a private public key pair") + + +@given(u"the public key from that key pair is stored in an s3 bucket") +def step_impl_2(context): + raise NotImplementedError( + u"STEP: Given the public key from that key pair is stored in an s3 bucket" + ) + + +@given(u"I have configured my settings in SSM") +def step_impl_3(context): + raise NotImplementedError(u"STEP: Given I have configured my settings in SSM") + + +@when(u"I run the aws-ssm-backup command") +def step_impl_4(context): + raise NotImplementedError(u"STEP: When I run the aws-ssm-backup command") + + +@then(u"a backup object should be created in the S3 destination bucket") +def step_impl_5(context): + raise NotImplementedError( + u"STEP: Then a backup object should be created in the S3 destination bucket" + ) + + +@then( + u"if I decrypt that file the content with the private key it should match the original" +) +def step_impl_6(context): + raise NotImplementedError( + u"STEP: Then if I decrypt that file the content with the private key it should match the original" + ) diff --git a/testing/test_parameter_access.py b/testing/test_parameter_access.py new file mode 100644 index 0000000..32fa893 --- /dev/null +++ b/testing/test_parameter_access.py @@ -0,0 +1,58 @@ +from unittest.mock import MagicMock, patch +from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict + + +def test_get_parameter_as_dict_should_get_and_describe(): + value = "empty desc demo" + type_name = "SecureString" + description = "the description" + ssm_mock = MagicMock() + ssm_mock.get_parameter.return_value = { + "Parameter": {"Name": "fakeparam", "Type": type_name, "Value": value} + } + ssm_mock.describe_parameters.return_value = { + "Parameters": [ + {"Name": "fakeparam", "Type": "SecureString", "Description": description} + ] + } + ssm_dict = aws_ssm_dict(ssm_client=ssm_mock, return_type="dict") + returned_param = ssm_dict.get_param_as_dict("fakeparam") + assert ( + returned_param["value"] == value + and returned_param["type"] == type_name + and returned_param["description"] == description + ), "return parameter mismatch" + ssm_mock.get_parameter.assert_called() + ssm_mock.describe_parameters.assert_called() + + +def test_get_parameter_desc_should_retry_and_then_return(): + description = "the description" + ssm_mock = MagicMock() + ssm_mock.exceptions.ParameterNotFound = Exception + ssm_mock.describe_parameters.side_effect = [ + {"Parameters": []}, + {"Parameters": []}, + {"Parameters": []}, + {"Parameters": []}, + {"Parameters": []}, + {"Parameters": []}, + { + "Parameters": [ + { + "Name": "fakeparam", + "Type": "SecureString", + "Description": description, + } + ] + }, + ] + ssm_dict = aws_ssm_dict(ssm_client=ssm_mock, return_type="dict") + + def mock_sleep(time): + pass + + with patch("backup_cloud_ssm.aws_ssm_dict.sleep"): + returned_desc = ssm_dict.desc_param("fakeparam") + assert returned_desc["Parameters"][0]["Name"] == "fakeparam" + ssm_mock.describe_parameters.assert_called() diff --git a/testing/test_parameter_storage.py b/testing/test_parameter_storage.py index 127d356..bdbb204 100644 --- a/testing/test_parameter_storage.py +++ b/testing/test_parameter_storage.py @@ -1,5 +1,5 @@ fixed_key = "/backup_cloud_aws_ssm_test/test_key" -from hypothesis import given, settings +from hypothesis import given, settings, example import hypothesis.strategies as strategies from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict from moto import mock_ssm @@ -20,6 +20,7 @@ def identity(ob): # this cannot be mocked via moto because moto accepts these keys! +@settings(deadline=3000) @given(strategies.sampled_from(badkey_examples)) def test_bad_key_raises_exception(key): ssm_dict = aws_ssm_dict() @@ -31,21 +32,23 @@ def test_bad_key_raises_exception(key): assert exception_thrown, "key: " + key + " failed to throw expected assertion" -def store_restore_key(ssm_dict, value): +def store_restore_key(ssm_dict, key, value): try: - del ssm_dict[fixed_key] + del ssm_dict[key] except KeyError: pass + ssm_dict[fixed_key] = value return ssm_dict[fixed_key] -def store_restore_key_with_types(ssm_dict, value, type_name, description): +def store_restore_key_with_types(ssm_dict, key, value, type_name, description): assert ssm_dict.return_type == "dict" try: del ssm_dict[fixed_key] except KeyError: pass + ssm_dict[fixed_key] = { "value": value, "type": type_name, @@ -59,13 +62,12 @@ def store_restore_key_with_types(ssm_dict, value, type_name, description): @given(strategies.text(min_size=1, max_size=400)) def test_store_returns_input(value): ssm_dict = aws_ssm_dict() - assert store_restore_key(ssm_dict, value) == value + assert store_restore_key(ssm_dict, fixed_key, value) == value type_names = ("SecureString", "String", "StringList") -@pytest.mark.wip @sometimes_mock_ssm @settings(deadline=3000) @given( @@ -73,16 +75,40 @@ def test_store_returns_input(value): strategies.sampled_from(type_names), strategies.text(min_size=0, max_size=400), ) +@example(value="0", type_name="SecureString", description="0") +@example(value="0", type_name="SecureString", description="") def test_store_returns_input_with_type(value, type_name, description): ssm_dict = aws_ssm_dict(return_type="dict") returned_param = store_restore_key_with_types( - ssm_dict, value, type_name, description + ssm_dict, fixed_key, value, type_name, description ) assert isinstance(returned_param, dict) assert ( - value == returned_param["value"] - and type_name == returned_param["type"] - and description == returned_param.get("description", "") + returned_param["value"] == value + and returned_param["type"] == type_name + and returned_param["description"] == description + ), "return parameter mismatch" + + +@pytest.mark.wip +def test_empty_string_is_stored_as_none_and_returned_as_empty(): + value = "empty desc demo" + type_name = "SecureString" + description = "" + ssm_dict = aws_ssm_dict(return_type="dict") + try: + del ssm_dict[fixed_key] + except KeyError: + pass + + ssm_dict[fixed_key] = (type_name, value, "") + + returned_param = ssm_dict[fixed_key] + assert isinstance(returned_param, dict) + assert ( + returned_param["value"] == value + and returned_param["type"] == type_name + and returned_param["description"] == description ), "return parameter mismatch" From bd470d685199eeb258dd07bf26b6a62d843d0944 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Thu, 14 Nov 2019 14:41:11 +0000 Subject: [PATCH 07/14] full workarounds for various SSM implementation artifacts --- README.md | 24 +++++-- backup_cloud_ssm/aws_ssm_dict.py | 93 ++++++++++++++----------- testing/test_parameter_access.py | 110 ++++++++++++++++-------------- testing/test_parameter_storage.py | 39 ++++++++--- 4 files changed, 159 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index efe8b0d..60c77b9 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ Backup SSM parameter store to a file. Optional (but default) encryption to be added. - ## Using CLI tools +The CLI tool provides a simple interface to dump or restore the full +set of SSM parameters. + 1) set up the appropriate environment including AWS variables 2) to backup run @@ -14,15 +16,29 @@ encryption to be added. aws-ssm-backup --restore > +Special notes: + +1) the tool does not overwrite - if you want to replace an existing +parameter, simply manually delete it and run again. + +2) ssm seems to be eventually consistent - you will not want to update +SSM shortly before doing a backup. You may want to wait a second or +so after restoring. + ## Using python interface +The backup and restore functions are provided as a libray. For backup + import backup_aws_ssm backup_aws_ssm.backup_to_file("myfile") +for restore: import backup_aws_ssm backup_aws_ssm.restore_from_file("myfile") +set the appropriate AWS variables to configure the aws region where +this will work. ## Using python ssm library @@ -33,7 +49,6 @@ which accesses SSM parameter store. This will likely, later, be split out into ssm_dict = aws_ssm_dict() ssm_dict["parameter"] = "value" print(ssm_dict["parameter"]) - SSM parameter store treats storing no description and storing the empty description ("") as the same thing and will not return any @@ -43,14 +58,15 @@ appreciated. When parameters are deleted the parameter description sometimes seems to persist for some time, possibly only when it was '0'. Do not rely -on the deescription to be empty or ee testing/test_parameter_storage +on the description to be empty or see testing/test_parameter_storage for how to handle this. + ## Development We aim to use Behavior Driven Development to encourage reasonable feature descriptions and a level of tests appropriate for the business functionality included here. Test Driven Development and to some extent Test Driven Design are encouraged in order to improve testability and eas of modification of the code. -Some of the tests are designed to run against either the Moto library or a real AWS instance. By defining the variable:\ +Some of the tests are designed to run against either the Moto library or a real AWS instance. By defining the variable: We aim to use Behavior Driven Development to encourage reasonable feature descriptions and a level of tests appropriate for the business functionality included here. Test Driven Development and to some extent Test Driven Design are encouraged in order to improve testability and eas of modification of the code. diff --git a/backup_cloud_ssm/aws_ssm_dict.py b/backup_cloud_ssm/aws_ssm_dict.py index a39568e..9a43be0 100644 --- a/backup_cloud_ssm/aws_ssm_dict.py +++ b/backup_cloud_ssm/aws_ssm_dict.py @@ -120,7 +120,7 @@ def iterate_parameter_list(self): for i in page["Parameters"]: yield i - def iterate_parameter_descriptions(self): + def iterate_param_descs_for_names(self): paginator = self.ssm.get_paginator("describe_parameters") page_iterator = paginator.paginate() for page in page_iterator: @@ -128,7 +128,7 @@ def iterate_parameter_descriptions(self): yield i["Name"] def iterate_for_tuples(self): - for i in self.iterate_parameter_descriptions(): + for i in self.iterate_param_descs_for_names(): name = i["Name"] yield (name, self.get_param_as_tuple(name)) @@ -137,7 +137,7 @@ def iterate_for_values(self): yield (i["Name"], i["Value"]) def __iter__(self): - yield from self.iterate_parameter_descriptions() + yield from self.iterate_param_descs_for_names() def __len__(self): pass @@ -151,65 +151,76 @@ def get_param(self, key: str): return response def desc_param(self, key: str, max_retries=15): - """get the description of a parameter + """get the description of a parameter we _know_ is there + + this tries to get the AWS parameter description from inside a + describe_parameters response for a parameter with the + parameter as the first (and only) parameter in the parameter + list. In the case that the initial call fails it retries + (max_retries times) in case a parameter has been created but + the data about it is not yet in sync. - this tries to get the describe_parameters response for a - parameter with the parameter as the first (and only) parameter - in the parameter list. In the case that the initial call - fails it retries (max_retries times) in case a parameter has - been created but the data about it is not yet in sync. + """ + + """Note on code here: + + "Request results are returned on a best-effort basis." + https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_DescribeParameters.html + + this means that even though we only want the first result, we + still have to paginate through what may be a number of + responses with empty parameter lists. """ count = 0 sleep_secs = 100 / 1000 sleep_mult = 1.2 while True: - try: - - describe_response = self.ssm.describe_parameters( - Filters=[{"Key": "Name", "Values": [key]}] - ) + paginator = self.ssm.get_paginator("describe_parameters") + page_iterator = paginator.paginate( + Filters=[{"Key": "Name", "Values": [key]}] + ) + param_pages = [page["Parameters"] for page in page_iterator] + paramlist = [param for sublist in param_pages for param in sublist] + if len(paramlist) == 0: + if count > max_retries: + raise KeyError("description retry count exceeded - aborting") + count += 1 logging.debug( - "Desc param - parameters returned: " - + str(describe_response["Parameters"]) + "sleeping " + + str(sleep_secs) + + " to give ssm time to retrieve description" ) - if describe_response["Parameters"][0]["Name"] == key: - break - except (self.ssm.exceptions.ParameterNotFound, IndexError) as e: - if count == max_retries: - raise KeyError("description retry count exceeded", e) - count += 1 - - logging.debug( - "sleeping " - + str(sleep_secs) - + " to give ssm time to retrieve description" - ) - sleep(sleep_secs) - sleep_secs = sleep_secs * sleep_mult + sleep(sleep_secs) + sleep_secs = sleep_secs * sleep_mult + else: + break + assert len(paramlist) == 1 + assert paramlist[0]["Name"] == key + return paramlist[0] - return describe_response + def retrieve_description(self, key: str): + parameter_describe = self.desc_param(key) + try: + description = parameter_describe["Description"] + except KeyError: + description = "" + return description def get_param_as_dict(self, key: str): get_response = self.get_param(key) + description = self.retrieve_description(key) + retval = { "value": get_response["Parameter"]["Value"], "type": get_response["Parameter"]["Type"], + "description": description, } - describe_response = self.desc_param(key) - try: - retval["description"] = describe_response["Parameters"][0]["Description"] - except KeyError: - retval["description"] = "" return retval def get_param_as_tuple(self, key: str): get_response = self.ssm.get_parameter(Name=key, WithDecryption=self.decrypt) - describe_response = self.desc_param(key) - try: - description = describe_response["Parameters"][0]["Description"] - except KeyError: - description = "" + description = self.retrieve_description(key) return ( get_response["Parameter"]["Type"], get_response["Parameter"]["Value"], diff --git a/testing/test_parameter_access.py b/testing/test_parameter_access.py index 32fa893..df7b0e2 100644 --- a/testing/test_parameter_access.py +++ b/testing/test_parameter_access.py @@ -1,58 +1,62 @@ -from unittest.mock import MagicMock, patch -from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict +pass +# from unittest.mock import MagicMock # , patch +# from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict +# re-implemented for now using paginator +# +# def test_get_parameter_as_dict_should_get_and_describe(): +# value = "empty desc demo" +# type_name = "SecureString" +# description = "the description" +# ssm_mock = MagicMock() +# ssm_mock.get_parameter.return_value = { +# "Parameter": {"Name": "fakeparam", "Type": type_name, "Value": value} +# } +# ssm_mock.describe_parameters.return_value = { +# "Parameters": [ +# {"Name": "fakeparam", "Type": "SecureString", "Description": description} +# ] +# } +# ssm_dict = aws_ssm_dict(ssm_client=ssm_mock, return_type="dict") +# returned_param = ssm_dict.get_param_as_dict("fakeparam") +# assert ( +# returned_param["value"] == value +# and returned_param["type"] == type_name +# and returned_param["description"] == description +# ), "return parameter mismatch" +# ssm_mock.get_parameter.assert_called() +# ssm_mock.describe_parameters.assert_called() -def test_get_parameter_as_dict_should_get_and_describe(): - value = "empty desc demo" - type_name = "SecureString" - description = "the description" - ssm_mock = MagicMock() - ssm_mock.get_parameter.return_value = { - "Parameter": {"Name": "fakeparam", "Type": type_name, "Value": value} - } - ssm_mock.describe_parameters.return_value = { - "Parameters": [ - {"Name": "fakeparam", "Type": "SecureString", "Description": description} - ] - } - ssm_dict = aws_ssm_dict(ssm_client=ssm_mock, return_type="dict") - returned_param = ssm_dict.get_param_as_dict("fakeparam") - assert ( - returned_param["value"] == value - and returned_param["type"] == type_name - and returned_param["description"] == description - ), "return parameter mismatch" - ssm_mock.get_parameter.assert_called() - ssm_mock.describe_parameters.assert_called() +# maybe needed in near future - changed implmenetation for now +# +# def test_get_parameter_desc_should_retry_and_then_return(): +# description = "the description" +# ssm_mock = MagicMock() +# ssm_mock.exceptions.ParameterNotFound = Exception +# ssm_mock.describe_parameters.side_effect = [ +# {"Parameters": []}, +# {"Parameters": []}, +# {"Parameters": []}, +# {"Parameters": []}, +# {"Parameters": []}, +# {"Parameters": []}, +# { +# "Parameters": [ +# { +# "Name": "fakeparam", +# "Type": "SecureString", +# "Description": description, +# } +# ] +# }, +# ] +# ssm_dict = aws_ssm_dict(ssm_client=ssm_mock, return_type="dict") -def test_get_parameter_desc_should_retry_and_then_return(): - description = "the description" - ssm_mock = MagicMock() - ssm_mock.exceptions.ParameterNotFound = Exception - ssm_mock.describe_parameters.side_effect = [ - {"Parameters": []}, - {"Parameters": []}, - {"Parameters": []}, - {"Parameters": []}, - {"Parameters": []}, - {"Parameters": []}, - { - "Parameters": [ - { - "Name": "fakeparam", - "Type": "SecureString", - "Description": description, - } - ] - }, - ] - ssm_dict = aws_ssm_dict(ssm_client=ssm_mock, return_type="dict") +# def mock_sleep(time): +# pass - def mock_sleep(time): - pass - - with patch("backup_cloud_ssm.aws_ssm_dict.sleep"): - returned_desc = ssm_dict.desc_param("fakeparam") - assert returned_desc["Parameters"][0]["Name"] == "fakeparam" - ssm_mock.describe_parameters.assert_called() +# with patch("backup_cloud_ssm.aws_ssm_dict.sleep"): +# returned_desc = ssm_dict.desc_param("fakeparam") +# assert returned_desc["Parameters"][0]["Name"] == "fakeparam" +# ssm_mock.describe_parameters.assert_called() diff --git a/testing/test_parameter_storage.py b/testing/test_parameter_storage.py index bdbb204..1d63f04 100644 --- a/testing/test_parameter_storage.py +++ b/testing/test_parameter_storage.py @@ -5,6 +5,8 @@ from moto import mock_ssm from os import environ as env import pytest +from time import sleep +from warnings import warn def identity(ob): @@ -20,7 +22,7 @@ def identity(ob): # this cannot be mocked via moto because moto accepts these keys! -@settings(deadline=3000) +@settings(deadline=10000) @given(strategies.sampled_from(badkey_examples)) def test_bad_key_raises_exception(key): ssm_dict = aws_ssm_dict() @@ -42,7 +44,7 @@ def store_restore_key(ssm_dict, key, value): return ssm_dict[fixed_key] -def store_restore_key_with_types(ssm_dict, key, value, type_name, description): +def store_key_with_types(ssm_dict, key, value, type_name, description): assert ssm_dict.return_type == "dict" try: del ssm_dict[fixed_key] @@ -54,11 +56,10 @@ def store_restore_key_with_types(ssm_dict, key, value, type_name, description): "type": type_name, "description": description, } - return ssm_dict[fixed_key] @sometimes_mock_ssm -@settings(deadline=3000) +@settings(deadline=10000) @given(strategies.text(min_size=1, max_size=400)) def test_store_returns_input(value): ssm_dict = aws_ssm_dict() @@ -67,9 +68,14 @@ def test_store_returns_input(value): type_names = ("SecureString", "String", "StringList") +# this test can be flaky, certainly in the time to run safely to the +# extent we should almost set deadline=None ; still it's the key +# function so we leave it in and keep increasing the time and retrys + +@pytest.mark.wip @sometimes_mock_ssm -@settings(deadline=3000) +@settings(deadline=100000) @given( strategies.text(min_size=1, max_size=400), strategies.sampled_from(type_names), @@ -79,9 +85,25 @@ def test_store_returns_input(value): @example(value="0", type_name="SecureString", description="") def test_store_returns_input_with_type(value, type_name, description): ssm_dict = aws_ssm_dict(return_type="dict") - returned_param = store_restore_key_with_types( - ssm_dict, fixed_key, value, type_name, description - ) + store_key_with_types(ssm_dict, fixed_key, value, type_name, description) + max_retries = 10 + count = 0 + sleep_secs = 300 / 1000 + sleep_mult = 1.4 + while True: + if count > max_retries: + raise Exception( + "Too many retries: description mismatch - probably update is broken" + ) + returned_param = ssm_dict[fixed_key] + if returned_param["description"] != description: + warn("description mismatch - sleep and try again") + sleep(sleep_secs) + count += 1 + sleep_secs = sleep_secs * sleep_mult + else: + break + assert isinstance(returned_param, dict) assert ( returned_param["value"] == value @@ -90,7 +112,6 @@ def test_store_returns_input_with_type(value, type_name, description): ), "return parameter mismatch" -@pytest.mark.wip def test_empty_string_is_stored_as_none_and_returned_as_empty(): value = "empty desc demo" type_name = "SecureString" From bba1b78a09f7db54dd3483b0d652ed5c7454829d Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Thu, 14 Nov 2019 16:23:30 +0000 Subject: [PATCH 08/14] push problem with name actually being a prefix --- backup_cloud_ssm/aws_ssm_dict.py | 9 ++--- testing/test_parameter_storage.py | 56 ++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/backup_cloud_ssm/aws_ssm_dict.py b/backup_cloud_ssm/aws_ssm_dict.py index 9a43be0..04cabfe 100644 --- a/backup_cloud_ssm/aws_ssm_dict.py +++ b/backup_cloud_ssm/aws_ssm_dict.py @@ -182,7 +182,8 @@ def desc_param(self, key: str, max_retries=15): ) param_pages = [page["Parameters"] for page in page_iterator] paramlist = [param for sublist in param_pages for param in sublist] - if len(paramlist) == 0: + ret_paramlist = [x for x in paramlist if x["Name"] == key] + if len(ret_paramlist) == 0: if count > max_retries: raise KeyError("description retry count exceeded - aborting") count += 1 @@ -195,9 +196,9 @@ def desc_param(self, key: str, max_retries=15): sleep_secs = sleep_secs * sleep_mult else: break - assert len(paramlist) == 1 - assert paramlist[0]["Name"] == key - return paramlist[0] + assert len(ret_paramlist) == 1, "Paramlist wrong length: " + str(ret_paramlist) + assert ret_paramlist[0]["Name"] == key + return ret_paramlist[0] def retrieve_description(self, key: str): parameter_describe = self.desc_param(key) diff --git a/testing/test_parameter_storage.py b/testing/test_parameter_storage.py index 1d63f04..335693c 100644 --- a/testing/test_parameter_storage.py +++ b/testing/test_parameter_storage.py @@ -40,22 +40,60 @@ def store_restore_key(ssm_dict, key, value): except KeyError: pass - ssm_dict[fixed_key] = value - return ssm_dict[fixed_key] + ssm_dict[key] = value + return ssm_dict[key] def store_key_with_types(ssm_dict, key, value, type_name, description): assert ssm_dict.return_type == "dict" try: - del ssm_dict[fixed_key] + del ssm_dict[key] except KeyError: pass - ssm_dict[fixed_key] = { - "value": value, - "type": type_name, - "description": description, - } + ssm_dict[key] = {"value": value, "type": type_name, "description": description} + + +@pytest.mark.wip +def test_store_restore_keys_with_common_start(): + key = "/test/fake/key_part" + value = "that" + key2 = key + "_too" + type_name = "String" + description = "two matching strings test parameter" + ssm_dict = aws_ssm_dict(return_type="dict") + try: + ssm_dict[key2] = "this" + except AttributeError as e: + if "ParameterAlreadyExists" not in str(e): + raise + + store_key_with_types(ssm_dict, key, value, type_name, description) + + max_retries = 10 + count = 0 + sleep_secs = 300 / 1000 + sleep_mult = 1.4 + while True: + if count > max_retries: + raise Exception( + "Too many retries: description mismatch - probably update is broken" + ) + returned_param = ssm_dict[key] + if returned_param["description"] != description: + warn("description mismatch - sleep and try again") + sleep(sleep_secs) + count += 1 + sleep_secs = sleep_secs * sleep_mult + else: + break + + assert isinstance(returned_param, dict) + assert ( + returned_param["value"] == value + and returned_param["type"] == type_name + and returned_param["description"] == description + ), "return parameter mismatch" @sometimes_mock_ssm @@ -73,7 +111,6 @@ def test_store_returns_input(value): # function so we leave it in and keep increasing the time and retrys -@pytest.mark.wip @sometimes_mock_ssm @settings(deadline=100000) @given( @@ -86,6 +123,7 @@ def test_store_returns_input(value): def test_store_returns_input_with_type(value, type_name, description): ssm_dict = aws_ssm_dict(return_type="dict") store_key_with_types(ssm_dict, fixed_key, value, type_name, description) + max_retries = 10 count = 0 sleep_secs = 300 / 1000 From d9d860953a10a04cbbc3a5ca1f4e7aada1daa65c Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Thu, 14 Nov 2019 17:45:13 +0000 Subject: [PATCH 09/14] Clean up and improve build to use more mocking --- Makefile | 12 ++++-- README.md | 8 +--- testing/test_parameter_access.py | 62 ------------------------------- testing/test_parameter_storage.py | 2 + 4 files changed, 12 insertions(+), 72 deletions(-) delete mode 100644 testing/test_parameter_access.py diff --git a/Makefile b/Makefile index fcd9238..6aed904 100644 --- a/Makefile +++ b/Makefile @@ -8,19 +8,23 @@ LIBFILES := $(shell find backup_cloud_ssm -name '*.py') all: lint test -test: develop pytest behave +# pytest-mocked is much faster than non-mocked which is slower even than +# the functional tests so run it first, then behave then ffinally the +# full pytest tests so that failures are detected early where possible. +test: develop pytest-mocked behave pytest behave: behave --tags ~@future +pytest-mocked: + MOCK_AWS=true pytest + pytest: pytest - wip: develop behave --wip - lint: pre-commit install --install-hooks pre-commit run -a @@ -34,4 +38,4 @@ develop: .develop.makestamp $(PYTHON) setup.py develop touch $@ -.PHONY: all test behave pytest wip lint develop +.PHONY: all test behave pytest-mocked pytest wip lint develop diff --git a/README.md b/README.md index 60c77b9..3fbbe42 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,11 @@ set of SSM parameters. 2) to backup run - aws-ssm-backup > + aws-ssm-backup > `` 3) to restore run - aws-ssm-backup --restore > + aws-ssm-backup --restore > `` Special notes: @@ -66,10 +66,6 @@ for how to handle this. We aim to use Behavior Driven Development to encourage reasonable feature descriptions and a level of tests appropriate for the business functionality included here. Test Driven Development and to some extent Test Driven Design are encouraged in order to improve testability and eas of modification of the code. -Some of the tests are designed to run against either the Moto library or a real AWS instance. By defining the variable: - - We aim to use Behavior Driven Development to encourage reasonable feature descriptions and a level of tests appropriate for the business functionality included here. Test Driven Development and to some extent Test Driven Design are encouraged in order to improve testability and eas of modification of the code. - Some of the tests are designed to run against either the Moto library or a real AWS instance. By defining the shell variable MOCK_AWS as "true" all of the tests which can be run in mocked form will be. export MOCK_AWS=true diff --git a/testing/test_parameter_access.py b/testing/test_parameter_access.py deleted file mode 100644 index df7b0e2..0000000 --- a/testing/test_parameter_access.py +++ /dev/null @@ -1,62 +0,0 @@ -pass -# from unittest.mock import MagicMock # , patch -# from backup_cloud_ssm.aws_ssm_dict import aws_ssm_dict - -# re-implemented for now using paginator -# -# def test_get_parameter_as_dict_should_get_and_describe(): -# value = "empty desc demo" -# type_name = "SecureString" -# description = "the description" -# ssm_mock = MagicMock() -# ssm_mock.get_parameter.return_value = { -# "Parameter": {"Name": "fakeparam", "Type": type_name, "Value": value} -# } -# ssm_mock.describe_parameters.return_value = { -# "Parameters": [ -# {"Name": "fakeparam", "Type": "SecureString", "Description": description} -# ] -# } -# ssm_dict = aws_ssm_dict(ssm_client=ssm_mock, return_type="dict") -# returned_param = ssm_dict.get_param_as_dict("fakeparam") -# assert ( -# returned_param["value"] == value -# and returned_param["type"] == type_name -# and returned_param["description"] == description -# ), "return parameter mismatch" -# ssm_mock.get_parameter.assert_called() -# ssm_mock.describe_parameters.assert_called() - - -# maybe needed in near future - changed implmenetation for now -# -# def test_get_parameter_desc_should_retry_and_then_return(): -# description = "the description" -# ssm_mock = MagicMock() -# ssm_mock.exceptions.ParameterNotFound = Exception -# ssm_mock.describe_parameters.side_effect = [ -# {"Parameters": []}, -# {"Parameters": []}, -# {"Parameters": []}, -# {"Parameters": []}, -# {"Parameters": []}, -# {"Parameters": []}, -# { -# "Parameters": [ -# { -# "Name": "fakeparam", -# "Type": "SecureString", -# "Description": description, -# } -# ] -# }, -# ] -# ssm_dict = aws_ssm_dict(ssm_client=ssm_mock, return_type="dict") - -# def mock_sleep(time): -# pass - -# with patch("backup_cloud_ssm.aws_ssm_dict.sleep"): -# returned_desc = ssm_dict.desc_param("fakeparam") -# assert returned_desc["Parameters"][0]["Name"] == "fakeparam" -# ssm_mock.describe_parameters.assert_called() diff --git a/testing/test_parameter_storage.py b/testing/test_parameter_storage.py index 335693c..c887bad 100644 --- a/testing/test_parameter_storage.py +++ b/testing/test_parameter_storage.py @@ -54,6 +54,7 @@ def store_key_with_types(ssm_dict, key, value, type_name, description): ssm_dict[key] = {"value": value, "type": type_name, "description": description} +@sometimes_mock_ssm @pytest.mark.wip def test_store_restore_keys_with_common_start(): key = "/test/fake/key_part" @@ -150,6 +151,7 @@ def test_store_returns_input_with_type(value, type_name, description): ), "return parameter mismatch" +@sometimes_mock_ssm def test_empty_string_is_stored_as_none_and_returned_as_empty(): value = "empty desc demo" type_name = "SecureString" From e85d9529ec239b215f1175188fcdedd76c25c7c5 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Thu, 28 Nov 2019 12:34:55 +0000 Subject: [PATCH 10/14] Minor fixes based on review --- Makefile | 5 ++--- README.md | 4 ++++ backup_cloud_ssm/aws_ssm_dict.py | 5 +---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 6aed904..9a1000d 100644 --- a/Makefile +++ b/Makefile @@ -2,14 +2,13 @@ AWS_ACCOUNT_NAME ?= michael AWS_DEFAULT_REGION ?= eu-west-1 PYTHON ?= python3 BEHAVE ?= behave -KEYFILE ?=.anslk_random_testkey LIBFILES := $(shell find backup_cloud_ssm -name '*.py') all: lint test # pytest-mocked is much faster than non-mocked which is slower even than -# the functional tests so run it first, then behave then ffinally the +# the functional tests so run it first, then behave then finally the # full pytest tests so that failures are detected early where possible. test: develop pytest-mocked behave pytest @@ -23,7 +22,7 @@ pytest: pytest wip: develop - behave --wip + $(BEHAVE) --wip lint: pre-commit install --install-hooks diff --git a/README.md b/README.md index 3fbbe42..367ae63 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,10 @@ set of SSM parameters. 1) set up the appropriate environment including AWS variables + export AWS_REGION=us-west-2 + export AWS_ACCESS_KEY_ID=AKIABCDEFGHIJKLMNOPQ + export AWS_SECRET_ACCESS_KEY=1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZABCD + 2) to backup run aws-ssm-backup > `` diff --git a/backup_cloud_ssm/aws_ssm_dict.py b/backup_cloud_ssm/aws_ssm_dict.py index 04cabfe..c0b6827 100644 --- a/backup_cloud_ssm/aws_ssm_dict.py +++ b/backup_cloud_ssm/aws_ssm_dict.py @@ -23,10 +23,7 @@ def __init__( self, decrypt=True, return_type="value", region_name=None, ssm_client=None ): if ssm_client is None: - if region_name is not None: - self.ssm = boto3.client("ssm", region_name=region_name) - else: - self.ssm = boto3.client("ssm", region_name=region_name) + self.ssm = boto3.client("ssm", region_name=region_name) else: self.ssm = ssm_client self.exceptions = self.ssm.exceptions From 377e36c7f09ff176150436f8ac8ab5fb4a7c463d Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Thu, 28 Nov 2019 13:53:55 +0000 Subject: [PATCH 11/14] add clean to Makefile and requirements.txt --- Makefile | 2 ++ requirements.txt | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 requirements.txt diff --git a/Makefile b/Makefile index 9a1000d..a4359a0 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,8 @@ lint: pre-commit install --install-hooks pre-commit run -a +clean: + python setup.py clean --all # develop is needed to install scripts that are called during testing develop: .develop.makestamp diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1d4d0f9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pytest +hypothesis +PyHamcrest +moto From e4f85c066bc36c4d7ec58416f090e466b1e154f9 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Mon, 2 Dec 2019 17:59:53 +0000 Subject: [PATCH 12/14] further readme and Makefile minor edits --- Makefile | 11 +++++++++++ README.md | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a4359a0..52e96b3 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,19 @@ AWS_DEFAULT_REGION ?= eu-west-1 PYTHON ?= python3 BEHAVE ?= behave +export AWS_DEFAULT_REGION + LIBFILES := $(shell find backup_cloud_ssm -name '*.py') +# we want to automate all the setup but we don't want to do it by surprise so we default +# to aborting with a message to correct things +abort: + @echo "***************************************************************************" + @echo "* please run 'make all' to install library and programs locally then test *" + @echo "***************************************************************************" + @echo + exit 2 + all: lint test # pytest-mocked is much faster than non-mocked which is slower even than diff --git a/README.md b/README.md index 367ae63..760375f 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ set of SSM parameters. 2) to backup run - aws-ssm-backup > `` + aws-ssm-backup > `` 3) to restore run - aws-ssm-backup --restore > `` + aws-ssm-backup --restore > `` Special notes: From d76b3d604f490f8cd2373d408b8fb66d6d790026 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Mon, 2 Dec 2019 18:25:15 +0000 Subject: [PATCH 13/14] 8 spaces for github markdown code --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 760375f..c32701d 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,17 @@ set of SSM parameters. 1) set up the appropriate environment including AWS variables - export AWS_REGION=us-west-2 - export AWS_ACCESS_KEY_ID=AKIABCDEFGHIJKLMNOPQ - export AWS_SECRET_ACCESS_KEY=1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZABCD + export AWS_REGION=us-west-2 + export AWS_ACCESS_KEY_ID=AKIABCDEFGHIJKLMNOPQ + export AWS_SECRET_ACCESS_KEY=1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZABCD 2) to backup run - aws-ssm-backup > `` + aws-ssm-backup > `` 3) to restore run - aws-ssm-backup --restore > `` + aws-ssm-backup --restore > `` Special notes: From cd45e426a357aa0606fc0579598342d4fa1b8787 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Tue, 3 Dec 2019 11:33:26 +0000 Subject: [PATCH 14/14] further makefile documentation according to review request --- Makefile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 52e96b3..b659f62 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ LIBFILES := $(shell find backup_cloud_ssm -name '*.py') # we want to automate all the setup but we don't want to do it by surprise so we default # to aborting with a message to correct things + +## default rule to avoid installing software without warning abort: @echo "***************************************************************************" @echo "* please run 'make all' to install library and programs locally then test *" @@ -16,35 +18,45 @@ abort: @echo exit 2 +## run all normal tasks to setup and verify all: lint test # pytest-mocked is much faster than non-mocked which is slower even than # the functional tests so run it first, then behave then finally the # full pytest tests so that failures are detected early where possible. +## standard tests test: develop pytest-mocked behave pytest +## behave tests only - excludes @future tagged tests to be used for planning behave: behave --tags ~@future +## pytest based tests accellerated with AWS service mocking mostly pytest-mocked: MOCK_AWS=true pytest +## pytest based tests only (unit or property testing) pytest: pytest +## current behave work in progress test wip: develop $(BEHAVE) --wip + +## standard linting and reformatting lint: pre-commit install --install-hooks pre-commit run -a +## remove python working files - does not unininstall results of develop clean: python setup.py clean --all -# develop is needed to install scripts that are called during testing +## locally install scripts that are called during testing develop: .develop.makestamp +# implementation of develop but using a file to record a timestamp .develop.makestamp: setup.py backup_cloud_ssm/aws_ssm_cli.py $(LIBFILES) $(PYTHON) setup.py install --force $(PYTHON) setup.py develop