Skip to content

Commit

Permalink
Merge pull request #580 from zalando-stups/ci/fix-pipeline
Browse files Browse the repository at this point in the history
Fix testing pipeline
  • Loading branch information
hughcapet authored Mar 27, 2023
2 parents 841a68e + e1bc0f5 commit 6c53449
Show file tree
Hide file tree
Showing 8 changed files with 76 additions and 66 deletions.
4 changes: 3 additions & 1 deletion delivery.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ notifications:
pipeline:
- id: build
type: script
overlay: ci/python
vm_config:
type: linux
image: cdp-runtime/python-3.9
commands:
- desc: "Install dependencies"
cmd: pip install -r requirements.txt
Expand Down
52 changes: 39 additions & 13 deletions senza/arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,50 @@
# pylint: disable=locally-disabled, invalid-name
import boto3.session
import click
import configparser
import os

from .error_handling import HandleExceptions


def get_region(region):
"""
Ensure region value.
If region is not provided, get one from the config file.
Raise error if still no region is set at the end.
"""
if not region:
config = configparser.ConfigParser()
try:
config.read(os.path.expanduser("~/.aws/config"))
if "default" in config:
region = config["default"]["region"]
except Exception:
pass

if not region:
raise click.UsageError(
"Please specify the AWS region on the "
"command line (--region) or in ~/.aws/config"
)

return region


def validate_region(ctx, param, value): # pylint: disable=locally-disabled, unused-argument
"""Validate Click region param parameter."""

if value is not None:
session = boto3.session.Session()
valid_regions = session.get_available_regions('cloudformation')
if value not in valid_regions:
valid_regions.sort()
raise click.BadParameter("'{}'. Region must be one of the "
"following AWS regions:\n"
" - {}".format(value,
"\n - ".join(valid_regions)))
"""Validate Click region parameter."""

value = get_region(value) # ensure region is set

session = boto3.session.Session()
valid_regions = session.get_available_regions('cloudformation')
if value not in valid_regions:
valid_regions.sort()
raise click.BadParameter("'{}'. Region must be one of the "
"following AWS regions:\n"
" - {}".format(value,
"\n - ".join(valid_regions)))

return value


Expand Down Expand Up @@ -74,5 +102,3 @@ def set_stacktrace_visible(ctx, param, value): # pylint: disable=locally-disabl
metavar='NAME',
multiple=True,
help='Specify field to be returned')

GLOBAL_OPTIONS = {}
44 changes: 1 addition & 43 deletions senza/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import base64
import calendar
import collections
import configparser
import datetime
import ipaddress
import json
Expand Down Expand Up @@ -37,7 +36,6 @@
from senza.components.elastigroup import ELASTIGROUP_RESOURCE_TYPE
from .spotinst.components import elastigroup_api
from .arguments import (
GLOBAL_OPTIONS,
json_output_option,
output_option,
parameter_file_option,
Expand All @@ -46,6 +44,7 @@
watch_option,
watchrefresh_option,
field_option,
get_region
)
from .aws import (
StackReference,
Expand Down Expand Up @@ -432,27 +431,6 @@ def read_parameter_file(parameter_file):
return tuple(paras)


def get_region(region):
if not region:
region = GLOBAL_OPTIONS.get("region")
if not region:
config = configparser.ConfigParser()
try:
config.read(os.path.expanduser("~/.aws/config"))
if "default" in config:
region = config["default"]["region"]
except Exception:
pass

if not region:
raise click.UsageError(
"Please specify the AWS region on the "
"command line (--region) or in ~/.aws/config"
)

return region


def check_credentials(region):
iam = BotoClientProxy("iam")
return iam.list_account_aliases()
Expand Down Expand Up @@ -523,7 +501,6 @@ def all_with_version(stack_refs: list):
def list_stacks(region, stack_ref, all, output, field, w, watch):
"""List Cloud Formation stacks"""

region = get_region(region)
check_credentials(region)

stack_refs = get_stack_refs(stack_ref)
Expand Down Expand Up @@ -652,7 +629,6 @@ def get_classic_load_balancer_metrics(cloudwatch, lb_name, start, now):
def health(region, stack_ref, all, output, field, w, watch):
"""Show stack health (ELB req/s, ..)"""

region = get_region(region)
check_credentials(region)

cloudwatch = BotoClientProxy("cloudwatch", region)
Expand Down Expand Up @@ -753,7 +729,6 @@ def create(
):
"""Create a new Cloud Formation stack from the given Senza definition file"""

region = get_region(region)
data = create_cf_template(
definition, region, version, parameter, force, parameter_file
)
Expand Down Expand Up @@ -825,7 +800,6 @@ def update(
"""Update an existing Cloud Formation stack from the given Senza
definition file"""

region = get_region(region)
data = create_cf_template(
definition, region, version, parameter, force, parameter_file
)
Expand All @@ -844,7 +818,6 @@ def update(
def print_cfjson(definition, region, version, parameter, output, force, parameter_file):
"""Print the generated Cloud Formation template"""

region = get_region(region)
data = create_cf_template(
definition, region, version, parameter, force, parameter_file, pretty=True
)
Expand Down Expand Up @@ -953,7 +926,6 @@ def delete(stack_ref, region, dry_run, force, interactive, ignore_non_existent):
"""Delete a single Cloud Formation stack"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

if not stack_refs:
Expand Down Expand Up @@ -1015,7 +987,6 @@ def resources(stack_ref, region, w, watch, output, field):
"""Show all resources of a single Cloud Formation stack"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)
cf = BotoClientProxy("cloudformation", region)

Expand Down Expand Up @@ -1063,7 +1034,6 @@ def events(stack_ref, region, w, watch, output, field):
"""Show all Cloud Formation events for a single stack"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)
cf = BotoClientProxy("cloudformation", region)

Expand Down Expand Up @@ -1120,7 +1090,6 @@ def events(stack_ref, region, w, watch, output, field):
def init(definition_file, region, template, user_variable):
"""Initialize a new Senza definition"""

region = get_region(region)
check_credentials(region)
account_info = AccountArguments(region=region)

Expand Down Expand Up @@ -1263,7 +1232,6 @@ def instances(
"""List the stack's EC2 instances"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

ec2 = boto3.resource("ec2", region)
Expand Down Expand Up @@ -1358,7 +1326,6 @@ def status(stack_ref, region, output, field, w, watch):
"""Show stack status information"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

ec2 = boto3.resource("ec2", region)
Expand Down Expand Up @@ -1451,7 +1418,6 @@ def domains(stack_ref, region, output, field, w, watch):
"""List the stack's Route53 domains"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

cf = boto3.resource("cloudformation", region)
Expand Down Expand Up @@ -1560,7 +1526,6 @@ def traffic(stack_name, stack_version, percentage, region, output, timeout, inte
"""Route traffic to a specific stack (weighted DNS record)"""

stack_refs = get_stack_refs([stack_name, stack_version])
region = get_region(region)
check_credentials(region)

with OutputFormat(output):
Expand Down Expand Up @@ -1593,7 +1558,6 @@ def images(stack_ref, region, output, field, hide_older_than, show_instances):
"""Show all used AMIs and available Taupage AMIs"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

ec2 = boto3.resource("ec2", region)
Expand Down Expand Up @@ -1730,7 +1694,6 @@ def console(instance_or_stack_ref, limit, region, w, watch):
# filter out instances not part of any stack
filters = [{"Name": "tag-key", "Values": ["aws:cloudformation:stack-name"]}]

region = get_region(region)
check_credentials(region)

ec2 = boto3.resource("ec2", region)
Expand Down Expand Up @@ -1784,7 +1747,6 @@ def dump(stack_ref, region, output):
"""Dump Cloud Formation template of existing stack"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

cf = BotoClientProxy("cloudformation", region)
Expand Down Expand Up @@ -1848,7 +1810,6 @@ def patch(stack_ref, region, image, instance_type, user_data):
Currently supports patching ASG launch configurations and ElastiGroup groups."""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

if image in taupage.CHANNELS:
Expand Down Expand Up @@ -1948,7 +1909,6 @@ def respawn_instances(stack_ref, inplace, force, batch_size_percentage, batch_pe
Performs a rolling update to prevent downtimes."""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

stacks = get_stacks(stack_refs, region)
Expand Down Expand Up @@ -1980,7 +1940,6 @@ def scale(stack_ref, region, desired_capacity, min_size, force):
"""Scale Auto Scaling Group to desired capacity"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)

stacks = get_stacks(stack_refs, region)
Expand Down Expand Up @@ -2120,7 +2079,6 @@ def wait(stack_ref, region, deletion, timeout, interval):
"""

stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
cf = BotoClientProxy("cloudformation", region)

target_status = (
Expand Down
2 changes: 1 addition & 1 deletion senza/components/elastic_load_balancer_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def component_elastic_load_balancer_v2(definition,
version = '{}-{}'.format(info["StackVersion"],
configuration['NameSuffix'])
loadbalancer_name = get_load_balancer_name(info["StackName"], version)
del(configuration['NameSuffix'])
del (configuration['NameSuffix'])
else:
loadbalancer_name = get_load_balancer_name(info["StackName"],
info["StackVersion"])
Expand Down
8 changes: 2 additions & 6 deletions senza/subcommands/root.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import senza
from clickclick import AliasedGroup, warning

from ..arguments import GLOBAL_OPTIONS, region_option
from ..error_handling import sentry

CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
Expand Down Expand Up @@ -132,16 +131,13 @@ def print_version(ctx, param, value):
is_eager=True,
help="Print the current version number and exit.",
)
@region_option
def cli(region):
def cli():
"""
Senza's root command.
It checks the version and sets the region global option before executing
the sub-commands.
Checks the version.
Sub command can be added by using `cli.add_command(SUB_COMMAND_FUNCTION)`
or using the `@cli.command()` decorator
"""
check_senza_version(senza.__version__)
GLOBAL_OPTIONS["region"] = region
2 changes: 1 addition & 1 deletion senza/templates/postgresapp.py
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ def gather_user_variables(variables, region, account_info):
variables['use_ebs'] = True

if variables['use_ebs']:
prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=defaults['volume_size'])
prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=str(defaults['volume_size']))
prompt(variables, 'volume_type', 'Database volume type (gp2, io1 or standard)',
default=defaults['volume_type'])
if variables['volume_type'] == 'io1':
Expand Down
20 changes: 19 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import setuptools
from setuptools.command.test import test as TestCommand
from setuptools import setup
from setuptools import setup, Command

if sys.version_info < (3, 4, 0):
sys.stderr.write('FATAL: STUPS Senza needs to be run with Python 3.4+\n')
Expand Down Expand Up @@ -91,6 +91,23 @@ def run_tests(self):
sys.exit(errno)


class Flake8(Command):

user_options = []

def initialize_options(self):
pass

def finalize_options(self):
pass

def run(self):
from flake8.main.cli import main

errno = main(['senza'])
sys.exit(errno)


def get_install_requirements(path):
content = open(os.path.join(__location__, path)).read()
return [req for req in content.split('\\n') if req != '']
Expand All @@ -104,6 +121,7 @@ def setup_package():
# Assemble additional setup commands
cmdclass = {}
cmdclass['test'] = PyTest
cmdclass['flake8'] = Flake8

install_reqs = get_install_requirements('requirements.txt')

Expand Down
10 changes: 10 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,16 @@ def my_resource(rtype, *args):
monkeypatch.setattr('boto3.resource', my_resource)
monkeypatch.setattr('senza.cli.AccountArguments', MagicMock())

kms_key = {'KeyId': 'key_a',
'aliases': ['something'],
'Description': 'This is key a',
'Arn': 'arn:partition:service:region:account-id:resource-id'}
monkeypatch.setattr('senza.templates.postgresapp.list_kms_keys',
MagicMock(return_value=[kms_key]))
monkeypatch.setattr('senza.templates.postgresapp.choice',
MagicMock(return_value='{}: {}'.format(kms_key['KeyId'], kms_key['Description'])))
monkeypatch.setattr('senza.templates.postgresapp.encrypt', MagicMock(return_value='encrypted_string'))

runner = CliRunner()

with runner.isolated_filesystem():
Expand Down

0 comments on commit 6c53449

Please sign in to comment.