diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000..e56a83b --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,34 @@ +name: Python package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + max-parallel: 3 + matrix: + python-version: [3.5, 3.6, 3.7] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --ignore=D203,D100,D103,D202,D200,W504 --max-complexity=10 --max-line-length=120 --statistics + - name: Test with pytest + run: | + pip install pytest + pytest --tb native -ra -v -s diff --git a/CHANGELOG b/CHANGELOG index 696bd38..daa3bb9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -8,7 +8,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - List yet unreleased changes. +## [0.2.0] - 2019-11-16 +### Changed +- Dependency versions updated to current libjuju (waiting only for macaroonbakery to also support python 3.8) +- Allow upgrade without using origin - if not supported by charm - origin now has to be specified explicitly +- Charm upgrade does not check locally built ('local:app') charms +- When using upgrade-only, charms are not compared with charmstore versions, skipping straight to upgrade of services +- Improved wait on charm config change when setting origin +### Added +- Support passing of --cacert $CACERT parameter in authentication using libjuju when accessing from remote machine +- Origin keys can be specified as parameter (key=value,key2=value2) +- Upgrade action name as parameters - defaulting to 'openstack-upgrade' in perform_upgrade +- Support key value parameters for upgrade action (key=value,key2=value2) +- Debug parameter - default log level without debug is INFO +- Async (class) methods mocking for unit-testing of jujuna code +- Tests for upgrade - perform_upgrade (covering simple, rolling and hacluster cases) + ## [0.1.0] - 2018-10-24 ### Added - Initial opensourcing commit - diff --git a/VERSION b/VERSION index c946ee6..0ea3a94 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.6 +0.2.0 diff --git a/jujuna/__main__.py b/jujuna/__main__.py index 874419f..28aa5b9 100644 --- a/jujuna/__main__.py +++ b/jujuna/__main__.py @@ -8,6 +8,8 @@ from juju import loop +from jujuna.helper import log_traceback + from jujuna.deploy import deploy # noqa from jujuna.upgrade import upgrade # noqa from jujuna.tests import test # noqa @@ -15,14 +17,14 @@ logger = logging.getLogger('jujuna') -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.INFO) # Double logging cleanup while logger.handlers: logger.handlers.pop() logHandler = logging.StreamHandler(sys.stdout) -logHandler.setLevel(logging.DEBUG) +logHandler.setLevel(logging.INFO) logFormatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') logHandler.setFormatter(logFormatter) logger.addHandler(logHandler) @@ -30,7 +32,6 @@ def get_parser(): parser = argparse.ArgumentParser( - # formatter_class=utils.ParagraphDescriptionFormatter, description="Deploy a local bundle, execute upgrade procedure, " "run the deployment through a suite of tests to ensure " "that it can handle the types of operations and failures " @@ -51,17 +52,21 @@ def get_parser(): help="Juju endpoint (requires model uuid instead of name)") p_deploy.add_argument("--username", default=None, dest="username", help="Juju username") p_deploy.add_argument("--password", default=None, dest="password", help="Juju password") + p_deploy.add_argument("--cacert", default=None, dest="cacert", help="Juju CA certificate") + p_deploy.add_argument("--debug", action='store_true', help="Log level debug.") p_upgrade = subparsers.add_parser('upgrade', help="Upgrade applications deployed in the current or selected model") p_upgrade.add_argument("-c", "--controller", default=None, dest="ctrl_name", help="Controller (def: current)") p_upgrade.add_argument("-m", "--model", default=None, dest="model_name", help="Model to use instead of current") - p_upgrade.add_argument("-o", "--origin", default='cloud:xenial-ocata', dest="origin", + p_upgrade.add_argument("-o", "--origin", default='', dest="origin", help="""Openstack origin: - cloud:xenial-newton, - cloud:xenial-ocata (default), - cloud:xenial-pike, - cloud:xenial-queens, - cloud:bionic-rocky + 'cloud:xenial-newton', + 'cloud:xenial-ocata', + 'cloud:xenial-pike', + 'cloud:xenial-queens', + 'cloud:bionic-rocky', + 'cloud:bionic-stein', + 'cloud:bionic-train', """) p_upgrade.add_argument("-a", "--apps", nargs='*', default=[], help="Apps to be upgraded (ordered)") p_upgrade.add_argument("-i", "--ignore-errors", action='store_true', dest='ignore_errors', @@ -70,8 +75,14 @@ def get_parser(): p_upgrade.add_argument("-e", "--evacuate", action='store_true', help="Evacuate nova-compute nodes during upgrade") p_upgrade.add_argument("--upgrade-only", action='store_true', dest="upgrade_only", help="Upgrade using upgrade hooks without changing the revision") - p_upgrade.add_argument("--charms-only", action='store_true', + p_upgrade.add_argument("--charms-only", action='store_true', dest="charms_only", help="Upgrade only charms without running upgrade hooks") + p_upgrade.add_argument("--upgrade-action", dest="upgrade_action", default=None, + help="Action name to upgrade application") + p_upgrade.add_argument("--upgrade-params", dest="upgrade_params", default=None, + help="Action parameters comma separated e.g. 'service=name,version=2'") + p_upgrade.add_argument("--origin-keys", dest="origin_keys", default=None, + help="Config keys to set origin in apps e.g. 'ceph-mon=source,ceph-mon=source'") p_upgrade.add_argument("--dry-run", action='store_true', dest="dry_run", help="Dry run - only show changes without upgrading") p_upgrade.add_argument("-t", "--timeout", default=0, type=int, help="Timeout after N seconds.") @@ -81,6 +92,8 @@ def get_parser(): help="Juju endpoint (requires model uuid instead of name)") p_upgrade.add_argument("--username", default=None, dest="username", help="Juju username") p_upgrade.add_argument("--password", default=None, dest="password", help="Juju password") + p_upgrade.add_argument("--cacert", default=None, dest="cacert", help="Juju CA certificate") + p_upgrade.add_argument("--debug", action='store_true', help="Log level debug.") p_test = subparsers.add_parser('test', help="Test applications in the current or selected model") p_test.add_argument("test_suite", type=argparse.FileType('r'), @@ -92,6 +105,8 @@ def get_parser(): help="Juju endpoint (requires model uuid instead of name)") p_test.add_argument("--username", default=None, dest="username", help="Juju username") p_test.add_argument("--password", default=None, dest="password", help="Juju password") + p_test.add_argument("--cacert", default=None, dest="cacert", help="Juju CA certificate") + p_test.add_argument("--debug", action='store_true', help="Log level debug.") p_clean = subparsers.add_parser( 'clean', @@ -109,14 +124,11 @@ def get_parser(): help="Juju endpoint (requires model uuid instead of name)") p_clean.add_argument("--username", default=None, dest="username", help="Juju username") p_clean.add_argument("--password", default=None, dest="password", help="Juju password") + p_clean.add_argument("--cacert", default=None, dest="cacert", help="Juju CA certificate") + p_clean.add_argument("--debug", action='store_true', help="Log level debug.") argcomplete.autocomplete(parser) - # options = add_bundle_opts(options, parser) - - # if not utils.valid_bundle_or_spell(options.path): - # parser.error('Invalid bundle directory: %s' % options.path) - # configLogging(options) return parser @@ -129,47 +141,70 @@ async def run_action(action, timeout, args): """ selected_action = globals()[action] - # Remove used vars - del vars(args)['action'] - if 'timeout' in vars(args): - del vars(args)['timeout'] - - if timeout: + if timeout == 0: + return await selected_action(**args) + else: at = None try: async with async_timeout.timeout(timeout) as at: - return await selected_action(**vars(args)) - except Exception: + ret = await selected_action(**args) + except Exception as e: # Exit with timeout code if expired if at and at.expired: - logger.warn('Operation timed out!') - return 124 + ret = 124 + logger.warn('Action {} timed out!'.format(action)) else: - raise + ret = 1 + logger.warn('Action {} failed!'.format(action)) + logger.warn(str(e)) + return ret + + +def load_kv_arg(args, item): + if args.get(item, False): + up_list = args[item].split(',') + upgrade_params = dict([ + item.split('=') if '=' in item else [item, True] for item in up_list + ]) + args[item] = upgrade_params else: - return await selected_action(**vars(args)) + args[item] = {} + return args -def action(args): +def parse_args(argv): + parser = get_parser() try: - if hasattr(args, 'action'): - return loop.run(run_action(args.action, args.timeout, args)) - except Exception as e: - logger.warn(e) - return 1 - - return 255 + parsed = parser.parse_args(argv) + args = vars(parsed) + action = args.pop('action') + timeout = args.pop('timeout') - -def parse_args(args): - parser = get_parser() - return parser.parse_args(args) + args = load_kv_arg(args, 'upgrade_params') + args = load_kv_arg(args, 'origin_keys') + except Exception as e: + log_traceback(e) + parser.print_help() + sys.exit(1) + try: + if args.get('debug', False): + logger.setLevel(logging.DEBUG) + logHandler.setLevel(logging.DEBUG) + except Exception: + pass + return action, timeout, args def main(): - args = parse_args(sys.argv[1:]) + action, timeout, args = parse_args(sys.argv[1:]) - ret = action(args) + try: + ret = loop.run( + run_action(action, timeout, args) + ) + except Exception as e: + log_traceback(e) + ret = 1 sys.exit(ret) diff --git a/jujuna/clean.py b/jujuna/clean.py index 3501301..21c5946 100644 --- a/jujuna/clean.py +++ b/jujuna/clean.py @@ -42,15 +42,17 @@ async def _block(log_count): async def clean( - ctrl_name=None, - model_name=None, + ctrl_name='', + model_name='', ignore=[], wait=False, force=False, dry_run=False, - endpoint=None, - username=None, - password=None + endpoint='', + username='', + password='', + cacert='', + **kwargs ): """Destroy applications present in the current or selected model. @@ -66,13 +68,15 @@ async def clean( :param endpoint: string :param username: string :param password: string + :param cacert: string """ controller, model = await connect_juju( ctrl_name, model_name, endpoint=endpoint, username=username, - password=password + password=password, + cacert=cacert ) try: diff --git a/jujuna/deploy.py b/jujuna/deploy.py index 755ada8..3f0b768 100644 --- a/jujuna/deploy.py +++ b/jujuna/deploy.py @@ -3,8 +3,8 @@ import asyncio import websockets import logging -from collections import defaultdict -from jujuna.helper import connect_juju +from collections import Counter +from jujuna.helper import connect_juju, log_traceback from juju.errors import JujuError @@ -14,12 +14,14 @@ async def deploy( bundle_file, - ctrl_name=None, - model_name=None, + ctrl_name='', + model_name='', wait=False, - endpoint=None, - username=None, - password=None + endpoint='', + username='', + password='', + cacert='', + **kwargs ): """Deploy a local juju bundle. @@ -35,6 +37,7 @@ async def deploy( :param endpoint: string :param username: string :param password: string + :param cacert: string """ log.info('Reading bundle: {}'.format(bundle_file.name)) entity_url = 'local:' + bundle_file.name.replace('/bundle.yaml', '') @@ -44,37 +47,51 @@ async def deploy( model_name, endpoint=endpoint, username=username, - password=password + password=password, + cacert=cacert ) try: # Deploy a bundle log.info("Deploy: {}".format(entity_url)) - deployed_app = await model.deploy( + deployed_apps = await model.deploy( entity_url ) if wait: await wait_until( model, - deployed_app, + deployed_apps, loop=model.loop ) + try: + d = Counter([a.status for a in deployed_apps]) + except Exception: + log.error('Collecting status failed') + d = {} + log.info('{} - Machines: {} Apps: {} Stats: {}'.format( + 'DEPLOYED' if wait else 'CURRENT', + len(model.machines), + len(model.applications), + dict(d) + )) + except JujuError as e: - log.info(str(e)) + log.error('JujuError during deploy') + log_traceback(e) finally: # Disconnect from the api server and cleanup. await model.disconnect() await controller.disconnect() -async def wait_until(model, deployed_app, log_time=5, timeout=None, wait_period=0.5, loop=None): +async def wait_until(model, apps, log_time=5, timeout=None, wait_period=0.5, loop=None): """Blocking with logs. Return only after all conditions are true. :param model: juju model - :param deployed_app: juju application + :param apps: juju application :param log_time: logging frequency (s) :param timeout: blocking timeout (s) :param wait_period: waiting time between checks (s) @@ -86,28 +103,28 @@ def _disconnected(): return not (model.is_connected() and model.connection().is_open) async def _block(log_count): - while not _disconnected() and not all(a.status == 'active' for a in deployed_app): + blockable = ['maintenance', 'blocked', 'waiting', 'error'] + while ( + not _disconnected() and + any(a.status != 'active' for a in apps) and + any(u.workload_status in blockable for a in apps for u in a.units) + ): await asyncio.sleep(wait_period, loop=loop) log_count += 0.5 if log_count % log_time == 0: - d = defaultdict(int) - for a in deployed_app: - d[a.status] += 1 - log.info('[RUNNING] Machines: {} Apps: {} Stats: {}'.format( + ass = Counter([ + a.status for a in apps + ]) + wss = Counter([ + unit.workload_status for a in apps for unit in a.units + ]) + log.info('PROGRESS - Machines: {} Apps: {} Stats: {} Workload: {}'.format( len(model.machines), len(model.applications), - dict(d) + dict(ass), + dict(wss) )) await asyncio.wait_for(_block(log_count), timeout, loop=loop) if _disconnected(): raise websockets.ConnectionClosed(1006, 'no reason') - - d = defaultdict(int) - for a in deployed_app: - d[a.status] += 1 - log.info('[DONE] Machines: {} Apps: {} Stats: {}'.format( - len(model.machines), - len(model.applications), - dict(d) - )) diff --git a/jujuna/helper.py b/jujuna/helper.py index c6296b2..d298f8e 100644 --- a/jujuna/helper.py +++ b/jujuna/helper.py @@ -1,3 +1,5 @@ +import logging +import traceback from jujuna.settings import MAX_FRAME_SIZE @@ -7,42 +9,50 @@ from theblues.charmstore import CharmStore +def log_traceback(ex, prefix=''): + if prefix and isinstance(prefix, str): + prefix = '{} - '.format(prefix.strip()) + ex_traceback = ex.__traceback__ + for line in traceback.format_exception( + ex.__class__, ex, ex_traceback + ): + logging.error('{}{}'.format(prefix, line.rstrip('\n'))) + + def cs_name_parse(name, series=None): """Charm store name parse. """ try: - cut = name.split(':', 1)[1] - arr = cut.split('/') if '/' in cut else ['', cut] + cscode, csuri = name.split(':', 1) + items = csuri.split('/') if '/' in csuri else ['', csuri] return { - 'series': arr[0] if arr[0] or not series else series, - 'charm': '-'.join(arr[1].split('-')[:-1]), - 'revision': int(arr[1].split('-')[-1]) + 'series': items[0] if items[0] or not series else series, + 'charm': '-'.join(items[1].split('-')[:-1]), + 'revision': int(items[1].split('-')[-1]), + 'charmstore': False if cscode == 'local' else True } - except: # noqa - print(name) + except Exception: + logging.warn('Failed to parse Charm name: {}'.format(name)) return {} -async def connect_juju(ctrl_name=None, model_name=None, endpoint=None, username=None, password=None): +async def connect_juju(ctrl_name=None, model_name=None, endpoint=None, username=None, password=None, cacert=None): controller = Controller(max_frame_size=MAX_FRAME_SIZE) # noqa - if ctrl_name: - if endpoint: - await controller.connect(endpoint=endpoint, username=username, password=password) - else: - await controller.connect(ctrl_name) + if endpoint: + await controller.connect(endpoint=endpoint, username=username, password=password, cacert=cacert) else: - await controller.connect_current() + await controller.connect(ctrl_name) if endpoint: model = Model(max_frame_size=MAX_FRAME_SIZE) - await model.connect(uuid=model_name, endpoint=endpoint, username=username, password=password) + await model.connect(uuid=model_name, endpoint=endpoint, username=username, password=password, cacert=cacert) elif model_name: model = await controller.get_model(model_name) else: model = Model(max_frame_size=MAX_FRAME_SIZE) # noqa - await model.connect_current() + await model.connect() # HACK low unsettable timeout in the model model.charmstore._cs = CharmStore(timeout=60) diff --git a/jujuna/tests.py b/jujuna/tests.py index 05732f5..de9a886 100644 --- a/jujuna/tests.py +++ b/jujuna/tests.py @@ -5,7 +5,7 @@ import async_timeout from collections import defaultdict from juju.errors import JujuError -from jujuna.helper import connect_juju +from jujuna.helper import connect_juju, log_traceback from jujuna.brokers.api import Api as ApiBroker from jujuna.brokers.file import File as FileBroker @@ -22,12 +22,14 @@ async def test( - test_suite=None, - ctrl_name=None, - model_name=None, - endpoint=None, - username=None, - password=None + test_suite='', + ctrl_name='', + model_name='', + endpoint='', + username='', + password='', + cacert='', + **kwargs ): """Run a test suite against applications deployed in the current or selected model. @@ -42,6 +44,7 @@ async def test( :param endpoint: string :param username: string :param password: string + :param cacert: string """ if test_suite: with open(test_suite.name, 'r') as stream: @@ -55,7 +58,8 @@ async def test( model_name, endpoint=endpoint, username=username, - password=password + password=password, + cacert=cacert ) model_passed, model_failed = 0, 0 @@ -94,7 +98,8 @@ async def test( log.info('[FINISHED] Passed tests: {} Failed tests: {}'.format(model_passed, model_failed)) except JujuError as e: - log.error(e.message) + log.error('JujuError during tests') + log_traceback(e) finally: # Disconnect from the api server and cleanup. await model.disconnect() diff --git a/jujuna/upgrade.py b/jujuna/upgrade.py index 28c8ecd..813acec 100644 --- a/jujuna/upgrade.py +++ b/jujuna/upgrade.py @@ -4,9 +4,9 @@ import websockets import async_timeout -from collections import defaultdict +from collections import Counter -from jujuna.helper import cs_name_parse, connect_juju +from jujuna.helper import cs_name_parse, connect_juju, log_traceback from jujuna.settings import ORIGIN_KEYS, SERVICES from juju.errors import JujuError @@ -20,17 +20,22 @@ async def upgrade( ctrl_name=None, model_name=None, apps=[], - origin='cloud:xenial-ocata', + origin='', ignore_errors=False, pause=False, evacuate=False, charms_only=False, upgrade_only=False, + upgrade_action='', + upgrade_params={}, + origin_keys={}, dry_run=False, - settings=False, - endpoint=None, - username=None, - password=None + settings=None, + endpoint='', + username='', + password='', + cacert='', + **kwargs ): """Upgrade applications deployed in the model. @@ -42,16 +47,20 @@ async def upgrade( :param ctrl_name: juju controller :param model_name: juju model name or uuid :param apps: ordered list of application names - :param origin: target openstack version string + :param origin: target openstack version string e.g. 'cloud:xenial-ocata' :param ignore_errors: boolean :param pause: boolean :param evacuate: boolean :param charms_only: boolean :param upgrade_only: boolean + :param upgrade_action: string + :param upgrade_params: dict + :param origin_keys: dict :param dry_run: boolean :param endpoint: string :param username: string :param password: string + :param cacert: string """ controller, model = await connect_juju( @@ -59,7 +68,8 @@ async def upgrade( model_name, endpoint=endpoint, username=username, - password=password + password=password, + cacert=cacert ) try: @@ -74,36 +84,25 @@ async def upgrade( except yaml.YAMLError as e: log.warn('Failed to load settings file: {}'.format(str(e))) - origin_keys = settings_data['origin_keys'] if 'origin_keys' in settings_data else ORIGIN_KEYS - services = settings_data['services'] if 'services' in settings_data else SERVICES - add_services = settings_data['add_services'] if 'add_services' in settings_data else [] + origin_keys = settings_data.get('origin_keys', origin_keys if origin_keys else ORIGIN_KEYS) + services = settings_data.get('services', SERVICES) + add_services = settings_data.get('add_services', []) # If apps are not specified in the order use configuration from settings if apps: services = apps add_services = [] - log.info('Services to upgrade: {}'.format(services)) - if add_services: - log.info('Charms only upgrade: {}'.format(add_services)) - - log.info('Upgrading charms') + log.info('Services to upgrade: {}'.format(', '.join(services))) + if add_services and not upgrade_only: + log.info('Charms only upgrade: {}'.format(', '.join(add_services))) + all_services = services + add_services # Upgrade charm revisions - upgraded, latest_charms = await upgrade_charms(model, services + add_services, upgrade_only, dry_run) - - wss = defaultdict(int) - wsm = defaultdict(int) - for app in model.applications.values(): - for unit in app.units: - wss[unit.workload_status] += 1 - if 'ready' not in unit.workload_status_message: - wsm[unit.workload_status_message] += 1 - log.info('Status of units after revision upgrade: {}'.format(dict(wss))) - log.info('Workload messages: {}'.format(dict(wsm))) - - if not ignore_errors and 'error' in wss.keys(): - raise Exception('Errors during upgrading charms to latest revision') + if not upgrade_only: + upgraded, latest_charms = await upgrade_charms( + model, all_services, dry_run, ignore_errors + ) # Ocata upgrade requires additional relation to succeed if not charms_only and origin == 'cloud:xenial-ocata' and 'nova-compute' in services: @@ -114,13 +113,13 @@ async def upgrade( # Upgrade services if not charms_only: - await upgrade_services(model, services, origin, origin_keys, pause, dry_run) + await upgrade_services( + model, services, origin, origin_keys, upgrade_action, upgrade_params, pause, dry_run + ) # Log status values - d = defaultdict(int) - for a in model.applications.values(): - d[a.status] += 1 - log.info('[STATUS] {}'.format(dict(d))) + d = Counter([a.status for a in model.applications.values()]) + log.info('STATUS: {}'.format(dict(d))) finally: await model.disconnect() @@ -135,30 +134,52 @@ async def ocata_relation_patch(model, dry_run, cinder_ceph): await model.add_relation('nova-compute:ceph-access', cinder_ceph_rel) # TODO add completion check await asyncio.sleep(120) + await wait_until( + model, + model.applications.values(), + timeout=1800, + loop=model.loop + ) log.info('Completed addition of relation') except Exception as e: if 'already exists' not in str(e): + log_traceback(e) raise e else: log.warn('Ignored: relation already exists') - await wait_until( - model, - model.applications.values(), - timeout=1800, - loop=model.loop - ) +def get_service_list(model, upgraded): + return [(name, get_service_version(model.applications.get(name, None))) for name in upgraded] + + +def get_service_version(application): + try: + serv_version = application.safe_data.get('workload-version', '') + except Exception: + serv_version = '' + return serv_version -async def upgrade_services(model, upgraded, origin, origin_keys, pause, dry_run): - log.info('Upgrading services') + +async def upgrade_services(model, upgraded, origin, origin_keys, upgrade_action, upgrade_params, pause, dry_run): + sl_before = get_service_list(model, upgraded) + log.info('Application upgrade order: {}'.format( + ', '.join(['{} ({})'.format(name, version) for name, version in sl_before]) + )) s_upgrade = 0 + # upgrade_action is none by default, otherwise enforcing perform_upgrade + use_action = upgrade_action if upgrade_action else 'openstack-upgrade' for app_name in upgraded: - if await is_rollable(model.applications[app_name]): - await perform_rolling_upgrade( + rollable_app = await is_rollable(model.applications[app_name], use_action) + if upgrade_action or rollable_app: + await perform_upgrade( + model, model.applications[app_name], origin_keys, + use_action, + upgrade_params, + rollable=rollable_app, origin=origin, pause=pause, dry_run=dry_run @@ -173,17 +194,21 @@ async def upgrade_services(model, upgraded, origin, origin_keys, pause, dry_run) ) s_upgrade += 1 - await wait_until( - model, - model.applications.values(), - timeout=1800, - loop=model.loop - ) + await wait_until( + model, + model.applications.values(), + timeout=1800, + loop=model.loop + ) + sl_after = get_service_list(model, upgraded) + log.info('Application upgrade order: {}'.format( + ', '.join(['{} ({}=>{})'.format(before[0], before[1], after[1]) for before, after in zip(sl_before, sl_after)]) + )) log.info('Upgrade finished ({} upgraded services)'.format(s_upgrade)) -async def upgrade_charms(model, apps, upgrade_only, dry_run): +async def upgrade_charms(model, apps, dry_run, ignore_errors): """Upgrade charm revisions in the model. Listed apps will be checked for new revisions in charmstore @@ -191,10 +216,10 @@ async def upgrade_charms(model, apps, upgrade_only, dry_run): :param model_name: juju model name or uuid :param apps: ordered list of application names - :param origin: target openstack version string - :param upgrade_only: boolean :param dry_run: boolean + :param ignore_errors: boolean """ + log.info('Upgrading charms') upgraded = [] latest_charms = [] @@ -208,25 +233,28 @@ async def upgrade_charms(model, apps, upgrade_only, dry_run): parse = cs_name_parse(charm_url) # Charmstore get latest revision - try: - charmstore_entity = await model.charmstore.entity( - charm_url, - include_stats=False, - includes=['revision-info'] - ) - latest = charmstore_entity['Meta']['revision-info']['Revisions'][0] - latest_revision = cs_name_parse(latest) - attemp_update = False - except Exception: - log.warn('Failed loading information from charmstore: {}'.format(charm_url)) + if parse.get('charmstore', False): + try: + charmstore_entity = await model.charmstore.entity( + charm_url, + include_stats=False, + includes=['revision-info'] + ) + latest = charmstore_entity['Meta']['revision-info']['Revisions'][0] + latest_revision = cs_name_parse(latest) + attemp_update = False + except Exception: + log.warn('Failed loading information from charmstore: {}'.format(charm_url)) + latest_revision = {'revision': 0} + attemp_update = True + else: + log.info('Not upgrading local charm: {}'.format(charm_url)) latest_revision = {'revision': 0} - attemp_update = True + attemp_update = False # Update if not newest or attempt to update if failed to find charmstore latest revision if ( - not upgrade_only and - parse['revision'] < latest_revision['revision'] or - attemp_update + (parse['revision'] < latest_revision['revision']) or attemp_update ): log.info('Upgrade {} from: {} to: {}'.format(app_name, parse['revision'], latest_revision['revision'])) try: @@ -241,17 +269,31 @@ async def upgrade_charms(model, apps, upgrade_only, dry_run): log.info('Upgraded: {} charms'.format(len(upgraded))) - await wait_until( - model, - model.applications.values(), - timeout=1800, - loop=model.loop - ) - - log.info('Collecting final workload status') if not dry_run and upgraded: - await asyncio.sleep(30) + await asyncio.sleep(20) + await wait_until( + model, + model.applications.values(), + timeout=1800, + loop=model.loop + ) + log.info('Collecting final workload status') + await asyncio.sleep(20) + + wss = Counter() + wsm = Counter() + for app in model.applications.values(): + if app.name in apps: + wss += Counter([u.workload_status for u in app.units]) + wsm += Counter([ + u.workload_status_message for u in app.units if 'ready' not in u.workload_status_message + ]) + log.info('Status of units after revision upgrade: {}'.format(dict(wss))) + log.info('Workload messages: {}'.format(dict(wsm))) + + if not ignore_errors and 'error' in wss.keys(): + raise Exception('Errors during upgrading charms to latest revision') return upgraded, latest_charms @@ -279,28 +321,33 @@ async def _block(log_count): await asyncio.sleep(wait_period, loop=loop) log_count += 0.5 if log_count % log_time == 0: - wss = defaultdict(int) - for app in model.applications.values(): - for unit in app.units: - wss[unit.workload_status] += 1 + wss = Counter([ + unit.workload_status for a in apps for unit in a.units + ]) log.info('[WAITING] Charm workload status: {}'.format(dict(wss))) + + await asyncio.sleep(2) await asyncio.wait_for(_block(log_count), timeout, loop=loop) if _disconnected(): raise websockets.ConnectionClosed(1006, 'no reason') -async def is_rollable(application): +async def is_rollable(application, upgrade_action): """Define whether the application is rollable. - Application is considered rollable if it provides openstack-upgrade action, is deployed with more than 1 units, + Application is considered rollable if it provides upgrade action + (e.g. openstack-upgrade), is deployed with more than 1 units, is not ceph and successfuly applies action-managed-upgrade config. :param application: juju application """ actions = await enumerate_actions(application) - if 'openstack-upgrade' not in actions: + if (not upgrade_action) or (upgrade_action not in actions): + log.warn('Upgrade action "{}" not in actions.'.format( + upgrade_action + )) return False if len(application.units) <= 1: @@ -310,7 +357,9 @@ async def is_rollable(application): log.info('Ceph is not rollable.') return False - if not await application.set_config({'action-managed-upgrade': 'True'}): + if upgrade_action == 'openstack-upgrade' and not await application.set_config( + {'action-managed-upgrade': 'True'} + ): log.warn('Failed to enable action-managed-upgrade mode.') return False @@ -338,7 +387,7 @@ def get_hacluster_subordinate_pairs(application): sub_pairs[unit.name] = sub_unit return sub_pairs - return None + return {} async def enumerate_actions(application): @@ -352,15 +401,15 @@ async def enumerate_actions(application): return actions.keys() -async def order_units(name, units): +async def order_units(label, units): """Determing order of units. Returns a list of units beginning with leader. - :param name: string + :param label: string :param units: list of juju units """ - log.info('Determining ordering for service: {}'.format(name)) + log.info('{} - Determining order of units'.format(label)) ordered = [] is_leader_data = [] @@ -379,92 +428,137 @@ async def order_units(name, units): else: ordered.append(unit) - log.info('Upgrade order is: {} Leader: {}'.format( - ', '.join([unit.name for unit in ordered]), - leader_unit + log.info('{} - Upgrade order is: {} (leader){}{}'.format( + label, + leader_unit, + ', ' if len(ordered) > 1 else '', + ', '.join([unit.name for unit in ordered if unit.name != leader_unit]), )) return ordered -async def perform_rolling_upgrade( +async def perform_upgrade( + model, application, origin_keys, + upgrade_action, + upgrade_params, dry_run=False, evacuate=False, + rollable=False, pause=False, origin='cloud:xenial-ocata' ): - """Perform rolling upgrade. + """Perform upgrade. Rolling upgrade is performed on the rollable application. :param application: juju application :param dry_run: boolean :param evacuate: boolean + :param rollable: boolean :param pause: boolean :param origin: origin string """ + label = application.name.upper() + log.info('{} - Begin rolling upgrade'.format(label)) actions = await enumerate_actions(application) - if not dry_run: + if origin and not dry_run: config_key = origin_keys.get(application.name, 'openstack-origin') - await application.set_config({config_key: origin}) + config = await application.get_config() + previous = config.get(config_key, {}).get('value', '') - ordered_units = await order_units(application.name, application.units) - hacluster_pairs = get_hacluster_subordinate_pairs(application) # TODO see fx + if previous == origin: + current = previous + else: + await application.set_config({config_key: origin}) + config_timeout = 60 + config_is_set = False + while config_timeout > 0 and not config_is_set: + config_timeout = config_timeout - 1 + await asyncio.sleep(5) + config = await application.get_config() + current = config.get(config_key, {}).get('value', '') + config_is_set = current == origin + log.info('{} - Setting config {} = {} => {}'.format(label, config_key, previous, current)) + log.info('{} - Config {} = {}'.format(label, config_key, current)) + + ordered_units = await order_units(label, application.units) if rollable else application.units + hacluster_pairs = get_hacluster_subordinate_pairs(application) if (rollable and pause) else {} # TODO see fx for unit in ordered_units: - if hacluster_pairs: - hacluster_unit = hacluster_pairs.get(unit.name, None) - else: - hacluster_unit = None + hacluster_unit = hacluster_pairs.get(unit.name, False) if evacuate and application.name == 'nova-compute': # NOT IMPLEMENTED - log.warn('Nova evacuation is not implemented and will be skipped') + log.warn('{} - Nova evacuation is not implemented, app will be skipped'.format(label)) + break if pause and hacluster_unit: # TODO this will pause all the units for hacluster subordinates - log.info('Pausing service on hacluster subordinate: {}'.format(hacluster_unit.name)) + log.info('{} - Pausing service on hacluster subordinate: {}'.format(label, hacluster_unit.name)) if not dry_run: async with async_timeout.timeout(300): action = await hacluster_unit.run_action('pause') - await application.model.wait_for_action(action.entity_id) - log.info('Service on hacluster subordinate {} is paused'.format(hacluster_unit.name)) + await action.wait() + log.debug('{} - Service action: {} on unit: {} status: {}'.format( + label, unit.name, 'pause', action.status + )) + log.info('{} - Service on hacluster subordinate {} is paused'.format(label, hacluster_unit.name)) if pause and 'pause' in actions: - log.info('Pausing service on unit: {}'.format(unit.name)) + log.info('{} - Pausing service on unit: {}'.format(label, unit.name)) if not dry_run: async with async_timeout.timeout(300): action = await unit.run_action('pause') - await application.model.wait_for_action(action.entity_id) - log.info('Service on unit {} is paused'.format(unit.name)) - - if 'openstack-upgrade' in actions: - log.info('Upgrading OpenStack for unit: {}'.format(unit.name)) + await action.wait() + log.debug('{} - Service action: {} on unit: {} status: {}'.format( + label, unit.name, 'pause', action.status + )) + log.info('{} - Service on unit {} is paused'.format(label, unit.name)) + + if upgrade_action in actions: + log.info('{} - Upgrading service for unit: {}'.format(label, unit.name)) if not dry_run: - action = await unit.run_action('openstack-upgrade') - await application.model.wait_for_action(action.entity_id) - log.info('Completed upgrade for unit: {}'.format(unit.name)) + action = await unit.run_action(upgrade_action, **upgrade_params) + await action.wait() + log.debug('{} - Service action: {} on unit: {} status: {}'.format( + label, unit.name, upgrade_action, action.status + )) + log.info('{} - Completed upgrade for unit: {}'.format(label, unit.name)) if pause and 'resume' in actions: - log.info('Resuming service on unit: {}'.format(unit.name)) + log.info('{} - Resuming service on unit: {}'.format(label, unit.name)) if not dry_run: async with async_timeout.timeout(300): action = await unit.run_action('resume') - await application.model.wait_for_action(action.entity_id) - log.info('Service on unit {} has resumed'.format(unit.name)) + await action.wait() + log.debug('{} - Service action: {} on unit: {} status: {}'.format( + label, unit.name, 'resume', action.status + )) + log.info('{} - Service on unit {} has resumed'.format(label, unit.name)) if pause and hacluster_unit: # TODO this will resume all the units for hacluster subordinates - log.info('Resuming service on hacluster subordinate: {}'.format(hacluster_unit.name)) + log.info('{} - Resuming service on hacluster subordinate: {}'.format(label, hacluster_unit.name)) if not dry_run: async with async_timeout.timeout(300): action = await hacluster_unit.run_action('resume') - await application.model.wait_for_action(action.entity_id) - log.info('Service on hacluster subordinate {} has resumed'.format(hacluster_unit.name)) - - log.info('Unit {} has finished the upgrade'.format(unit.name)) + await action.wait() + log.debug('{} - Service action: {} on unit: {} status: {}'.format( + label, unit.name, 'resume', action.status + )) + log.info('{} - Service on hacluster subordinate {} has resumed'.format(label, hacluster_unit.name)) + + await wait_until( + model, + list(model.applications.values()), + timeout=1800, + loop=model.loop + ) + log.info('{} - Unit {} has finished the upgrade'.format(label, unit.name)) + log.info('{} - Finish rolling upgrade'.format(label)) async def perform_bigbang_upgrade( @@ -483,8 +577,8 @@ async def perform_bigbang_upgrade( :param pause: boolean :param origin: origin string """ - log.info('Performing a big-bang upgrade for service: {}'.format(application.name)) - if not dry_run: + log.info('Big-bang upgrade: {}'.format(application.name)) + if origin and not dry_run: config_key = origin_keys.get(application.name, 'openstack-origin') if config_key not in await application.get_config(): log.warn('Unable to set source/origin during big-bang upgrade for service: {}'.format(application.name)) @@ -495,11 +589,12 @@ async def perform_bigbang_upgrade( await application.set_config({config_key: origin}) await asyncio.sleep(15) + if not dry_run: upgrade_in_progress = True while upgrade_in_progress: - # service = Juju.current().get_service(service.name) - # unit_uip = [u.is_upgrading() for u in service.units()] - unit_uip = [u.workload_status.lower().find('upgrad') >= 0 for u in application.units] - upgrade_in_progress = any(unit_uip) - if upgrade_in_progress: + if any( + [u.workload_status.lower().find('upgrad') >= 0 for u in application.units] + ): await asyncio.sleep(5) + else: + upgrade_in_progress = False diff --git a/requirements.txt b/requirements.txt index 3eaf1f1..6f2c18e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -pip>=10 -pyyaml<=4.2,>=3.0 -async-timeout==2.0.1 -tox>=3.3 -juju>=0.10.2 -argcomplete -theblues>=0.5.1 +pip>=19 +pyyaml<=6.0,>=5.1.2 +async-timeout<=3.99,>=2.0.1 +tox>=3.14.1 +juju<=2.6.99,>=2.6.3 +argcomplete==1.10.0 +theblues==0.5.2 diff --git a/setup.py b/setup.py index 4e064f4..e094223 100644 --- a/setup.py +++ b/setup.py @@ -35,12 +35,20 @@ "Environment :: Console" ], 'install_requires': [ - 'pyyaml<=4.2,>=3.0', - 'async-timeout==2.0.1', - 'juju>=0.10.2', - 'argcomplete', - 'theblues>=0.5.1' + 'async-timeout<=3.99,>=2.0.1', + 'argcomplete==1.10.0', + 'theblues==0.5.2' ], + 'extras_require': { + ":python_version>'3.5.2'": [ + 'juju<=2.6.99,>=0.11.7', + 'pyyaml<=6.0,>=3.0', + ], + ":python_version<='3.5.2'": [ + 'juju==0.11.7', + 'pyyaml<=4.2,>=3.0', + ] + } } diff --git a/tests/unit/asyncio_mocks.py b/tests/unit/asyncio_mocks.py new file mode 100644 index 0000000..ca7f800 --- /dev/null +++ b/tests/unit/asyncio_mocks.py @@ -0,0 +1,46 @@ +from unittest.mock import MagicMock +# from unittest.mock import AsyncMock as AsyncioMock +# from asyncio import Future +import asyncio + + +def loop(coro): + return asyncio.get_event_loop().run_until_complete(coro) + + +def AsyncMock(*args, **kwargs): + m = MagicMock(*args, **kwargs) + + async def mock_coro(*args, **kwargs): + return m(*args, **kwargs) + + mock_coro.mock = m + return mock_coro + + +def AsyncClassMethodMock(*args, **kwargs): + m = MagicMock(*args, **kwargs) + + async def mock_coro(self, *args, **kwargs): + return m(*args, **kwargs) + + mock_coro.mock = m + return mock_coro + + +def AsyncClassMock(*args, static=[], props={}): + """ + Async Class Mock. + + Provide names of mocked methods as args. + """ + class mock_class(MagicMock): + pass + + for item in args: + setattr(mock_class, item, AsyncClassMethodMock()) + for item in static: + setattr(mock_class, item, AsyncMock()) + for prop, value in props.items(): + setattr(mock_class, prop, value) + return mock_class diff --git a/tests/unit/test_args.py b/tests/unit/test_args.py index d499fca..eb7c804 100644 --- a/tests/unit/test_args.py +++ b/tests/unit/test_args.py @@ -14,23 +14,36 @@ class TestArguments(unittest.TestCase): def test_args_upgrade(self): """Testing upgrade parser.""" - args = parse_args(['upgrade', '--dry-run']) - self.assertEqual(args.action, 'upgrade') + action, timeout, args = parse_args(['upgrade', '--dry-run']) + self.assertEqual(action, 'upgrade') + self.assertEqual(timeout, 0) - self.assertFalse(args.upgrade_only) - self.assertFalse(args.charms_only) - self.assertTrue(args.dry_run) + self.assertFalse(args['upgrade_only']) + self.assertFalse(args['charms_only']) + self.assertTrue(args['dry_run']) - self.assertEqual(args.model_name, None) - self.assertEqual(args.ctrl_name, None) + self.assertEqual(args['model_name'], None) + self.assertEqual(args['ctrl_name'], None) def test_args_clean(self): """Testing clean parser.""" - args = parse_args(['clean', '--model', 'cloud', '-w']) - self.assertEqual(args.action, 'clean') + action, timeout, args = parse_args(['clean', '--model', 'cloud', '-w']) + self.assertEqual(action, 'clean') + self.assertEqual(timeout, 0) - self.assertFalse(args.dry_run) - self.assertTrue(args.wait) + self.assertFalse(args['dry_run']) + self.assertTrue(args['wait']) - self.assertEqual(args.model_name, 'cloud') - self.assertEqual(args.ctrl_name, None) + self.assertEqual(args['model_name'], 'cloud') + self.assertEqual(args['ctrl_name'], None) + + def test_kv_parse(self): + """Testing clean parser.""" + action, timeout, args = parse_args( + ['upgrade', '-t', '5', '--upgrade-params', 'version=luminuous', '--origin-keys', 'ceph=source'] + ) + self.assertEqual(action, 'upgrade') + self.assertEqual(timeout, 5) + + self.assertEqual(args['upgrade_params'], {'version': 'luminuous'}) + self.assertEqual(args['origin_keys'], {'ceph': 'source'}) diff --git a/tests/unit/test_connect.py b/tests/unit/test_connect.py new file mode 100644 index 0000000..7318b57 --- /dev/null +++ b/tests/unit/test_connect.py @@ -0,0 +1,81 @@ +""" +Tests for upgrade action. + +""" + +from unittest.mock import patch +from unittest import TestCase +from .asyncio_mocks import AsyncClassMock +from .asyncio_mocks import loop +from jujuna.helper import connect_juju + + +class TestConnect(TestCase): + """Test connection helper. + + """ + + @patch('jujuna.helper.Controller', new=AsyncClassMock('connect')) + @patch('jujuna.helper.Model', new=AsyncClassMock('connect')) + def test_remote_connection(self): + """Testing remote connection.""" + from jujuna.helper import Controller + from jujuna.helper import Model + + ctrl_name = 'test_controller' + uuid = '11188111-1111-2222-a0bf-a00777888999' + endpoint = '127.0.0.1:17070' + username = 'user' + password = 'badpassword' + cacert = 'PKcert' + loop(connect_juju( + ctrl_name, + uuid, + endpoint=endpoint, + username=username, + password=password, + cacert=cacert + )) + Controller.connect.mock.assert_called_once_with( + endpoint=endpoint, + username=username, + password=password, + cacert=cacert + ) + Model.connect.mock.assert_called_once_with( + uuid=uuid, + endpoint=endpoint, + username=username, + password=password, + cacert=cacert + ) + + @patch('jujuna.helper.Controller', new=AsyncClassMock('connect', 'get_model')) + def test_local_connection(self): + """Testing local connection.""" + from jujuna.helper import Controller + + ctrl_name = 'test_controller' + uuid = '11188111-1111-2222-a0bf-a00777888999' + loop(connect_juju( + ctrl_name, + uuid + )) + + Controller.connect.mock.assert_called_once_with(ctrl_name) + Controller.get_model.mock.assert_called_once_with(uuid) + + @patch('jujuna.helper.Controller', new=AsyncClassMock('connect')) + @patch('jujuna.helper.Model', new=AsyncClassMock('connect')) + def test_local_connection_model(self): + """Testing local connection.""" + from jujuna.helper import Controller + from jujuna.helper import Model + + ctrl_name = 'test_controller' + loop(connect_juju( + ctrl_name + )) + + Controller.connect.mock.assert_called_once_with(ctrl_name) + Model.connect.mock.assert_called_once_with() diff --git a/tests/unit/test_upgrade.py b/tests/unit/test_upgrade.py new file mode 100644 index 0000000..84ab619 --- /dev/null +++ b/tests/unit/test_upgrade.py @@ -0,0 +1,207 @@ +""" +Tests for upgrade action. + +""" + +from jujuna.upgrade import perform_upgrade +from unittest.mock import patch +from unittest import TestCase +from collections import namedtuple +from .asyncio_mocks import AsyncMock, AsyncClassMock, loop +from jujuna.upgrade import logging +logging.disable(logging.CRITICAL) + + +def unit_mock(**kwargs): + namedtuple('Unit', ['name']) + + +class TestUpgrade(TestCase): + """Test upgrade action. + + """ + + @patch('asyncio.sleep', new=AsyncMock()) + @patch('jujuna.upgrade.wait_until', new=AsyncMock()) + @patch('jujuna.upgrade.enumerate_actions', new=AsyncMock(return_value=['openstack-upgrade'])) + def test_perform_upgrade_simple( + self # , get_hacluster_subordinate_pairs + ): + """Testing upgrade parser.""" + from jujuna.upgrade import wait_until + from jujuna.upgrade import enumerate_actions + + origin_previous = 'cloud:xenial-newton' + origin = 'cloud:xenial-ocata' + upgrade_action = 'openstack-upgrade' + ORIGIN_CONFIG = [{'openstack-origin': {'value': origin}}, {'openstack-origin': {'value': origin_previous}}] + + def func(*args, **kwargs): + return ORIGIN_CONFIG.pop() + + unit = AsyncClassMock(static=['run_action'], props={'name': 'test/0'}) + app = AsyncClassMock( + static=['set_config', 'get_config'], + props={'name': 'test', 'units': [unit], 'relations': []} + ) + model = AsyncClassMock(props={'applications': {'test': app}, 'loop': None}) + + app.get_config.mock.side_effect = func + unit.run_action.mock.return_value = AsyncClassMock( + static=['wait', 'status'], + props={'results': 'results'} + ) + + loop(perform_upgrade( + model, + app, + {}, + upgrade_action, + {}, + dry_run=False, + evacuate=False, + rollable=False, + pause=True, + origin=origin + )) + app.set_config.mock.assert_called_once_with({'openstack-origin': origin}) + enumerate_actions.mock.assert_called_once_with(app) + unit.run_action.mock.assert_called_once_with(upgrade_action) + wait_until.mock.assert_called_once_with(model, list(model.applications.values()), loop=None, timeout=1800) + + @patch('asyncio.sleep', new=AsyncMock()) + @patch('jujuna.upgrade.get_hacluster_subordinate_pairs') + @patch('jujuna.upgrade.wait_until', new=AsyncMock()) + @patch('jujuna.upgrade.order_units', new=AsyncMock()) + @patch('jujuna.upgrade.enumerate_actions', new=AsyncMock(return_value=['openstack-upgrade', 'pause', 'resume'])) + def test_perform_upgrade_rolling( + self, get_hacluster_subordinate_pairs + ): + """Testing upgrade parser.""" + from jujuna.upgrade import wait_until + from jujuna.upgrade import order_units + from jujuna.upgrade import enumerate_actions + + origin_previous = 'cloud:xenial-newton' + origin = 'cloud:xenial-ocata' + upgrade_action = 'openstack-upgrade' + ORIGIN_CONFIG = [{'openstack-origin': {'value': origin}}, {'openstack-origin': {'value': origin_previous}}] + + def func(*args, **kwargs): + return ORIGIN_CONFIG.pop() + + unit = AsyncClassMock(static=['run_action'], props={'name': 'test/0'}) + app = AsyncClassMock( + static=['set_config', 'get_config'], + props={'name': 'test', 'units': [unit], 'relations': []} + ) + model = AsyncClassMock(props={'applications': {'test': app}, 'loop': None}) + + get_hacluster_subordinate_pairs.return_value = {} + app.get_config.mock.side_effect = func + order_units.mock.return_value = [unit] + unit.run_action.mock.return_value = AsyncClassMock( + static=['wait', 'status'], + props={'results': 'results'} + ) + + loop(perform_upgrade( + model, + app, + {}, + upgrade_action, + {}, + dry_run=False, + evacuate=False, + rollable=True, + pause=True, + origin=origin + )) + app.set_config.mock.assert_called_once_with({'openstack-origin': origin}) + enumerate_actions.mock.assert_called_once_with(app) + order_units.mock.assert_called_once_with(app.name.upper(), app.units) + unit.run_action.mock.assert_any_call('pause') + unit.run_action.mock.assert_any_call(upgrade_action) + unit.run_action.mock.assert_called_with('resume') + wait_until.mock.assert_called_once_with(model, list(model.applications.values()), loop=None, timeout=1800) + + @patch('asyncio.sleep', new=AsyncMock()) + @patch('jujuna.upgrade.get_hacluster_subordinate_pairs') + @patch('jujuna.upgrade.wait_until', new=AsyncMock()) + @patch('jujuna.upgrade.order_units', new=AsyncMock()) + @patch('jujuna.upgrade.enumerate_actions', new=AsyncMock(return_value=['openstack-upgrade', 'pause', 'resume'])) + def test_perform_upgrade_rolling_ha( + self, get_hacluster_subordinate_pairs + ): + """Testing upgrade parser.""" + from jujuna.upgrade import wait_until + from jujuna.upgrade import order_units + from jujuna.upgrade import enumerate_actions + + origin_previous = 'cloud:xenial-newton' + origin = 'cloud:xenial-ocata' + upgrade_action = 'openstack-upgrade' + ORIGIN_CONFIG = [{'openstack-origin': {'value': origin}}, {'openstack-origin': {'value': origin_previous}}] + + def func(*args, **kwargs): + return ORIGIN_CONFIG.pop() + + unit0 = AsyncClassMock(static=['run_action'], props={'name': 'test/0'}) + unit1 = AsyncClassMock(static=['run_action'], props={'name': 'test/1'}) + unit2 = AsyncClassMock(static=['run_action'], props={'name': 'test/2'}) + + unit0.run_action.mock.return_value = AsyncClassMock(static=['wait', 'status'], props={'results': 'results'}) + unit1.run_action.mock.return_value = AsyncClassMock(static=['wait', 'status'], props={'results': 'results'}) + unit2.run_action.mock.return_value = AsyncClassMock(static=['wait', 'status'], props={'results': 'results'}) + + unit0_ha = AsyncClassMock(static=['run_action'], props={'name': 'test-hacluster/0'}) + unit1_ha = AsyncClassMock(static=['run_action'], props={'name': 'test-hacluster/1'}) + unit2_ha = AsyncClassMock(static=['run_action'], props={'name': 'test-hacluster/2'}) + + unit0_ha.run_action.mock.return_value = AsyncClassMock(static=['wait', 'status'], props={'results': 'results'}) + unit1_ha.run_action.mock.return_value = AsyncClassMock(static=['wait', 'status'], props={'results': 'results'}) + unit2_ha.run_action.mock.return_value = AsyncClassMock(static=['wait', 'status'], props={'results': 'results'}) + + units = [unit0, unit1, unit2] + units_ha = [unit0_ha, unit1_ha, unit2_ha] + + app = AsyncClassMock( + static=['set_config', 'get_config'], + props={'name': 'test', 'units': units, 'relations': []} + ) + apps = {'test': app} + model = AsyncClassMock(props={'applications': apps, 'loop': None}) + + app.get_config.mock.side_effect = func + order_units.mock.return_value = units + get_hacluster_subordinate_pairs.return_value = { + 'test/0': unit1_ha, 'test/1': unit2_ha, 'test/2': unit0_ha, + } + + loop(perform_upgrade( + model, + app, + {}, + upgrade_action, + {}, + dry_run=False, + evacuate=False, + rollable=True, + pause=True, + origin=origin + )) + app.set_config.mock.assert_called_once_with({'openstack-origin': origin}) + enumerate_actions.mock.assert_called_once_with(app) + order_units.mock.assert_called_once_with(app.name.upper(), app.units) + get_hacluster_subordinate_pairs.assert_called_once_with(app) + + for unit in units: + unit.run_action.mock.assert_any_call('pause') + unit.run_action.mock.assert_any_call(upgrade_action) + unit.run_action.mock.assert_called_with('resume') + + for unit_ha in units_ha: + unit_ha.run_action.mock.assert_any_call('pause') + unit_ha.run_action.mock.assert_called_with('resume') + + wait_until.mock.assert_called_with(model, list(model.applications.values()), loop=None, timeout=1800) diff --git a/tox.ini b/tox.ini index ce58f61..bd47d35 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ install_requires = pytest flake8 - pip>=10 + pip>=19 name = jujuna [tox]