From ecdfe7af9b8768e4b1add33af7eaed6395d3715b Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Thu, 15 Aug 2024 15:59:11 +0200 Subject: [PATCH] live_mode: add support in the already-existing core actors Modify core actors to support upgrades with "live mode". Whereas live mode implies a new, separate, code path for generating the live image initramfs, the changes introduced in add_upgrade_boot_entry actor interfere deeply with the old implementation. Kernel cmdline arguments for the created boot entry are now manipulated uniformly, avoiding ad- hoc string formatting. It is also possible to remove kernel cmdline args from the entry. Addition of arguments precedes removal, i.e., if arg=value should be added and also removed, it will be removed. The root cmdline parameter is modified separately, due to a bug in grubby. Jira ref: RHEL-45280 (cherry picked from commit b807c273801577b85820a295e26d440ea28fbe60) --- .../actors/addupgradebootentry/actor.py | 23 +- .../libraries/addupgradebootentry.py | 243 ++++++++++++++++-- .../tests/unit_test_addupgradebootentry.py | 184 ++++++++++++- .../actors/commonleappdracutmodules/actor.py | 7 +- .../libraries/modscan.py | 9 + .../test_modscan_commonleappdracutmodules.py | 2 + .../upgradeinitramfsgenerator/actor.py | 2 + .../libraries/upgradeinitramfsgenerator.py | 169 ++++++++++-- .../unit_test_upgradeinitramfsgenerator.py | 5 +- .../actors/targetuserspacecreator/actor.py | 2 + 10 files changed, 593 insertions(+), 53 deletions(-) diff --git a/repos/system_upgrade/common/actors/addupgradebootentry/actor.py b/repos/system_upgrade/common/actors/addupgradebootentry/actor.py index 2e28971e79..f400ebf82f 100644 --- a/repos/system_upgrade/common/actors/addupgradebootentry/actor.py +++ b/repos/system_upgrade/common/actors/addupgradebootentry/actor.py @@ -3,7 +3,17 @@ from leapp.actors import Actor from leapp.exceptions import StopActorExecutionError from leapp.libraries.actor.addupgradebootentry import add_boot_entry, fix_grub_config_error -from leapp.models import BootContent, FirmwareFacts, GrubConfigError, TargetKernelCmdlineArgTasks, TransactionDryRun +from leapp.models import ( + BootContent, + FirmwareFacts, + GrubConfigError, + KernelCmdline, + LiveImagePreparationInfo, + LiveModeArtifacts, + LiveModeConfig, + TargetKernelCmdlineArgTasks, + TransactionDryRun +) from leapp.tags import InterimPreparationPhaseTag, IPUWorkflowTag @@ -15,7 +25,16 @@ class AddUpgradeBootEntry(Actor): """ name = 'add_upgrade_boot_entry' - consumes = (BootContent, GrubConfigError, FirmwareFacts, TransactionDryRun) + consumes = ( + BootContent, + GrubConfigError, + FirmwareFacts, + LiveImagePreparationInfo, + LiveModeArtifacts, + LiveModeConfig, + KernelCmdline, + TransactionDryRun + ) produces = (TargetKernelCmdlineArgTasks,) tags = (IPUWorkflowTag, InterimPreparationPhaseTag) diff --git a/repos/system_upgrade/common/actors/addupgradebootentry/libraries/addupgradebootentry.py b/repos/system_upgrade/common/actors/addupgradebootentry/libraries/addupgradebootentry.py index 3474867a91..6af3ef8d3a 100644 --- a/repos/system_upgrade/common/actors/addupgradebootentry/libraries/addupgradebootentry.py +++ b/repos/system_upgrade/common/actors/addupgradebootentry/libraries/addupgradebootentry.py @@ -1,33 +1,144 @@ +import itertools import os import re from leapp.exceptions import StopActorExecutionError -from leapp.libraries.common.config import architecture +from leapp.libraries.common.config import architecture, get_env from leapp.libraries.stdlib import api, CalledProcessError, run -from leapp.models import BootContent, KernelCmdlineArg, TargetKernelCmdlineArgTasks +from leapp.models import ( + BootContent, + KernelCmdline, + KernelCmdlineArg, + LiveImagePreparationInfo, + LiveModeArtifacts, + LiveModeConfig, + TargetKernelCmdlineArgTasks +) + + +def collect_boot_args(livemode_enabled): + args = { + 'enforcing': '0', + 'rd.plymouth': '0', + 'plymouth.enable': '0' + } + + if get_env('LEAPP_DEBUG', '0') == '1': + args['debug'] = None + + if get_env('LEAPP_DEVEL_INITRAM_NETWORK') in ('network-manager', 'scripts'): + args['ip'] = 'dhcp' + args['rd.neednet'] = '1' + + if livemode_enabled: + livemode_args = construct_cmdline_args_for_livemode() + args.update(livemode_args) + + return args + + +def collect_undesired_args(livemode_enabled): + args = {} + if livemode_enabled: + args = dict(zip(('ro', 'rhgb', 'quiet'), itertools.repeat(None))) + args['rd.lvm.lv'] = _get_rdlvm_arg_values() + + return args + + +def format_grubby_args_from_args_dict(args_dict): + """ Format the given args dictionary in a form required by grubby's --args. """ + + def fmt_single_arg(arg_pair): + key, value = arg_pair + if not value: + return str(key) + return '{key}={value}'.format(key=key, value=value) + + def flatten_arguments(arg_pair): + """ Expand multi-valued values into an iterable (key, value1), (key, value2) """ + key, value = arg_pair + if isinstance(value, (tuple, list)): + # value is multi-valued (a tuple of values) + for value_elem in value: # yield from is not available in python2.7 + yield (key, value_elem) + else: + yield (key, value) # Just a single (key, value) pair + + arg_sequence = itertools.chain(*(flatten_arguments(arg_pair) for arg_pair in args_dict.items())) + + # Sorting should be fine as only values can be None, but we cannot have a (key, None) and (key, value) in + # the dictionary at the same time. + cmdline_pieces = (fmt_single_arg(arg_pair) for arg_pair in sorted(arg_sequence)) + cmdline = ' '.join(cmdline_pieces) + + return cmdline + + +def figure_out_commands_needed_to_add_entry(kernel_path, initramfs_path, args_to_add, args_to_remove): + boot_entry_modification_commands = [] + + args_to_add_str = format_grubby_args_from_args_dict(args_to_add) + + create_entry_cmd = [ + '/usr/sbin/grubby', + '--add-kernel', '{0}'.format(kernel_path), + '--initrd', '{0}'.format(initramfs_path), + '--title', 'ELevate-Upgrade-Initramfs', + '--copy-default', + '--make-default', + '--args', args_to_add_str + ] + boot_entry_modification_commands.append(create_entry_cmd) + + # We need to update root= param separately, since we cannot do it during --add-kernel with --copy-default. + # This is likely a bug in grubby. + root_param_value = args_to_add.get('root', None) + if root_param_value: + enforce_root_param_for_the_entry_cmd = [ + '/usr/sbin/grubby', + '--update-kernel', kernel_path, + '--args', 'root={0}'.format(root_param_value) + ] + boot_entry_modification_commands.append(enforce_root_param_for_the_entry_cmd) + + if args_to_remove: + args_to_remove_str = format_grubby_args_from_args_dict(args_to_remove) + remove_undesired_args_cmd = [ + '/usr/sbin/grubby', + '--update-kernel', kernel_path, + '--remove-args', args_to_remove_str + ] + boot_entry_modification_commands.append(remove_undesired_args_cmd) + return boot_entry_modification_commands def add_boot_entry(configs=None): - debug = 'debug' if os.getenv('LEAPP_DEBUG', '0') == '1' else '' - enable_network = os.getenv('LEAPP_DEVEL_INITRAM_NETWORK') in ('network-manager', 'scripts') - ip_arg = ' ip=dhcp rd.neednet=1' if enable_network else '' kernel_dst_path, initram_dst_path = get_boot_file_paths() _remove_old_upgrade_boot_entry(kernel_dst_path, configs=configs) + + livemode_enabled = next(api.consume(LiveImagePreparationInfo), None) is not None + + cmdline_args = collect_boot_args(livemode_enabled) + undesired_cmdline_args = collect_undesired_args(livemode_enabled) + + commands_to_run = figure_out_commands_needed_to_add_entry(kernel_dst_path, + initram_dst_path, + args_to_add=cmdline_args, + args_to_remove=undesired_cmdline_args) + + def run_commands_adding_entry(extra_command_suffix=None): + if not extra_command_suffix: + extra_command_suffix = [] + for command in commands_to_run: + run(command + extra_command_suffix) + try: - cmd = [ - '/usr/sbin/grubby', - '--add-kernel', '{0}'.format(kernel_dst_path), - '--initrd', '{0}'.format(initram_dst_path), - '--title', 'ELevate-Upgrade-Initramfs', - '--copy-default', - '--make-default', - '--args', '{DEBUG}{NET} enforcing=0 rd.plymouth=0 plymouth.enable=0'.format(DEBUG=debug, NET=ip_arg) - ] if configs: for config in configs: - run(cmd + ['-c', config]) + run_commands_adding_entry(extra_command_suffix=['-c', config]) else: - run(cmd) + run_commands_adding_entry(extra_command_suffix=None) if architecture.matches_architecture(architecture.ARCH_S390X): # on s390x we need to call zipl explicitly because of issue in grubby, @@ -35,7 +146,7 @@ def add_boot_entry(configs=None): # See https://bugzilla.redhat.com/show_bug.cgi?id=1764306 run(['/usr/sbin/zipl']) - if debug: + if 'debug' in cmdline_args: # The kernelopts for target kernel are generated based on the cmdline used in the upgrade initramfs, # therefore, if we enabled debug above, and the original system did not have the debug kernelopt, we # need to explicitly remove it from the target os boot entry. @@ -114,3 +225,101 @@ def fix_grub_config_error(conf_file, error_type): elif error_type == 'missing newline': write_to_file(conf_file, config + '\n') + + +def local_os_stat(path): + """ Local wrapper around os.stat so we can safely mock it in tests. """ + return os.stat(path) + + +def _get_device_uuid(path): + """ + Find the UUID of a device in which the given path is located. + """ + while not os.path.ismount(path): + path = os.path.dirname(path) + + needle_dev_id = local_os_stat(path).st_dev + + for uuid in os.listdir('/dev/disk/by-uuid'): + uuid_fullpath = os.path.join('/dev/disk/by-uuid/', uuid) + dev_path = os.readlink(uuid_fullpath) + + # The link target is likely relative to the UUID_fullpath, e.g., ../../dm-1. + # Joining it will '/dev/disk/by-uuid' will resolve the relative path. + # If dev_path is absolute it returns dev_path. + dev_path = os.path.join('/dev/disk/by-uuid', dev_path) + dev_path = os.path.abspath(dev_path) + + dev_id = local_os_stat(dev_path).st_rdev + if dev_id == needle_dev_id: + return uuid + + return None + + +def _get_rdlvm_arg_values(): + # should we not check args returned by grubby instead? + cmdline_msg = next(api.consume(KernelCmdline), None) + + if not cmdline_msg: + raise StopActorExecutionError('Did not receive any KernelCmdline arguments.') + + rd_lvm_values = sorted(arg.value for arg in cmdline_msg.parameters if arg.key == 'rd.lvm.lv') + api.current_logger().debug('Collected the following rd.lvm.lv args that are undesired for the squashfs: %s', + rd_lvm_values) + + return rd_lvm_values + + +def construct_cmdline_args_for_livemode(): + """ + Prepare cmdline parameters for the live mode + """ + # boot locally by default + + livemode_config = next(api.consume(LiveModeConfig), None) + if not livemode_config: + raise StopActorExecutionError('Did not receive any livemode configuration message although it is enabled.') + + livemode_artifacts = next(api.consume(LiveModeArtifacts), None) + if not livemode_artifacts: + raise StopActorExecutionError('Did not receive any livemode artifacts message although it is enabled.') + + liveimg_filename = os.path.basename(livemode_artifacts.squashfs_path) + dir_path_containing_liveimg = os.path.dirname(livemode_artifacts.squashfs_path) + + args = {'rw': None} + + # if an URL is defined, boot over the network (http, nfs, ftp, ...) + if livemode_config.url_to_load_squashfs_from: + args['root'] = 'live:{}'.format(livemode_config.url_to_load_squashfs_from) + else: + args['root'] = 'live:UUID={}'.format(_get_device_uuid(dir_path_containing_liveimg)) + args['rd.live.dir'] = dir_path_containing_liveimg + args['rd.live.squashimg'] = liveimg_filename + + if livemode_config.dracut_network: + network_fragments = livemode_config.dracut_network.split('=', 1) + + # @Todo(mhecko): verify this during config scan + if len(network_fragments) == 1 or network_fragments[0] != 'ip': + msg = ('The livemode dracut_network configuration value is incorrect - it does not ' + 'have the form of a key=value cmdline arg: `{0}`.') + msg = msg.format(livemode_config.dracut_network) + + api.current_logger().error(msg) + raise StopActorExecutionError('Livemode is not configured correctly.', details={'details': msg}) + + args['ip'] = network_fragments[1] + args['rd.needsnet'] = '1' + + autostart_state = '1' if livemode_config.autostart_upgrade_after_reboot else '0' + args['upgrade.autostart'] = autostart_state + + if livemode_config.capture_upgrade_strace_into: + args['upgrade.strace'] = livemode_config.capture_upgrade_strace_into + + api.current_logger().info('The use of live mode image implies the following cmdline args: %s', args) + + return args diff --git a/repos/system_upgrade/common/actors/addupgradebootentry/tests/unit_test_addupgradebootentry.py b/repos/system_upgrade/common/actors/addupgradebootentry/tests/unit_test_addupgradebootentry.py index a9c406918d..b6d2e235bf 100644 --- a/repos/system_upgrade/common/actors/addupgradebootentry/tests/unit_test_addupgradebootentry.py +++ b/repos/system_upgrade/common/actors/addupgradebootentry/tests/unit_test_addupgradebootentry.py @@ -8,7 +8,14 @@ from leapp.libraries.common.config.architecture import ARCH_S390X, ARCH_X86_64 from leapp.libraries.common.testutils import CurrentActorMocked, produce_mocked from leapp.libraries.stdlib import api -from leapp.models import BootContent, KernelCmdlineArg, TargetKernelCmdlineArgTasks +from leapp.models import ( + BootContent, + KernelCmdline, + KernelCmdlineArg, + LiveModeArtifacts, + LiveModeConfig, + TargetKernelCmdlineArgTasks +) CUR_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -46,7 +53,7 @@ def __call__(self, filename, content): '--copy-default', '--make-default', '--args', - 'debug enforcing=0 rd.plymouth=0 plymouth.enable=0' + 'debug enforcing=0 plymouth.enable=0 rd.plymouth=0' ] run_args_zipl = ['/usr/sbin/zipl'] @@ -66,9 +73,8 @@ def get_boot_file_paths_mocked(): monkeypatch.setattr(addupgradebootentry, 'get_boot_file_paths', get_boot_file_paths_mocked) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setenv('LEAPP_DEBUG', '1') monkeypatch.setattr(addupgradebootentry, 'run', run_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(arch)) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(arch, envars={'LEAPP_DEBUG': '1'})) addupgradebootentry.add_boot_entry() @@ -91,10 +97,10 @@ def get_boot_file_paths_mocked(): monkeypatch.setattr(addupgradebootentry, 'get_boot_file_paths', get_boot_file_paths_mocked) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setenv('LEAPP_DEBUG', '1' if is_leapp_invoked_with_debug else '0') monkeypatch.setattr(addupgradebootentry, 'run', run_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + monkeypatch.setattr(api, 'current_actor', + CurrentActorMocked(envars={'LEAPP_DEBUG': str(int(is_leapp_invoked_with_debug))})) addupgradebootentry.add_boot_entry() @@ -114,9 +120,8 @@ def get_boot_file_paths_mocked(): return '/abc', '/def' monkeypatch.setattr(addupgradebootentry, 'get_boot_file_paths', get_boot_file_paths_mocked) - monkeypatch.setenv('LEAPP_DEBUG', '1') monkeypatch.setattr(addupgradebootentry, 'run', run_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(envars={'LEAPP_DEBUG': '1'})) monkeypatch.setattr(api, 'produce', produce_mocked()) addupgradebootentry.add_boot_entry(CONFIGS) @@ -128,7 +133,7 @@ def get_boot_file_paths_mocked(): assert addupgradebootentry.run.args[3] == run_args_add + ['-c', CONFIGS[1]] assert api.produce.model_instances == [ TargetKernelCmdlineArgTasks(to_remove=[KernelCmdlineArg(key='debug')]), - TargetKernelCmdlineArgTasks(to_remove=[KernelCmdlineArg(key='enforcing', value='0')]) + TargetKernelCmdlineArgTasks(to_remove=[KernelCmdlineArg(key='enforcing', value='0')]), ] @@ -167,3 +172,164 @@ def test_fix_grub_config_error(monkeypatch, error_type, test_file_name): with open(os.path.join(CUR_DIR, 'files/{}.fixed'.format(test_file_name))) as f: assert addupgradebootentry.write_to_file.content == f.read() + + +@pytest.mark.parametrize( + ('is_debug_enabled', 'network_enablement_type'), + ( + (True, 'network-manager'), + (True, 'scripts'), + (True, False), + (False, False), + ) +) +def test_collect_boot_args(monkeypatch, is_debug_enabled, network_enablement_type): + env_vars = {'LEAPP_DEBUG': str(int(is_debug_enabled))} + if network_enablement_type: + env_vars['LEAPP_DEVEL_INITRAM_NETWORK'] = network_enablement_type + + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(envars=env_vars)) + monkeypatch.setattr(addupgradebootentry, 'construct_cmdline_args_for_livemode', + lambda *args: {'livemodearg': 'value'}) + + args = addupgradebootentry.collect_boot_args(livemode_enabled=True) + + assert args['enforcing'] == '0' + assert args['rd.plymouth'] == '0' + assert args['plymouth.enable'] == '0' + assert args['livemodearg'] == 'value' + + if is_debug_enabled: + assert args['debug'] is None + + if network_enablement_type: + assert args['ip'] == 'dhcp' + assert args['rd.neednet'] == '1' + + +@pytest.mark.parametrize( + 'livemode_config', + ( + LiveModeConfig(is_enabled=True, + squashfs_fullpath='/dir/squashfs.img', + url_to_load_squashfs_from='my-url'), + LiveModeConfig(is_enabled=True, + squashfs_fullpath='/dir/squashfs.img', + dracut_network="ip=192.168.122.146::192.168.122.1:255.255.255.0:foo::none"), + LiveModeConfig(is_enabled=True, + squashfs_fullpath='/dir/squashfs.img', + autostart_upgrade_after_reboot=False), + LiveModeConfig(is_enabled=True, + squashfs_fullpath='/dir/squashfs.img', + autostart_upgrade_after_reboot=True), + LiveModeConfig(is_enabled=True, + squashfs_fullpath='/dir/squashfs.img', + capture_upgrade_strace_into='/var/strace.out'), + ), +) +def test_construct_cmdline_for_livemode(monkeypatch, livemode_config): + artifacts = LiveModeArtifacts(squashfs_path=livemode_config.squashfs_fullpath) + messages = [livemode_config, artifacts] + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=messages)) + + monkeypatch.setattr(addupgradebootentry, '_get_device_uuid', lambda *args: 'MY_UUID') + + args = addupgradebootentry.construct_cmdline_args_for_livemode() + + assert 'rw' in args + + if livemode_config.url_to_load_squashfs_from: + assert args['root'] == 'live:my-url' + else: + assert args['root'] == 'live:UUID=MY_UUID' + assert args['rd.live.dir'] == '/dir' + assert args['rd.live.squashimg'] == 'squashfs.img' + + if livemode_config.dracut_network: + assert args['ip'] == '192.168.122.146::192.168.122.1:255.255.255.0:foo::none' + assert args['rd.needsnet'] == '1' + + assert args['upgrade.autostart'] == str(int(livemode_config.autostart_upgrade_after_reboot)) + + if livemode_config.capture_upgrade_strace_into: + assert args['upgrade.strace'] == livemode_config.capture_upgrade_strace_into + + +def test_get_rdlvm_arg_values(monkeypatch): + cmdline = [ + KernelCmdlineArg(key='debug', value=None), + KernelCmdlineArg(key='rd.lvm.lv', value='A'), + KernelCmdlineArg(key='other', value='A'), + KernelCmdlineArg(key='rd.lvm.lv', value='B') + ] + messages = [KernelCmdline(parameters=cmdline)] + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=messages)) + + args = addupgradebootentry._get_rdlvm_arg_values() + + assert args == ['A', 'B'] + + +def test_get_device_uuid(monkeypatch): + """ + The file in question is /var/lib/file + Underlying partition /var is a device /dev/sda1 (dev_id=10) linked to from /dev/disk/by-uuid/MY_UUID1 + """ + + execution_stats = { + 'is_mount_call_count': 0 + } + + def is_mount_mock(path): + execution_stats['is_mount_call_count'] += 1 + assert execution_stats['is_mount_call_count'] <= 3 + return path == '/var' + + monkeypatch.setattr(os.path, 'ismount', is_mount_mock) + + StatResult = namedtuple('StatResult', ('st_dev', 'st_rdev')) + + def stat_mock(path): + known_paths_table = { + '/var': StatResult(st_dev=1, st_rdev=None), + '/dev/sda1': StatResult(st_dev=0, st_rdev=1), + '/dev/sda2': StatResult(st_dev=0, st_rdev=2), + '/dev/vda0': StatResult(st_dev=0, st_rdev=3), + } + return known_paths_table[path] + + monkeypatch.setattr(addupgradebootentry, 'local_os_stat', stat_mock) + + def listdir_mock(path): + assert path == '/dev/disk/by-uuid' + return ['MY_UUID0', 'MY_UUID1', 'MY_UUID2'] + + monkeypatch.setattr(os, 'listdir', listdir_mock) + + def readlink_mock(path): + known_links = { + '/dev/disk/by-uuid/MY_UUID0': '/dev/vda0', + '/dev/disk/by-uuid/MY_UUID1': '../../sda1', + '/dev/disk/by-uuid/MY_UUID2': '../../sda2', + } + return known_links[path] + + monkeypatch.setattr(os, 'readlink', readlink_mock) + + path = '/var/lib/file' + uuid = addupgradebootentry._get_device_uuid(path) + + assert uuid == 'MY_UUID1' + + +@pytest.mark.parametrize( + ('args', 'expected_result'), + ( + ([('argA', 'val'), ('argB', 'valB'), ('argC', None), ], 'argA=val argB=valB argC'), + ([('argA', ('val1', 'val2'))], 'argA=val1 argA=val2') + ) +) +def test_format_grubby_args_from_args_dict(args, expected_result): + actual_result = addupgradebootentry.format_grubby_args_from_args_dict(dict(args)) + + assert actual_result == expected_result diff --git a/repos/system_upgrade/common/actors/commonleappdracutmodules/actor.py b/repos/system_upgrade/common/actors/commonleappdracutmodules/actor.py index aae42bbb21..6be0657f6d 100644 --- a/repos/system_upgrade/common/actors/commonleappdracutmodules/actor.py +++ b/repos/system_upgrade/common/actors/commonleappdracutmodules/actor.py @@ -1,9 +1,10 @@ from leapp.actors import Actor from leapp.libraries.actor import modscan -from leapp.tags import FactsPhaseTag, IPUWorkflowTag +from leapp.tags import InterimPreparationPhaseTag, IPUWorkflowTag from leapp.utils.deprecation import suppress_deprecation from leapp.models import ( # isort:skip + LiveModeConfig, RequiredUpgradeInitramPackages, # deprecated UpgradeDracutModule, # deprecated TargetUserSpaceUpgradeTasks, @@ -23,14 +24,14 @@ class CommonLeappDracutModules(Actor): """ name = 'common_leapp_dracut_modules' - consumes = () + consumes = (LiveModeConfig,) produces = ( RequiredUpgradeInitramPackages, # deprecated TargetUserSpaceUpgradeTasks, UpgradeDracutModule, # deprecated UpgradeInitramfsTasks, ) - tags = (IPUWorkflowTag, FactsPhaseTag) + tags = (IPUWorkflowTag, InterimPreparationPhaseTag) def process(self): modscan.process() diff --git a/repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py b/repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py index 15150a5038..43cf60e66f 100644 --- a/repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py +++ b/repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py @@ -7,6 +7,7 @@ from leapp.models import ( # isort:skip CopyFile, + LiveModeConfig, RequiredUpgradeInitramPackages, # deprecated UpgradeDracutModule, # deprecated DracutModule, @@ -61,6 +62,14 @@ def _create_initram_networking_tasks(): # @suppress_deprecation(UpgradeDracutModule) def _create_dracut_modules(): dracut_base_path = api.get_actor_folder_path('dracut') + + livemode_config = next(api.consume(LiveModeConfig), None) + if livemode_config and livemode_config.is_enabled: + msg = ('Skipping requesting leapp\'s dracut modules to be included into ' + 'upgrade initramfs due to livemode being enabled.') + api.current_logger().debug(msg) + return + if dracut_base_path: dracut_base_path = os.path.abspath(dracut_base_path) for module in os.listdir(dracut_base_path): diff --git a/repos/system_upgrade/common/actors/commonleappdracutmodules/tests/test_modscan_commonleappdracutmodules.py b/repos/system_upgrade/common/actors/commonleappdracutmodules/tests/test_modscan_commonleappdracutmodules.py index 9c52b51f3e..781d0b4845 100644 --- a/repos/system_upgrade/common/actors/commonleappdracutmodules/tests/test_modscan_commonleappdracutmodules.py +++ b/repos/system_upgrade/common/actors/commonleappdracutmodules/tests/test_modscan_commonleappdracutmodules.py @@ -26,6 +26,8 @@ def _files_get_folder_path(name): @suppress_deprecation(UpgradeDracutModule) def test_created_modules(monkeypatch): + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + monkeypatch.setattr(api, 'get_actor_folder_path', _files_get_folder_path) path = os.path.abspath(api.get_actor_folder_path('dracut')) required_modules = {'sys-upgrade', 'sys-upgrade-redhat'} diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/actor.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/actor.py index 2c52e817a0..d99bab4823 100644 --- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/actor.py +++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/actor.py @@ -5,6 +5,7 @@ from leapp.models import ( BootContent, FIPSInfo, + LiveModeConfig, TargetOSInstallationImage, TargetUserSpaceInfo, TargetUserSpaceUpgradeTasks, @@ -29,6 +30,7 @@ class UpgradeInitramfsGenerator(Actor): name = 'upgrade_initramfs_generator' consumes = ( FIPSInfo, + LiveModeConfig, RequiredUpgradeInitramPackages, # deprecated TargetOSInstallationImage, TargetUserSpaceInfo, diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py index 04fa061b00..02c3fd9d4a 100644 --- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py +++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py @@ -1,5 +1,7 @@ +import itertools import os import shutil +from collections import namedtuple from leapp.exceptions import StopActorExecutionError from leapp.libraries.common import dnfplugin, mounting @@ -9,6 +11,7 @@ from leapp.models import UpgradeDracutModule # deprecated from leapp.models import ( BootContent, + LiveModeConfig, TargetOSInstallationImage, TargetUserSpaceInfo, TargetUserSpaceUpgradeTasks, @@ -314,6 +317,30 @@ def _check_free_space(context): ) +InitramfsIncludes = namedtuple('InitramfsIncludes', ('files', 'dracut_modules', 'kernel_modules')) + + +def collect_initramfs_includes(): + """ + Collect modules and files requested to be included in initramfs. + + :returns: A summary of requested initramfs includes + :rtype: InitramfsIncludes + """ + dracut_modules = _get_dracut_modules() # Use lists as leapp's models are not hashable + kernel_modules = list() + additional_initramfs_files = set() + + for task in api.consume(UpgradeInitramfsTasks): + dracut_modules.extend(task.include_dracut_modules) + kernel_modules.extend(task.include_kernel_modules) + additional_initramfs_files.update(task.include_files) + + return InitramfsIncludes(files=list(additional_initramfs_files), + dracut_modules=list(dracut_modules), + kernel_modules=list(kernel_modules)) + + def generate_initram_disk(context): """ Function to actually execute the init ramdisk creation. @@ -328,18 +355,13 @@ def generate_initram_disk(context): # TODO(pstodulk): Add possibility to add particular drivers # Issue #645 - modules = { - 'dracut': _get_dracut_modules(), # deprecated - 'kernel': [], - } - files = set() - for task in api.consume(UpgradeInitramfsTasks): - modules['dracut'].extend(task.include_dracut_modules) - modules['kernel'].extend(task.include_kernel_modules) - files.update(task.include_files) + initramfs_includes = collect_initramfs_includes() + + copy_dracut_modules(context, initramfs_includes.dracut_modules) + copy_kernel_modules(context, initramfs_includes.kernel_modules) - copy_dracut_modules(context, modules['dracut']) - copy_kernel_modules(context, modules['kernel']) + def fmt_module_list(module_list): + return ','.join(mod.name for mod in module_list) # FIXME: issue #376 context.call([ @@ -349,14 +371,115 @@ def generate_initram_disk(context): 'LEAPP_ADD_KERNEL_MODULES="{kernel_modules}" ' 'LEAPP_DRACUT_INSTALL_FILES="{files}" {cmd}'.format( kernel_version=_get_target_kernel_version(context), - dracut_modules=','.join([mod.name for mod in modules['dracut']]), - kernel_modules=','.join([mod.name for mod in modules['kernel']]), + dracut_modules=fmt_module_list(initramfs_includes.dracut_modules), + kernel_modules=fmt_module_list(initramfs_includes.kernel_modules), arch=api.current_actor().configuration.architecture, - files=' '.join(files), + files=' '.join(initramfs_includes.files), cmd=os.path.join('/', INITRAM_GEN_SCRIPT_NAME)) ], env=env) - copy_boot_files(context) + boot_files_info = copy_boot_files(context) + return boot_files_info + + +def get_boot_artifact_names(): + """ + Get the name of leapp's initramfs and upgrade kernel. + + :returns: A tuple (kernel_name, initramfs_name). + :rtype: Tuple[str, str] + """ + + arch = api.current_actor().configuration.architecture + + kernel = 'vmlinuz-upgrade.{}'.format(arch) + initramfs = 'initramfs-upgrade.{}.img'.format(arch) + + return (kernel, initramfs) + + +def copy_target_kernel_from_userspace_into_boot(context, target_kernel_ver, kernel_artifact_name): + userspace_kernel_installation_path = '/lib/modules/{}/vmlinuz'.format(target_kernel_ver) + api.current_logger().info( + 'Copying target kernel ({0}) into host system\'s /boot'.format(userspace_kernel_installation_path) + ) + host_kernel_dest = os.path.join('/boot', kernel_artifact_name) + context.copy_from(userspace_kernel_installation_path, host_kernel_dest) + + +def _generate_livemode_initramfs(context, userspace_initramfs_dest, target_kernel_ver): + """ + Generate livemode initramfs + + Collects modifications requested by received messages, synthesize and executed corresponding + dracut command. The created initramfs is placed at USERSPACE_ARTIFACTS_PATH/ + in the userspace. + + :param userspace_initramfs_dest str: The path at which the generated initramfs will be placed. + :param target_kernel_ver str: Kernel version installed into the userspace that will be used by the live image. + :returns: None + """ + env = {} + if get_target_major_version() == '9': + env = {'SYSTEMD_SECCOMP': '0'} + + initramfs_includes = collect_initramfs_includes() + + copy_dracut_modules(context, initramfs_includes.dracut_modules) + copy_kernel_modules(context, initramfs_includes.kernel_modules) + + dracut_modules = ['livenet', 'dmsquash-live'] + [mod.name for mod in initramfs_includes.dracut_modules] + + cmd = ['dracut', '--verbose', '--compress', 'xz', + '--no-hostonly', '--no-hostonly-default-device', + '-o', 'plymouth dash resume ifcfg earlykdump', + '--lvmconf', '--mdadmconf', + '--kver', target_kernel_ver, '-f', userspace_initramfs_dest] + + # Add dracut modules + cmd.extend(itertools.chain(*(('--add', module) for module in dracut_modules))) + + # Add kernel modules + cmd.extend(itertools.chain(*(('--add-drivers', module.name) for module in initramfs_includes.kernel_modules))) + + try: + context.call(cmd, env=env) + except CalledProcessError as error: + api.current_logger().error('Failed to generate (live) upgrade image. Error: %s', error) + raise StopActorExecutionError( + 'Cannot generate the initramfs for the live mode.', + details={'Problem': 'the dracut command failed: {}'.format(cmd)}) + + +def prepare_boot_files_for_livemode(context): + """ + Generate the initramfs for the live mode using dracut modules: dracut-live dracut-squash. + Silently replace upgrade boot images. + """ + api.current_logger().info('Building initramfs for the live upgrade image.') + + # @Todo(mhecko): See whether we need to do permission manipulation from dracut_install_modules. + # @Todo(mhecko): We need to handle upgrade kernel HMAC if we ever want to boot with FIPS in livemode + + target_kernel_ver = _get_target_kernel_version(context) + kernel_artifact_name, initramfs_artifact_name = get_boot_artifact_names() + + copy_target_kernel_from_userspace_into_boot(context, target_kernel_ver, kernel_artifact_name) + + USERSPACE_ARTIFACTS_PATH = '/artifacts' + context.makedirs(USERSPACE_ARTIFACTS_PATH, exists_ok=True) + userspace_initramfs_dest = os.path.join(USERSPACE_ARTIFACTS_PATH, initramfs_artifact_name) + + _generate_livemode_initramfs(context, userspace_initramfs_dest, target_kernel_ver) + + api.current_logger().debug('Copying artifacts from userspace into host\'s /boot') + host_initramfs_dest = os.path.join('/boot', initramfs_artifact_name) + host_kernel_dest = os.path.join('/boot', kernel_artifact_name) + context.copy_from(userspace_initramfs_dest, host_initramfs_dest) + + return BootContent(kernel_path=host_kernel_dest, + initram_path=host_initramfs_dest, + kernel_hmac_path='') def create_upgrade_hmac_from_target_hmac(original_hmac_path, upgrade_hmac_path, upgrade_kernel): @@ -379,8 +502,10 @@ def create_upgrade_hmac_from_target_hmac(original_hmac_path, upgrade_hmac_path, def copy_boot_files(context): """ - Function to copy the generated initram and corresponding kernel to /boot - Additionally produces a BootContent - message with their location. + Function to copy the generated initram and corresponding kernel to /boot + + + :returns: BootContent message containing the information about where the artifacts can be found. """ curr_arch = api.current_actor().configuration.architecture kernel = 'vmlinuz-upgrade.{}'.format(curr_arch) @@ -401,13 +526,19 @@ def copy_boot_files(context): kernel_hmac_path = context.full_path(os.path.join('/artifacts', kernel_hmac)) create_upgrade_hmac_from_target_hmac(kernel_hmac_path, content.kernel_hmac_path, kernel) - api.produce(content) + return content def process(): userspace_info = next(api.consume(TargetUserSpaceInfo), None) target_iso = next(api.consume(TargetOSInstallationImage), None) + livemode_config = next(api.consume(LiveModeConfig), None) + with mounting.NspawnActions(base_dir=userspace_info.path) as context: with mounting.mount_upgrade_iso_to_root_dir(userspace_info.path, target_iso): prepare_userspace_for_initram(context) - generate_initram_disk(context) + if livemode_config and livemode_config.is_enabled: + boot_file_info = prepare_boot_files_for_livemode(context) + else: + boot_file_info = generate_initram_disk(context) + api.produce(boot_file_info) diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py index 7397b82b93..8408233e0d 100644 --- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py +++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py @@ -166,13 +166,12 @@ def create_upgrade_hmac_from_target_hmac_mock(original_hmac_path, upgrade_hmac_p 'create_upgrade_hmac_from_target_hmac', create_upgrade_hmac_from_target_hmac_mock) - upgradeinitramfsgenerator.copy_boot_files(context) + actual_boot_content = upgradeinitramfsgenerator.copy_boot_files(context) assert len(context.called_copy_from) == 2 assert (os.path.join('/artifacts', kernel), bootc.kernel_path) in context.called_copy_from assert (os.path.join('/artifacts', initram), bootc.initram_path) in context.called_copy_from - assert upgradeinitramfsgenerator.api.produce.called == 1 - assert upgradeinitramfsgenerator.api.produce.model_instances[0] == bootc + assert actual_boot_content == bootc class MockedCopyArgs(object): diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/actor.py b/repos/system_upgrade/common/actors/targetuserspacecreator/actor.py index b1225230ef..ddfa7ada0c 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/actor.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/actor.py @@ -5,6 +5,7 @@ from leapp.models import TMPTargetRepositoriesFacts # deprecated from leapp.models import ( CustomTargetRepositoryFile, + LiveModeConfig, PkgManagerInfo, Report, RepositoriesFacts, @@ -37,6 +38,7 @@ class TargetUserspaceCreator(Actor): name = 'target_userspace_creator' consumes = ( CustomTargetRepositoryFile, + LiveModeConfig, RHSMInfo, RHUIInfo, RepositoriesFacts,