From b25fd04aac43aeb38ba66c13e9e207bc8454cce3 Mon Sep 17 00:00:00 2001 From: Travis Holloway Date: Mon, 1 Apr 2024 12:50:37 -0500 Subject: [PATCH] Convert CCS blocker to component Case RE-78: This change converts the CCS blocker to a component. * Remove the CCS blocker * Export the CCS data to a secure directory prior to starting leapp * Remove the CCS package prior to starting leapp * Remove some cruft that the CCS package leaves behind prior to leapp * Install the CCS package after leapp completes * Ensure that the CCS service is running * Import the CCS data after leapp completes Changelog: Convert CCS blocker to component --- elevate-cpanel | 425 +++++++++++++++++++++++++++++-- lib/Elevate/Blockers/WHM.pm | 13 - lib/Elevate/Components/CCS.pm | 461 ++++++++++++++++++++++++++++++++++ script/elevate-cpanel.PL | 12 +- t/blocker-whm.t | 20 -- t/components-CCS.t | 156 ++++++++++++ 6 files changed, 1037 insertions(+), 50 deletions(-) create mode 100644 lib/Elevate/Components/CCS.pm create mode 100644 t/components-CCS.t diff --git a/elevate-cpanel b/elevate-cpanel index 700cc68f..7fecb8ba 100755 --- a/elevate-cpanel +++ b/elevate-cpanel @@ -36,6 +36,7 @@ BEGIN { # Suppress load of all of these at earliest point. $INC{'Elevate/Blockers/AutoSSL.pm'} = 'script/elevate-cpanel.PL.static'; $INC{'Elevate/Components/Base.pm'} = 'script/elevate-cpanel.PL.static'; $INC{'Elevate/Components/AbsoluteSymlinks.pm'} = 'script/elevate-cpanel.PL.static'; + $INC{'Elevate/Components/CCS.pm'} = 'script/elevate-cpanel.PL.static'; $INC{'Elevate/Components/cPanelPlugins.pm'} = 'script/elevate-cpanel.PL.static'; $INC{'Elevate/Components/cPanelPrep.pm'} = 'script/elevate-cpanel.PL.static'; $INC{'Elevate/Components/EA4.pm'} = 'script/elevate-cpanel.PL.static'; @@ -2042,7 +2043,6 @@ EOS use Cpanel::Version::Tiny (); use Cpanel::Update::Tiers (); use Cpanel::License (); - use Cpanel::Pkgr (); use Cpanel::Unix::PID::Tiny (); # use Elevate::Blockers::Base(); @@ -2068,7 +2068,6 @@ EOS $ok = 0 unless $self->_blocker_is_sandbox; $ok = 0 unless $self->_blocker_is_upcp_running; $ok = 0 unless $self->_blocker_is_cpanel_backup_running; - $ok = 0 unless $self->_blocker_is_calendar_installed; return $ok; } @@ -2165,17 +2164,6 @@ EOS return 0; } - sub _blocker_is_calendar_installed ($self) { - if ( Cpanel::Pkgr::is_installed('cpanel-ccs-calendarserver') ) { - return $self->has_blocker( <<~'EOS'); - You have the cPanel Calendar Server installed. Upgrades with this server in place are not supported. - Removal of this server can lead to data loss. - EOS - } - - return 0; - } - sub _blocker_is_upcp_running ($self) { return 0 unless $self->getopt('start'); @@ -2469,6 +2457,405 @@ EOS } # --- END lib/Elevate/Components/AbsoluteSymlinks.pm +{ # --- BEGIN lib/Elevate/Components/CCS.pm + + package Elevate::Components::CCS; + + use cPstrict; + + use Try::Tiny; + + use File::Path (); + use File::Copy (); + + use Cpanel::Autodie (); + use Cpanel::Config::Users (); + use Cpanel::JSON (); + use Cpanel::Pkgr (); + + use Elevate::Notify (); + use Elevate::StageFile (); + + # use Log::Log4perl qw(:easy); + INIT { Log::Log4perl->import(qw{:easy}); } + + # use Elevate::Components::Base(); + our @ISA; + BEGIN { push @ISA, qw(Elevate::Components::Base); } + + use constant CCS_PACKAGE => 'cpanel-ccs-calendarserver'; + use constant ZPUSH_PACKAGE => 'cpanel-z-push'; + use constant EXPORT_DIR => '/var/cpanel/elevate_ccs_export'; + use constant CCS_RESTART_SCRIPT => '/usr/local/cpanel/scripts/restartsrv_cpanel_ccs'; + use constant TASK_QUEUE_SCRIPT => '/usr/local/cpanel/bin/servers_queue'; + + use constant DUMP_TYPES => ( + calendars => 'ics', + contacts => 'vcard', + ); + + sub pre_leapp ($self) { + my $ccs_installed = Cpanel::Pkgr::is_installed(CCS_PACKAGE); + Elevate::StageFile::update_stage_file( { ccs_installed => $ccs_installed } ); + return unless $ccs_installed; + + $self->_load_ccs_modules(); + + $self->run_once('export_ccs_data'); + $self->remove_ccs_and_dependencies(); + + $self->clean_up_pkg_cruft(); + + return; + } + + sub clean_up_pkg_cruft ($self) { + $self->move_pgsql_directory(); + $self->remove_cpanel_ccs_home_directory(); + return; + } + + sub remove_cpanel_ccs_home_directory ($self) { + File::Path::remove_tree('/opt/cpanel-ccs') if -d '/opt/cpanel-ccs'; + return; + } + + sub move_pgsql_directory ($self) { + my $pg_dir = '/var/lib/pgsql'; + my $pg_backup_dir = '/var/lib/pgsql_pre_elevate'; + + File::Path::remove_tree($pg_backup_dir) if -e $pg_backup_dir && -d $pg_backup_dir; + + $pg_backup_dir .= '_' . time() . '_' . $$ if -e $pg_backup_dir; + + File::Path::remove_tree($pg_backup_dir) if -e $pg_backup_dir && -d $pg_backup_dir; + + if ( -e $pg_backup_dir ) { + die <<~"EOS"; + Unable to ensure a valid backup path for $pg_dir. + Please ensure that '/var/lib/pgsql_pre_elevate' does not exist on your system and execute this script again with + + /scripts/elevate-cpanel --continue + + EOS + } + + INFO( <<~"EOS" ); + Moving the PostgreSQL data dir located at $pg_dir to $pg_backup_dir + to ensure a functioning PostgreSQL server after the elevation completes. + EOS + + File::Copy::move( $pg_dir, $pg_backup_dir ) if -d $pg_dir; + + return; + } + + sub remove_ccs_and_dependencies ($self) { + + my $zpush_installed = Cpanel::Pkgr::is_installed(ZPUSH_PACKAGE); + Elevate::StageFile::update_stage_file( { zpush_installed => $zpush_installed } ); + + my @ccs_dependencies = qw{ + postgresql + postgresql-devel + postgresql-server + }; + + push @ccs_dependencies, ZPUSH_PACKAGE(); + + $self->yum->remove( CCS_PACKAGE(), @ccs_dependencies ); + + return; + } + + sub _load_ccs_modules ($self) { + require Cpanel::LoadModule::Custom; + + Cpanel::LoadModule::Custom::load_perl_module('Cpanel::CCS::Delegates'); + Cpanel::LoadModule::Custom::load_perl_module('Cpanel::CCS::DBUtils'); + Cpanel::LoadModule::Custom::load_perl_module('Cpanel::CCS::Userdata'); + + return; + } + + sub export_ccs_data ($self) { + my $export_dir = EXPORT_DIR(); + INFO("Exporting CCS data to '$export_dir'. A backup of this data will be left in place after elevate completes."); + $self->_ensure_export_directory(); + + my @users = Cpanel::Config::Users::getcpusers(); + foreach my $user (@users) { + INFO(" Exporting data for $user"); + $self->_export_data_for_single_user($user); + } + + INFO('Completed exporting CCS data for all users'); + + return; + } + + sub _export_data_for_single_user ( $self, $user ) { + my $users_ccs_info = $self->_get_ccs_info_for_user($user); + my @webmail_users = keys %{ $users_ccs_info->{users} }; + + next if ( !@webmail_users ); + + $self->_make_backup_paths_for_user($user); + $self->_dump_persistence_data_for_user($user); + $self->_dump_delegation_data_for_user($user); + + foreach my $webmail_user (@webmail_users) { + $self->_process_calendar_and_contacts_for_webmail_user( $user, $webmail_user ); + } + + return; + } + + sub _process_calendar_and_contacts_for_webmail_user ( $self, $user, $webmail_user ) { + my $path = $self->_get_export_path_for_user($user); + my $users_ccs_info = $self->_get_ccs_info_for_user($user); + my $uuid = $users_ccs_info->{users}{$webmail_user}; + my %dump_types = DUMP_TYPES(); + my $dbh = $self->_get_dbh(); + + foreach my $type ( keys %dump_types ) { + my ( $query_string, $query_args ) = $self->_get_query_for_type( $type, $uuid ); + my $sth = $dbh->prepare($query_string); + + $sth->execute(@$query_args); + + my $num_rows = $sth->rows; + next if !$num_rows; + + my $dump_file = "$path/$type/${uuid}_${type}.$dump_types{$type}"; + + Cpanel::Autodie::open( my $dh, ">", $dump_file ); + binmode( $dh, ":encoding(UTF-8)" ) or die "Can't set binmode to UTF-8 on $dump_file: $!"; + + while ( my $text = $sth->fetch ) { + for (@$text) { + my $txt = $_; + $txt =~ tr/'//d; + print $dh $txt; + } + } + } + + return; + } + + sub _dump_delegation_data_for_user ( $self, $user ) { + my $path = $self->_get_export_path_for_user($user); + my $dbh = $self->_get_dbh(); + + my @webmail_users_info = Cpanel::CCS::Userdata::get_users($user); + my $delegates_ar = Cpanel::CCS::Delegates::get( @webmail_users_info, $dbh ); + + my $delegate_file = $path . '/' . 'delegates.json'; + Cpanel::JSON::DumpFile( $delegate_file, $delegates_ar ); + return; + } + + sub _dump_persistence_data_for_user ( $self, $user ) { + my $path = $self->_get_export_path_for_user($user); + my $persistence_file = $path . '/' . 'persistence.json'; + my $users_ccs_info = $self->_get_ccs_info_for_user($user); + + Cpanel::JSON::DumpFile( $persistence_file, $users_ccs_info ); + return; + } + + sub _make_backup_paths_for_user ( $self, $user ) { + my $path = $self->_get_export_path_for_user($user); + File::Path::make_path($path); + + my %dump_types = DUMP_TYPES(); + for ( keys(%dump_types) ) { File::Path::make_path("$path/$_"); } + return; + } + + sub _get_query_for_type ( $self, $type, $uuid ) { + my %querydata = ( + 'calendars' => { + 'args' => [ $uuid, '1', 'f' ], + 'query' => "SELECT icalendar_text + FROM + calendar_object + INNER JOIN calendar_bind ON calendar_bind.calendar_resource_id = calendar_object.calendar_resource_id + INNER JOIN calendar_metadata ON calendar_metadata.resource_id = calendar_bind.calendar_resource_id + INNER JOIN calendar_home ON calendar_home.resource_id = calendar_bind.calendar_home_resource_id + WHERE calendar_home.owner_uid = ? + AND calendar_bind.bind_status = ? + AND calendar_metadata.is_in_trash = ?;", + }, + 'contacts' => { + 'args' => [ $uuid, 'f' ], + 'query' => "SELECT vcard_text + FROM + addressbook_object + INNER JOIN addressbook_home ON addressbook_home.resource_id = addressbook_object.addressbook_home_resource_id + WHERE addressbook_home.owner_uid = ? + AND addressbook_object.is_in_trash = ?;", + }, + ); + + return ( $querydata{$type}{'query'}, $querydata{$type}{'args'} ); + } + + sub _get_dbh ($self) { + $self->{dbh} ||= Cpanel::CCS::DBUtils::get_dbh(); + return $self->{dbh}; + } + + sub _get_export_path_for_user ( $self, $user ) { + $self->{$user}{export_path} ||= EXPORT_DIR() . '/' . $user . '/calendar_and_contacts'; + return $self->{$user}{export_path}; + } + + sub _get_ccs_info_for_user ( $self, $user ) { + $self->{$user}{info} ||= Cpanel::CCS::Userdata::get_cpanel_account_users_uuids($user); + return $self->{$user}{info}; + } + + sub _ensure_export_directory ($self) { + File::Path::make_path(EXPORT_DIR); + + chmod 0700, EXPORT_DIR; + + return; + } + + sub post_leapp ($self) { + return unless Elevate::StageFile::read_stage_file('ccs_installed'); + + $self->_install_ccs_and_dependencies(); + + $self->_clear_task_queue(); + + $self->_ensure_ccs_service_is_up(); + $self->run_once('import_ccs_data'); + + return; + } + + sub _install_ccs_and_dependencies ($self) { + my @packages_to_install = ( CCS_PACKAGE() ); + + push @packages_to_install, ZPUSH_PACKAGE() if Elevate::StageFile::read_stage_file('zpush_installed'); + + $self->dnf->install(@packages_to_install); + + return; + } + + sub _clear_task_queue ($self) { + + $self->ssystem( TASK_QUEUE_SCRIPT, 'run' ); + return; + } + + sub _ensure_ccs_service_is_up ($self) { + + INFO('Attempting to ensure that the CCS service is running'); + + my $attempts = 1; + my $max_attempts = 5; + while ( $attempts <= $max_attempts ) { + DEBUG("Attempt $attempts of $max_attempts to verify that the CCS service is up"); + + if ( $self->_ccs_service_is_up() ) { + INFO('Verified that the CCS service is up'); + return; + } + + $self->remove_ccs_and_dependencies(); + $self->remove_cpanel_ccs_home_directory(); + $self->_clear_task_queue(); + $self->_install_ccs_and_dependencies(); + $self->_clear_task_queue(); + + sleep 5; + $attempts++; + } + + WARN("Failed to start CCS service. Importing CCS data may fail."); + return; + } + + sub _attempt_to_start_service ($self) { + $self->ssystem(CCS_RESTART_SCRIPT); + return; + } + + sub _ccs_service_is_up ($self) { + my $out = $self->ssystem_capture_output( CCS_RESTART_SCRIPT, '--status' ); + return grep { $_ =~ m/is running as cpanel-ccs with PID/ } @{ $out->{stdout} }; + } + + sub import_ccs_data ($self) { + INFO("Importing CCS data"); + + my @failed_users; + my @users = Cpanel::Config::Users::getcpusers(); + foreach my $user (@users) { + try { + INFO(" Importing data for $user"); + $self->_import_data_for_single_user($user); + } + catch { + push @failed_users, $user; + }; + } + + INFO('Completed importing CCS data for all users'); + + if (@failed_users) { + my $export_dir = EXPORT_DIR(); + my $message = "The CCS data failed to import for the following users:\n\n"; + $message .= join "\n", sort(@failed_users); + $message .= <<~"EOS"; + + A backup of this data is located at $export_dir + + If this data is crucial, you may want to consider reaching out to cPanel Support for further assistance: + + https://docs.cpanel.net/knowledge-base/technical-support-services/how-to-open-a-technical-support-ticket/ + + EOS + + Elevate::Notify::add_final_notification($message); + } + + return; + } + + sub _import_data_for_single_user ( $self, $user ) { + + require '/var/cpanel/perl5/lib/CCSHooks.pm'; ##no critic qw(RequireBarewordIncludes) + + my $extract_dir = EXPORT_DIR() . '/' . $user; + my $import_data = { + user => $user, + extract_dir => $extract_dir, + }; + + try { + CCSHooks::pkgacct_restore( undef, $import_data ); + } + catch { + my $err = $_; + WARN("Failed to restore CCS data for '$user'"); + DEBUG($err); + die "CCS import failed for $user\n"; + }; + + return; + } + + 1; + +} # --- END lib/Elevate/Components/CCS.pm + { # --- BEGIN lib/Elevate/Components/cPanelPlugins.pm package Elevate::Components::cPanelPlugins; @@ -6931,6 +7318,7 @@ use Elevate::Blockers::AutoSSL (); # - fatpack Components use Elevate::Components::Base (); use Elevate::Components::AbsoluteSymlinks (); +use Elevate::Components::CCS (); use Elevate::Components::cPanelPlugins (); use Elevate::Components::cPanelPrep (); use Elevate::Components::EA4 (); @@ -7617,14 +8005,19 @@ sub run_stage_2 ($self) { Elevate::Marker::startup(); + Elevate::Motd->setup(); + $self->ssystem(qw{/usr/bin/yum clean all}); $self->ssystem_and_die(qw{/scripts/update-packages}); $self->ssystem_and_die(qw{/usr/bin/yum -y update}); + # This needs to execute before we disable cPanel services + # or exporting the CCS data will fail + $self->run_component_once( 'CCS' => 'pre_leapp' ); $self->run_component_once( 'DatabaseUpgrade' => 'pre_leapp' ); - $self->run_component_once( 'cPanelPrep', => 'pre_leapp' ); - Elevate::Motd->setup(); + # This disable cPanel services + $self->run_component_once( 'cPanelPrep' => 'pre_leapp' ); $self->run_component_once( 'SSH' => 'pre_leapp' ); $self->run_component_once( 'AutoSSL' => 'pre_leapp' ); @@ -7787,6 +8180,8 @@ sub run_stage_4 ($self) { } ); + $self->run_component_once( 'CCS' => 'post_leapp' ); + return ACTION_REBOOT_NEEDED; } diff --git a/lib/Elevate/Blockers/WHM.pm b/lib/Elevate/Blockers/WHM.pm index 61b0875f..882b4103 100644 --- a/lib/Elevate/Blockers/WHM.pm +++ b/lib/Elevate/Blockers/WHM.pm @@ -19,7 +19,6 @@ use Cpanel::Backup::Sync (); use Cpanel::Version::Tiny (); use Cpanel::Update::Tiers (); use Cpanel::License (); -use Cpanel::Pkgr (); use Cpanel::Unix::PID::Tiny (); use parent qw{Elevate::Blockers::Base}; @@ -42,7 +41,6 @@ sub check ($self) { $ok = 0 unless $self->_blocker_is_sandbox; $ok = 0 unless $self->_blocker_is_upcp_running; $ok = 0 unless $self->_blocker_is_cpanel_backup_running; - $ok = 0 unless $self->_blocker_is_calendar_installed; return $ok; } @@ -139,17 +137,6 @@ sub _blocker_is_sandbox ($self) { return 0; } -sub _blocker_is_calendar_installed ($self) { - if ( Cpanel::Pkgr::is_installed('cpanel-ccs-calendarserver') ) { - return $self->has_blocker( <<~'EOS'); - You have the cPanel Calendar Server installed. Upgrades with this server in place are not supported. - Removal of this server can lead to data loss. - EOS - } - - return 0; -} - sub _blocker_is_upcp_running ($self) { return 0 unless $self->getopt('start'); diff --git a/lib/Elevate/Components/CCS.pm b/lib/Elevate/Components/CCS.pm new file mode 100644 index 00000000..c9475065 --- /dev/null +++ b/lib/Elevate/Components/CCS.pm @@ -0,0 +1,461 @@ +package Elevate::Components::CCS; + +=encoding utf-8 + +=head1 NAME + +Elevate::Components::CCS + +pre_leapp: Export CCS data to a root owned backup directory and remove CCS + package + +post_leapp: Install CCS package and import the backups taken during pre_leapp + +=cut + +use cPstrict; + +use Try::Tiny; + +use File::Path (); +use File::Copy (); + +use Cpanel::Autodie (); +use Cpanel::Config::Users (); +use Cpanel::JSON (); +use Cpanel::Pkgr (); + +use Elevate::Notify (); +use Elevate::StageFile (); + +use Log::Log4perl qw(:easy); + +use parent qw{Elevate::Components::Base}; + +use constant CCS_PACKAGE => 'cpanel-ccs-calendarserver'; +use constant ZPUSH_PACKAGE => 'cpanel-z-push'; +use constant EXPORT_DIR => '/var/cpanel/elevate_ccs_export'; +use constant CCS_RESTART_SCRIPT => '/usr/local/cpanel/scripts/restartsrv_cpanel_ccs'; +use constant TASK_QUEUE_SCRIPT => '/usr/local/cpanel/bin/servers_queue'; + +use constant DUMP_TYPES => ( + calendars => 'ics', + contacts => 'vcard', +); + +sub pre_leapp ($self) { + my $ccs_installed = Cpanel::Pkgr::is_installed(CCS_PACKAGE); + Elevate::StageFile::update_stage_file( { ccs_installed => $ccs_installed } ); + return unless $ccs_installed; + + $self->_load_ccs_modules(); + + $self->run_once('export_ccs_data'); + $self->remove_ccs_and_dependencies(); + + # Removing the PKG will leave this directory in place + # This results in PostGreSQL/CCS failing to start after leapp completes + $self->clean_up_pkg_cruft(); + + return; +} + +sub clean_up_pkg_cruft ($self) { + $self->move_pgsql_directory(); + $self->remove_cpanel_ccs_home_directory(); + return; +} + +=head1 remove_cpanel_ccs_home_directory + +Removing the package removes the `cpanel-ccs` user, but leaves behind +the data in the home directory. This results in the service failing +to start after the elevation has completed + +=cut + +sub remove_cpanel_ccs_home_directory ($self) { + File::Path::remove_tree('/opt/cpanel-ccs') if -d '/opt/cpanel-ccs'; + return; +} + +=head1 move_pgsql_directory + +Removing the PKG will leave this directory in place +This results in PostGreSQL/CCS failing to start after leapp completes + +=cut + +sub move_pgsql_directory ($self) { + my $pg_dir = '/var/lib/pgsql'; + my $pg_backup_dir = '/var/lib/pgsql_pre_elevate'; + + # Remove the backup path if it exists as a directory + File::Path::remove_tree($pg_backup_dir) if -e $pg_backup_dir && -d $pg_backup_dir; + + # If we were unable to remove the backup path above, then change it to something that + # should be unique + $pg_backup_dir .= '_' . time() . '_' . $$ if -e $pg_backup_dir; + + # Make sure the path that should be unique does not exist + File::Path::remove_tree($pg_backup_dir) if -e $pg_backup_dir && -d $pg_backup_dir; + + # Give it up if we still do not have a candidate to back the data up to + if ( -e $pg_backup_dir ) { + die <<~"EOS"; + Unable to ensure a valid backup path for $pg_dir. + Please ensure that '/var/lib/pgsql_pre_elevate' does not exist on your system and execute this script again with + + /scripts/elevate-cpanel --continue + + EOS + } + + INFO( <<~"EOS" ); + Moving the PostgreSQL data dir located at $pg_dir to $pg_backup_dir + to ensure a functioning PostgreSQL server after the elevation completes. + EOS + + File::Copy::move( $pg_dir, $pg_backup_dir ) if -d $pg_dir; + + return; +} + +sub remove_ccs_and_dependencies ($self) { + + my $zpush_installed = Cpanel::Pkgr::is_installed(ZPUSH_PACKAGE); + Elevate::StageFile::update_stage_file( { zpush_installed => $zpush_installed } ); + + # There are other dependencies but these are the 3 that we are concerned with + my @ccs_dependencies = qw{ + postgresql + postgresql-devel + postgresql-server + }; + + push @ccs_dependencies, ZPUSH_PACKAGE(); + + $self->yum->remove( CCS_PACKAGE(), @ccs_dependencies ); + + return; +} + +sub _load_ccs_modules ($self) { + require Cpanel::LoadModule::Custom; + + Cpanel::LoadModule::Custom::load_perl_module('Cpanel::CCS::Delegates'); + Cpanel::LoadModule::Custom::load_perl_module('Cpanel::CCS::DBUtils'); + Cpanel::LoadModule::Custom::load_perl_module('Cpanel::CCS::Userdata'); + + return; +} + +=head1 + +This export code is largely based on the code in +'/var/cpanel/perl/Cpanel/Pkgacct/Components/CCSPkgAcct.pm' which is provided +by the CCS package. Unfortunately, I could not call into that code here +as that code is intended to be used by '/scripts/pkgacct' and uses +'Cpanel::Pkgacct::Component' as a parent. Due to that, I made the decision +to port that code over to elevate directly. + +TL;DR: We will want to monitor CCS for bug fixes/changes as we may need to +update this code to resolve issues that get resolved there + +=cut + +sub export_ccs_data ($self) { + my $export_dir = EXPORT_DIR(); + INFO("Exporting CCS data to '$export_dir'. A backup of this data will be left in place after elevate completes."); + $self->_ensure_export_directory(); + + my @users = Cpanel::Config::Users::getcpusers(); + foreach my $user (@users) { + INFO(" Exporting data for $user"); + $self->_export_data_for_single_user($user); + } + + INFO('Completed exporting CCS data for all users'); + + return; +} + +sub _export_data_for_single_user ( $self, $user ) { + my $users_ccs_info = $self->_get_ccs_info_for_user($user); + my @webmail_users = keys %{ $users_ccs_info->{users} }; + + # Should be impossible to hit this condition, but be paranoid + next if ( !@webmail_users ); + + $self->_make_backup_paths_for_user($user); + $self->_dump_persistence_data_for_user($user); + $self->_dump_delegation_data_for_user($user); + + foreach my $webmail_user (@webmail_users) { + $self->_process_calendar_and_contacts_for_webmail_user( $user, $webmail_user ); + } + + return; +} + +sub _process_calendar_and_contacts_for_webmail_user ( $self, $user, $webmail_user ) { + my $path = $self->_get_export_path_for_user($user); + my $users_ccs_info = $self->_get_ccs_info_for_user($user); + my $uuid = $users_ccs_info->{users}{$webmail_user}; + my %dump_types = DUMP_TYPES(); + my $dbh = $self->_get_dbh(); + + foreach my $type ( keys %dump_types ) { + my ( $query_string, $query_args ) = $self->_get_query_for_type( $type, $uuid ); + my $sth = $dbh->prepare($query_string); + + $sth->execute(@$query_args); + + my $num_rows = $sth->rows; + next if !$num_rows; + + # Write the relevant file from the dump + my $dump_file = "$path/$type/${uuid}_${type}.$dump_types{$type}"; + + Cpanel::Autodie::open( my $dh, ">", $dump_file ); + binmode( $dh, ":encoding(UTF-8)" ) or die "Can't set binmode to UTF-8 on $dump_file: $!"; + + while ( my $text = $sth->fetch ) { + for (@$text) { + my $txt = $_; + $txt =~ tr/'//d; + print $dh $txt; + } + } + } + + return; +} + +sub _dump_delegation_data_for_user ( $self, $user ) { + my $path = $self->_get_export_path_for_user($user); + my $dbh = $self->_get_dbh(); + + my @webmail_users_info = Cpanel::CCS::Userdata::get_users($user); + my $delegates_ar = Cpanel::CCS::Delegates::get( @webmail_users_info, $dbh ); + + my $delegate_file = $path . '/' . 'delegates.json'; + Cpanel::JSON::DumpFile( $delegate_file, $delegates_ar ); + return; +} + +sub _dump_persistence_data_for_user ( $self, $user ) { + my $path = $self->_get_export_path_for_user($user); + my $persistence_file = $path . '/' . 'persistence.json'; + my $users_ccs_info = $self->_get_ccs_info_for_user($user); + + Cpanel::JSON::DumpFile( $persistence_file, $users_ccs_info ); + return; +} + +sub _make_backup_paths_for_user ( $self, $user ) { + my $path = $self->_get_export_path_for_user($user); + File::Path::make_path($path); + + my %dump_types = DUMP_TYPES(); + for ( keys(%dump_types) ) { File::Path::make_path("$path/$_"); } + return; +} + +sub _get_query_for_type ( $self, $type, $uuid ) { + my %querydata = ( + 'calendars' => { + 'args' => [ $uuid, '1', 'f' ], + 'query' => "SELECT icalendar_text + FROM + calendar_object + INNER JOIN calendar_bind ON calendar_bind.calendar_resource_id = calendar_object.calendar_resource_id + INNER JOIN calendar_metadata ON calendar_metadata.resource_id = calendar_bind.calendar_resource_id + INNER JOIN calendar_home ON calendar_home.resource_id = calendar_bind.calendar_home_resource_id + WHERE calendar_home.owner_uid = ? + AND calendar_bind.bind_status = ? + AND calendar_metadata.is_in_trash = ?;", + }, + 'contacts' => { + 'args' => [ $uuid, 'f' ], + 'query' => "SELECT vcard_text + FROM + addressbook_object + INNER JOIN addressbook_home ON addressbook_home.resource_id = addressbook_object.addressbook_home_resource_id + WHERE addressbook_home.owner_uid = ? + AND addressbook_object.is_in_trash = ?;", + }, + ); + + return ( $querydata{$type}{'query'}, $querydata{$type}{'args'} ); +} + +sub _get_dbh ($self) { + $self->{dbh} ||= Cpanel::CCS::DBUtils::get_dbh(); + return $self->{dbh}; +} + +sub _get_export_path_for_user ( $self, $user ) { + $self->{$user}{export_path} ||= EXPORT_DIR() . '/' . $user . '/calendar_and_contacts'; + return $self->{$user}{export_path}; +} + +sub _get_ccs_info_for_user ( $self, $user ) { + $self->{$user}{info} ||= Cpanel::CCS::Userdata::get_cpanel_account_users_uuids($user); + return $self->{$user}{info}; +} + +sub _ensure_export_directory ($self) { + File::Path::make_path(EXPORT_DIR); + + # Do not use File::Path to do this since it only + # runs chmod if it creates the directory and we want + # to make sure it is 0700 regardless if it was just created + # or not + chmod 0700, EXPORT_DIR; + + return; +} + +#################################### +##### post_leapp code below this ### +#################################### + +sub post_leapp ($self) { + return unless Elevate::StageFile::read_stage_file('ccs_installed'); + + $self->_install_ccs_and_dependencies(); + + # This needs to happen before verifying that the service is up + # There is a task created that makes a schema update that can + # cause CCS to fail to start + $self->_clear_task_queue(); + + $self->_ensure_ccs_service_is_up(); + $self->run_once('import_ccs_data'); + + return; +} + +sub _install_ccs_and_dependencies ($self) { + my @packages_to_install = ( CCS_PACKAGE() ); + + push @packages_to_install, ZPUSH_PACKAGE() if Elevate::StageFile::read_stage_file('zpush_installed'); + + $self->dnf->install(@packages_to_install); + + return; +} + +sub _clear_task_queue ($self) { + + # CCS queues up tasks when it is first installed + # These need to run before we attempt to import data + $self->ssystem( TASK_QUEUE_SCRIPT, 'run' ); + return; +} + +sub _ensure_ccs_service_is_up ($self) { + + INFO('Attempting to ensure that the CCS service is running'); + + my $attempts = 1; + my $max_attempts = 5; + while ( $attempts <= $max_attempts ) { + DEBUG("Attempt $attempts of $max_attempts to verify that the CCS service is up"); + + if ( $self->_ccs_service_is_up() ) { + INFO('Verified that the CCS service is up'); + return; + } + + # If the service was not up at this point, it is likely that + # the schema update failed during the install. In this case, + # the only way I have found to get things working again is to + # completely remove the package and start the install over + + $self->remove_ccs_and_dependencies(); + $self->remove_cpanel_ccs_home_directory(); + $self->_clear_task_queue(); + $self->_install_ccs_and_dependencies(); + $self->_clear_task_queue(); + + sleep 5; + $attempts++; + } + + WARN("Failed to start CCS service. Importing CCS data may fail."); + return; +} + +sub _attempt_to_start_service ($self) { + $self->ssystem(CCS_RESTART_SCRIPT); + return; +} + +sub _ccs_service_is_up ($self) { + my $out = $self->ssystem_capture_output( CCS_RESTART_SCRIPT, '--status' ); + return grep { $_ =~ m/is running as cpanel-ccs with PID/ } @{ $out->{stdout} }; +} + +sub import_ccs_data ($self) { + INFO("Importing CCS data"); + + my @failed_users; + my @users = Cpanel::Config::Users::getcpusers(); + foreach my $user (@users) { + try { + INFO(" Importing data for $user"); + $self->_import_data_for_single_user($user); + } + catch { + push @failed_users, $user; + }; + } + + INFO('Completed importing CCS data for all users'); + + if (@failed_users) { + my $export_dir = EXPORT_DIR(); + my $message = "The CCS data failed to import for the following users:\n\n"; + $message .= join "\n", sort(@failed_users); + $message .= <<~"EOS"; + + A backup of this data is located at $export_dir + + If this data is crucial, you may want to consider reaching out to cPanel Support for further assistance: + + https://docs.cpanel.net/knowledge-base/technical-support-services/how-to-open-a-technical-support-ticket/ + + EOS + + Elevate::Notify::add_final_notification($message); + } + + return; +} + +sub _import_data_for_single_user ( $self, $user ) { + + require '/var/cpanel/perl5/lib/CCSHooks.pm'; ##no critic qw(RequireBarewordIncludes) + + my $extract_dir = EXPORT_DIR() . '/' . $user; + my $import_data = { + user => $user, + extract_dir => $extract_dir, + }; + + try { + CCSHooks::pkgacct_restore( undef, $import_data ); + } + catch { + my $err = $_; + WARN("Failed to restore CCS data for '$user'"); + DEBUG($err); + die "CCS import failed for $user\n"; + }; + + return; +} + +1; diff --git a/script/elevate-cpanel.PL b/script/elevate-cpanel.PL index 64d218ec..32c9dc04 100755 --- a/script/elevate-cpanel.PL +++ b/script/elevate-cpanel.PL @@ -257,6 +257,7 @@ use Elevate::Blockers::AutoSSL (); # - fatpack Components use Elevate::Components::Base (); use Elevate::Components::AbsoluteSymlinks (); +use Elevate::Components::CCS (); use Elevate::Components::cPanelPlugins (); use Elevate::Components::cPanelPrep (); use Elevate::Components::EA4 (); @@ -943,14 +944,19 @@ sub run_stage_2 ($self) { Elevate::Marker::startup(); + Elevate::Motd->setup(); + $self->ssystem(qw{/usr/bin/yum clean all}); $self->ssystem_and_die(qw{/scripts/update-packages}); $self->ssystem_and_die(qw{/usr/bin/yum -y update}); + # This needs to execute before we disable cPanel services + # or exporting the CCS data will fail + $self->run_component_once( 'CCS' => 'pre_leapp' ); $self->run_component_once( 'DatabaseUpgrade' => 'pre_leapp' ); - $self->run_component_once( 'cPanelPrep', => 'pre_leapp' ); - Elevate::Motd->setup(); + # This disable cPanel services + $self->run_component_once( 'cPanelPrep' => 'pre_leapp' ); $self->run_component_once( 'SSH' => 'pre_leapp' ); $self->run_component_once( 'AutoSSL' => 'pre_leapp' ); @@ -1113,6 +1119,8 @@ sub run_stage_4 ($self) { } ); + $self->run_component_once( 'CCS' => 'post_leapp' ); + return ACTION_REBOOT_NEEDED; } diff --git a/t/blocker-whm.t b/t/blocker-whm.t index 685345ce..ddcea2c1 100644 --- a/t/blocker-whm.t +++ b/t/blocker-whm.t @@ -309,24 +309,4 @@ my $whm = $cpev->get_blocker('WHM'); $whm->blockers->abort_on_first_blocker(0); } -{ - note "CCS CalendarServer"; - - my $pkgr_mock = Test::MockModule->new('Cpanel::Pkgr'); - my %installed = ( 'cpanel-ccs-calendarserver' => 9.2 ); - $pkgr_mock->redefine( 'is_installed' => sub ($rpm) { return defined $installed{$rpm} ? 1 : 0 } ); - $pkgr_mock->redefine( 'get_package_version' => sub ($rpm) { return $installed{$rpm} } ); - - is( - $whm->_blocker_is_calendar_installed(), - { - id => q[Elevate::Blockers::WHM::_blocker_is_calendar_installed], - msg => "You have the cPanel Calendar Server installed. Upgrades with this server in place are not supported.\nRemoval of this server can lead to data loss.\n", - }, - 'CCS server is a blocker..' - ); - delete $installed{'cpanel-ccs-calendarserver'}; - is( $whm->_blocker_is_calendar_installed(), 0, "if CCS isn't installed, we're ok" ); -} - done_testing(); diff --git a/t/components-CCS.t b/t/components-CCS.t new file mode 100644 index 00000000..70ac3782 --- /dev/null +++ b/t/components-CCS.t @@ -0,0 +1,156 @@ +#!/usr/local/cpanel/3rdparty/bin/perl + +# Copyright 2024 WebPros International, LLC +# All rights reserved. +# copyright@cpanel.net http://cpanel.net +# This code is subject to the cPanel license. Unauthorized copying is prohibited. + +package test::cpev::components; + +use FindBin; + +use Test2::V0; +use Test2::Tools::Explain; +use Test2::Plugin::NoWarnings; +use Test2::Tools::Exception; + +use Test::MockModule qw/strict/; + +use lib $FindBin::Bin . "/lib"; +use Test::Elevate; + +use cPstrict; + +my $ccs = bless {}, 'Elevate::Components::CCS'; + +my $mock_ccs = Test::MockModule->new('Elevate::Components::CCS'); +my $mock_stagefile = Test::MockModule->new('Elevate::StageFile'); + +{ + note "Checking pre_leapp"; + + my $installed = 0; + + my $mock_pkgr = Test::MockModule->new('Cpanel::Pkgr'); + $mock_pkgr->redefine( + is_installed => sub { return $installed; }, + ); + + $mock_stagefile->redefine( + update_stage_file => 0, + ); + + my $called__load_ccs_modules = 0; + my $called_clean_up_pkg_cruft = 0; + my @called_for_user; + my @ssystem_and_die_params; + $mock_ccs->redefine( + _load_ccs_modules => sub { $called__load_ccs_modules = 1; }, + _ensure_export_directory => 0, + _export_data_for_single_user => sub ( $self, $user ) { push @called_for_user, $user; }, + clean_up_pkg_cruft => sub { $called_clean_up_pkg_cruft = 1; }, + run_once => sub { $ccs->export_ccs_data(); }, + ssystem_and_die => sub { + shift; + @ssystem_and_die_params = @_; + return; + }, + ); + + is( $ccs->pre_leapp(), undef, 'pre_leapp is basically a noop if CCS is not installed' ); + is( $called__load_ccs_modules, 0, 'pre_leapp returned before loading CCS modules when CCS was not installed' ); + + my @cpusers = qw{ foo bar baz }; + my $mock_cpanel_config_users = Test::MockModule->new('Cpanel::Config::Users'); + $mock_cpanel_config_users->redefine( + getcpusers => sub { return @cpusers; }, + ); + + $installed = 1; + + $ccs->pre_leapp(); + is( $called__load_ccs_modules, 1, '_load_ccs_modules is called when CCS is installed' ); + + message_seen( 'INFO', qr/^Exporting CCS data to/ ); + + is( \@called_for_user, \@cpusers, 'The expected users had data exported' ); + + is( + \@ssystem_and_die_params, + [qw{ /usr/bin/yum -y remove cpanel-ccs-calendarserver postgresql postgresql-devel postgresql-server cpanel-z-push }], + 'The expected packages were removed' + ); + + is( $called_clean_up_pkg_cruft, 1, 'Package cruft leftover after removal was cleaned up' ); + + for my $user (@cpusers) { + message_seen( 'INFO', " Exporting data for $user" ); + } + + message_seen( 'INFO', 'Completed exporting CCS data for all users' ); + + no_messages_seen(); +} + +{ + note "Checking post_leapp"; + + my $installed = 0; + $mock_stagefile->redefine( + read_stage_file => sub { return $installed; }, + ); + + my $ssystem_and_die_params = []; + my @called_for_user; + $mock_ccs->redefine( + ssystem_and_die => sub { + shift; + push @{$ssystem_and_die_params}, @_; + return; + }, + ssystem => sub { + shift; + push @{$ssystem_and_die_params}, @_; + return; + }, + _ensure_ccs_service_is_up => 0, + run_once => sub { $ccs->import_ccs_data(); }, + _import_data_for_single_user => sub ( $self, $user ) { push @called_for_user, $user; }, + ); + + is( $ccs->post_leapp(), undef, 'post_leapp is a noop if CCS was not installed' ); + is( $ssystem_and_die_params, [], 'No system commands were called when CCS was not installed' ); + + my @cpusers; + my $mock_cpanel_config_users = Test::MockModule->new('Cpanel::Config::Users'); + $mock_cpanel_config_users->redefine( + getcpusers => sub { return @cpusers; }, + ); + + $installed = 1; + + $ccs->post_leapp(); + + message_seen( 'INFO', 'Importing CCS data' ); + + is( + $ssystem_and_die_params, + [ + qw{/usr/bin/dnf -y install cpanel-ccs-calendarserver cpanel-z-push}, + qw{/usr/local/cpanel/bin/servers_queue run}, + ], + 'The expected commands are called during post_leapp when CCS was installed', + ); + + is( \@called_for_user, \@cpusers, 'CCS data was imported for the expected users' ); + + foreach my $user (@cpusers) { + message_seen( 'INFO', " Importing data for $user" ); + } + + message_seen( 'INFO', 'Completed importing CCS data for all users' ); + + no_messages_seen(); +} + +done_testing();