diff --git a/.github/workflows/openstack.yml b/.github/workflows/openstack.yml index 03f0be9a..f45a2bce 100644 --- a/.github/workflows/openstack.yml +++ b/.github/workflows/openstack.yml @@ -150,10 +150,12 @@ jobs: username: 'root' key: ${{ secrets.SSH_PRIVATE_KEY }} port: '22' + timeout: 30m + command_timeout: 30m script: | cp -pv /opt/elevate/elevate-cpanel /scripts/elevate-cpanel - cp -pv /opt/elevate/.github/workflows/openstack/status_marker.sh /scripts/status_marker.sh - cp -pv /opt/elevate/.github/workflows/openstack/reboot_watch.sh /scripts/reboot_watch.sh + cp -pv /opt/elevate/.github/workflows/openstack/status_marker /scripts/status_marker + cp -pv /opt/elevate/.github/workflows/openstack/reboot_watch /scripts/reboot_watch chmod -v +x /scripts/elevate-cpanel /usr/local/cpanel/cpkeyclt /scripts/elevate-cpanel --non-interactive --start & @@ -169,7 +171,7 @@ jobs: - name: Wait For VM to Come Back From Stage 2 Reboot working-directory: "./.github/workflows/openstack/" run: | - ./ssh_retry.sh ${{ needs.start_elevate.outputs.VM_IP }} + ./ssh_retry ${{ needs.start_elevate.outputs.VM_IP }} watch_for_stage_3_reboot: runs-on: self-hosted @@ -185,12 +187,12 @@ jobs: key: ${{ secrets.SSH_PRIVATE_KEY }} port: '22' timeout: 60m - command_timeout: 20m + command_timeout: 30m debug: true script: | - /scripts/status_marker.sh 3 + /scripts/status_marker 3 /scripts/elevate-cpanel --log & - REGEX="Rebooting into stage 3 of 5" RETVAL=1 /scripts/reboot_watch.sh + REGEX="Rebooting into stage 3 of 5" RETVAL=1 /scripts/reboot_watch wait_for_stage_3_reboot: runs-on: self-hosted @@ -201,7 +203,7 @@ jobs: - name: Wait For VM to Come Back From Stage 3 Reboot working-directory: "./.github/workflows/openstack/" run: | - ./ssh_retry.sh ${{ needs.watch_for_stage_3_reboot.outputs.VM_IP }} + ./ssh_retry ${{ needs.watch_for_stage_3_reboot.outputs.VM_IP }} watch_for_stage_4_reboot: runs-on: self-hosted @@ -217,11 +219,11 @@ jobs: key: ${{ secrets.SSH_PRIVATE_KEY }} port: '22' timeout: 30m - command_timeout: 20m + command_timeout: 30m script: | - /scripts/status_marker.sh 4 + /scripts/status_marker 4 /scripts/elevate-cpanel --log & - REGEX="Rebooting into stage 4 of 5" RETVAL=1 /scripts/reboot_watch.sh + REGEX="Rebooting into stage 4 of 5" RETVAL=1 /scripts/reboot_watch wait_for_stage_4_reboot: runs-on: self-hosted @@ -232,7 +234,7 @@ jobs: - name: Wait For VM to Come Back From Stage 4 Reboot working-directory: "./.github/workflows/openstack/" run: | - ./ssh_retry.sh ${{ needs.watch_for_stage_4_reboot.outputs.VM_IP }} + ./ssh_retry ${{ needs.watch_for_stage_4_reboot.outputs.VM_IP }} watch_for_stage_5_reboot: runs-on: self-hosted @@ -247,12 +249,12 @@ jobs: username: 'root' key: ${{ secrets.SSH_PRIVATE_KEY }} port: '22' - timeout: 30m - command_timeout: 20m + timeout: 45m + command_timeout: 35m script: | - /scripts/status_marker.sh 5 + /scripts/status_marker 5 /scripts/elevate-cpanel --log & - REGEX="Rebooting into stage 5 of 5" RETVAL=1 /scripts/reboot_watch.sh + REGEX="Rebooting into stage 5 of 5" RETVAL=1 /scripts/reboot_watch wait_for_stage_5_reboot: runs-on: self-hosted @@ -263,7 +265,7 @@ jobs: - name: Wait For VM to Come Back From Stage 5 Reboot working-directory: "./.github/workflows/openstack/" run: | - ./ssh_retry.sh ${{ needs.watch_for_stage_5_reboot.outputs.VM_IP }} + ./ssh_retry ${{ needs.watch_for_stage_5_reboot.outputs.VM_IP }} watch_for_final_reboot: runs-on: self-hosted @@ -279,10 +281,10 @@ jobs: key: ${{ secrets.SSH_PRIVATE_KEY }} port: '22' timeout: 30m - command_timeout: 20m + command_timeout: 30m script: | /scripts/elevate-cpanel --log & - REGEX="Doing final reboot" RETVAL=1 /scripts/reboot_watch.sh + REGEX="Doing final reboot" RETVAL=1 /scripts/reboot_watch wait_for_final_reboot: runs-on: self-hosted @@ -293,7 +295,7 @@ jobs: - name: Wait For VM to Come Back From Final Reboot working-directory: "./.github/workflows/openstack/" run: | - ./ssh_retry.sh ${{ needs.watch_for_final_reboot.outputs.VM_IP }} + ./ssh_retry ${{ needs.watch_for_final_reboot.outputs.VM_IP }} verify_upgraded_os: runs-on: self-hosted diff --git a/.github/workflows/openstack/reboot_watch b/.github/workflows/openstack/reboot_watch new file mode 100755 index 00000000..8352bd6b --- /dev/null +++ b/.github/workflows/openstack/reboot_watch @@ -0,0 +1,30 @@ +#!/usr/local/cpanel/3rdparty/bin/perl + +use constant ELEVATE_LOG_PATH => '/var/log/elevate-cpanel.log'; + +use Env; + +use File::Tail; +use POSIX; + +my $RETVAL = 1; +my $RETRIES = 0; + +while ( $RETVAL != 0 ) { + _check_elevate_log_for_regex( ELEVATE_LOG_PATH, ${REGEX}, $RETRIES ); +} + +sub _check_elevate_log_for_regex { + my ( $filepath, $REGEX, $RETRIES ) = @_; + + my $time = POSIX::strftime( "%Y-%m-%d %H:%M:%S", localtime ); + + $file = File::Tail->new( name => $filepath, maxinterval => 1, adjustafter => 7, interval => 1 ); + while ( defined( $line = $file->read ) ) { + if ( grep { /$REGEX/m } $line ) { + print "## [$time] [INFO]: SUCCESS: Reboot regex ( $REGEX ) found in /var/log/elevate-cpanel.log ##\n"; + exit 0; + } + $RETRIES++; + } +} diff --git a/.github/workflows/openstack/reboot_watch.sh b/.github/workflows/openstack/reboot_watch.sh deleted file mode 100755 index 838ca35f..00000000 --- a/.github/workflows/openstack/reboot_watch.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -while [ $RETVAL -ne 0 ]; do - grep "${REGEX}" /var/log/elevate-cpanel.log; - RETVAL=$?; - [ $RETVAL -eq 0 ] && echo "## [INFO] SUCCESS: Reboot text found in /var/log/elevate-cpanel.log ##" && exit 0; - RETRIES=$((RETRIES+1)); - [ $RETVAL -ne 0 ] && echo "## [DEBUG]: Retrying Reboot REGEX Search: Attempt ${RETRIES} ..."; - sleep 1; -done diff --git a/.github/workflows/openstack/ssh_retry b/.github/workflows/openstack/ssh_retry new file mode 100755 index 00000000..c98cf7c1 --- /dev/null +++ b/.github/workflows/openstack/ssh_retry @@ -0,0 +1,36 @@ +#!/usr/bin/env perl + +use POSIX; + +my $HOST = $ARGV[0]; +my $PORT = $ARGV[1] // 22; +my $RETVAL = 1; +my $RETRIES = 0; +my $RETRY = $ARGV[2] // 60; + +return unless defined $HOST; + +while ( $RETVAL != 0 ) { + my $cmd = qq{ /usr/bin/nc -z -w 3 $HOST $PORT }; + my $output = `$cmd`; + my $time = POSIX::strftime( "%Y-%m-%d %H:%M:%S", localtime ); + + $RETVAL = $?; + + if ( $RETVAL == 0 ) { + print "## [$time] [INFO] SUCCESS: Connected to SSH on $HOST ##\n"; + exit 0; + } + + $RETRIES++; + + if ( $RETVAL != 0 ) { + print "## [$time] [INFO]: Retrying SSH Connect: Attempt ${RETRIES} ...\n"; + } + + if ( $RETRIES >= $RETRY ) { + print "## [$time] [ERROR]: ssh_retry.pl: MAX_RETRIES has been reached.\n"; + exit 1; + } + sleep 15; +} diff --git a/.github/workflows/openstack/ssh_retry.sh b/.github/workflows/openstack/ssh_retry.sh deleted file mode 100755 index 4f5c0680..00000000 --- a/.github/workflows/openstack/ssh_retry.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -RETVAL=1; -RETRIES=0; -HOST=$1; -PORT=$2 -PORT="${PORT:=22}"; -# Default 30 RETRIES usually one second apart -RETRY=$3 -RETRY="${RETRY:=30}"; - -while [ $RETVAL -ne 0 ]; -do - # We want to exit immediately after we actually connect to SSH on default port. - nc -z ${HOST} ${PORT} - RETVAL=$?; - [ $RETVAL -eq 0 ] && echo "## [INFO] SUCCESS: Connected to SSH on ${HOST} ##" && exit 0; - RETRIES=$((RETRIES+1)); - [ $RETVAL -ne 0 ] && echo "## [DEBUG]: Retrying SSH Connect: Attempt ${RETRIES} ..."; - - if [ ${RETRIES} -ge ${RETRY} ]; - then - echo "## [ERROR]: ssh_retry.sh: MAX_RETRIES has been reached."; - exit 1; - fi; - sleep 5; -done diff --git a/.github/workflows/openstack/status_marker b/.github/workflows/openstack/status_marker new file mode 100755 index 00000000..f26776ab --- /dev/null +++ b/.github/workflows/openstack/status_marker @@ -0,0 +1,29 @@ +#!/usr/bin/env perl + +my $STAGE = $ARGV[0]; +my $RELEASE_INFO = `cat /etc/redhat-release`; +my $CPANEL_VERSION = `cat /usr/local/cpanel/version`; + +use strict; + +print "######################################\n"; + +sub main { + my @arr = ( + [ 'Stage:', $STAGE ], [ 'OS Release:', $RELEASE_INFO ] + , [ 'cP Version:', $CPANEL_VERSION ] + ); + + for my $row (@arr) { + format STDOUT = +@<<<<<<<<<<<<<<<<< @<<<<<<<<<<<<<<<<< + @$row +. + write; + } + +} + +main(); + +print "######################################\n"; diff --git a/.github/workflows/openstack/status_marker.sh b/.github/workflows/openstack/status_marker.sh deleted file mode 100755 index 8246705f..00000000 --- a/.github/workflows/openstack/status_marker.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -STAGE=$1; -RELEASE_INFO=$(cat /etc/redhat-release); -CPANEL_VERSION=$(cat /usr/local/cpanel/version); - -RC=$(echo ${RELEASE_INFO} | wc -c) - - - -for ((i=1; i<=${RC}+17; i++)); do echo -n "#"; done -echo; -echo "# STAGE: ${STAGE} of 5 #"; -echo "# OS Release: ${RELEASE_INFO} #"; -echo "# cP Version: ${CPANEL_VERSION} #"; -for ((i=1; i<=${RC}+17; i++)); do echo -n "#"; done -echo;