diff --git a/installation_and_upgrade/ibex_install_utils/install_tasks.py b/installation_and_upgrade/ibex_install_utils/install_tasks.py index 3778af1..a83acf3 100644 --- a/installation_and_upgrade/ibex_install_utils/install_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/install_tasks.py @@ -14,6 +14,7 @@ from ibex_install_utils.tasks.server_tasks import ServerTasks from ibex_install_utils.tasks.system_tasks import SystemTasks from ibex_install_utils.tasks.vhd_tasks import VHDTasks +from ibex_install_utils.user_prompt import UserPrompt class UpgradeInstrument: @@ -21,13 +22,13 @@ class UpgradeInstrument: def __init__( self, - user_prompt, - server_source_dir, - client_source_dir, - genie_python3_dir, - ibex_version, - file_utils=FileUtils(), - ): + user_prompt: UserPrompt, + server_source_dir: str, + client_source_dir: str, + genie_python3_dir: str, + ibex_version: str, + file_utils: FileUtils = FileUtils(), + ) -> None: """Initializer. Args: @@ -111,14 +112,14 @@ def __init__( ) @staticmethod - def icp_in_labview_modules(): + def icp_in_labview_modules() -> bool: """Condition on which to install ICP_Binaries or :return: True if the ICP is installed in labview modules, False otherwise """ return os.path.exists(LABVIEW_DAE_DIR) - def run_test_update(self): + def run_test_update(self) -> None: """Run a complete test upgrade on the current system""" self._system_tasks.user_confirm_upgrade_type_on_machine("Training Machine") self._system_tasks.install_or_upgrade_git() @@ -136,10 +137,11 @@ def run_test_update(self): self._system_tasks.upgrade_notepad_pp() self._server_tasks.setup_log_rotation() - def remove_all_and_install_client_and_server(self): + def remove_all_and_install_client_and_server(self) -> None: """Either install or upgrade the ibex client and server""" self._system_tasks.confirm( - "This script removes IBEX client and server and installs the latest build of both, and upgrade the " + "This script removes IBEX client and server and installs " + "the latest build of both, and upgrade the " "config/schema without any extra steps. Proceed?" ) @@ -153,13 +155,13 @@ def remove_all_and_install_client_and_server(self): self._server_tasks.upgrade_instrument_configuration() self._server_tasks.install_shared_scripts_repository() - def run_instrument_tests(self): + def run_instrument_tests(self) -> None: """Run through client and server tests once installation / deployment has completed.""" self._client_tasks.perform_client_tests() self._server_tasks.perform_server_tests() self._system_tasks.inform_instrument_scientists() - def run_instrument_install(self): + def run_instrument_install(self) -> None: """Do a first installation of IBEX on a new instrument.""" self._system_tasks.confirm( "This script performs a first-time full installation of the IBEX server and client" @@ -198,19 +200,20 @@ def run_instrument_install(self): self._python_tasks.update_script_definitions() self._server_tasks.setup_log_rotation() - def save_motor_params(self): + def save_motor_params(self) -> None: self._server_tasks.save_motor_parameters_to_file() - def run_instrument_deploy(self): + def run_instrument_deploy(self) -> None: """Deploy a full IBEX upgrade on an existing instrument.""" self._system_tasks.confirm( - "This script performs a full upgrade of the IBEX server and client on an existing instrument. Proceed?" + "This script performs a full upgrade of the IBEX server " + "and client on an existing instrument. Proceed?" ) self.run_instrument_deploy_pre_stop() self.run_instrument_deploy_main() self.run_instrument_deploy_post_start() - def run_instrument_deploy_post_start(self): + def run_instrument_deploy_post_start(self) -> None: """Upgrade an instrument. Steps to do after ibex has been started. Current the server can not be started in this python script. @@ -225,8 +228,9 @@ def run_instrument_deploy_post_start(self): self._system_tasks.put_autostart_script_in_startup_area() self._system_tasks.inform_instrument_scientists() - def run_instrument_deploy_main(self): - """Upgrade an instrument. Steps to do after ibex has been stopped but before it is restarted. + def run_instrument_deploy_main(self) -> None: + """Upgrade an instrument. Steps to do after ibex has been stopped + but before it is restarted. Current the server can not be started or stopped in this python script. """ @@ -253,7 +257,7 @@ def run_instrument_deploy_main(self): self._python_tasks.remove_instrument_script_githooks() self._server_tasks.setup_log_rotation() - def run_instrument_deploy_pre_stop(self): + def run_instrument_deploy_pre_stop(self) -> None: """Upgrade an instrument. Steps to do before ibex is stopped. Current the server can not be started or stopped in this python script. @@ -267,7 +271,7 @@ def run_instrument_deploy_pre_stop(self): bytes(central_inst_info, encoding="utf8") ).decode("utf-8") central_inst_info = json.loads(central_inst_info) - except: + except Exception: central_inst_info = {} central_specific_inst_info = None @@ -284,24 +288,38 @@ def run_instrument_deploy_pre_stop(self): self._server_tasks.save_motor_blocks_blockserver_to_file() - def run_truncate_database(self): + def run_truncate_database(self) -> None: """Backup and truncate databases only""" self._mysql_tasks.backup_database() self._mysql_tasks.truncate_database() - def run_force_upgrade_mysql(self): + def run_force_upgrade_mysql(self) -> None: """:key Do upgrade of mysql, with data dump. """ self._mysql_tasks.install_mysql(force=True) - def run_developer_update(self): + def run_upgrade_mysql(self) -> None: + """:key + Do upgrade of mysql with no table recreate. + """ + self._mysql_tasks.install_mysql(force=False) + + def run_update_calibrations_repository(self) -> None: + """update_calibrations_repository""" + self._server_tasks.update_calibrations_repository() + + def run_setup_log_rotation(self) -> None: + """setup_log_rotation""" + self._server_tasks.setup_log_rotation() + + def run_developer_update(self) -> None: """Update all the developer tools to latest version""" self._mysql_tasks.install_mysql(force=False) self._system_tasks.check_java_installation() self._system_tasks.install_or_upgrade_git() - def run_vhd_creation(self): + def run_vhd_creation(self) -> None: """Automated job which creates a set of VHDs containing all IBEX components. Note: this will run under jenkins, don't add interactive tasks to this list. @@ -331,21 +349,23 @@ def run_vhd_creation(self): self._vhd_tasks.deploy_vhds() - def mount_vhds(self): + def mount_vhds(self) -> None: """Task which actually mounts the VHDs (will be run as admin)""" self._vhd_tasks.mount_vhds() - def dismount_vhds(self): + def dismount_vhds(self) -> None: """Task which actually dismounts the VHDs (will be run as admin)""" self._vhd_tasks.dismount_vhds() - def request_dismount_vhds(self): + def request_dismount_vhds(self) -> None: """Standalone task to request VHDs to be dismounted""" self._vhd_tasks.request_dismount_vhds() - def run_vhd_post_install(self): - """This job is run by the MDT build system when it has built a windows image and mounted the VHDS - It will tidy up and remaining jobs that were not possible when the vdh was created e.g. register mysql service + def run_vhd_post_install(self) -> None: + """This job is run by the MDT build system when it has built + a windows image and mounted the VHDS + It will tidy up and remaining jobs that were not possible when + the vdh was created e.g. register mysql service """ # self._server_tasks.update_icp(self.icp_in_labview_modules()) self._mysql_tasks.configure_mysql_for_vhd_post_install() @@ -388,7 +408,19 @@ def run_vhd_post_install(self): ), "force_upgrade_mysql": ( UpgradeInstrument.run_force_upgrade_mysql, - "upgrade mysql version to latest", + "upgrade mysql version to latest and recreate tables", + ), + "upgrade_mysql": ( + UpgradeInstrument.run_upgrade_mysql, + "upgrade mysql version to latest but do not recreate tables", + ), + "update_calibrations_repository": ( + UpgradeInstrument.run_update_calibrations_repository, + "update calibrations repository", + ), + "setup_log_rotation": ( + UpgradeInstrument.run_setup_log_rotation, + "setup log rotation", ), "developer_update": (UpgradeInstrument.run_developer_update, "install latest developer tools"), "create_vhds": ( diff --git a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py index 382d266..7ee7759 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/git_tasks.py @@ -32,7 +32,7 @@ def checkout_to_release_branch(self) -> None: try: # assumes the alias 'origin' does not exist yet subprocess.check_call( - f"cd {EPICS_PATH} && git remote add origin http://control-svcs.isis.cclrc.ac.uk/gitroot/releases/{version}/{remote_repo}", + f"cd /d {EPICS_PATH} && git remote add origin http://control-svcs.isis.cclrc.ac.uk/gitroot/releases/{version}/{remote_repo}", shell=True, ) print("Added the remote") @@ -40,22 +40,24 @@ def checkout_to_release_branch(self) -> None: print(f"Error creating remote: {e}") try: - subprocess.check_call(f"cd {EPICS_PATH} && git fetch", shell=True) + subprocess.check_call(f"cd /d {EPICS_PATH} && git fetch", shell=True) print("Fetched remote") except subprocess.CalledProcessError as e: print(f"Error fetching remote: {e}") try: # run a git status to rebuild index if needed - subprocess.check_call(f"cd {EPICS_PATH} && git status", shell=True) + subprocess.check_call(f"cd /d {EPICS_PATH} && git status", shell=True) except subprocess.CalledProcessError as e: print(f"Error running git status: {e}") try: - subprocess.check_call(f"cd {EPICS_PATH} && git checkout -b %COMPUTERNAME%", shell=True) + subprocess.check_call( + f"cd /d {EPICS_PATH} && git checkout -b %COMPUTERNAME%", shell=True + ) print("Checked out to the new release branch") subprocess.check_call( - f"cd {EPICS_PATH} && git push -u origin %COMPUTERNAME%", shell=True + f"cd /d {EPICS_PATH} && git push -u origin %COMPUTERNAME%", shell=True ) print("Pushed to the remote") except subprocess.CalledProcessError as e: diff --git a/installation_and_upgrade/ibex_install_utils/tasks/mysql_tasks.py b/installation_and_upgrade/ibex_install_utils/tasks/mysql_tasks.py index 884e915..b816f2a 100644 --- a/installation_and_upgrade/ibex_install_utils/tasks/mysql_tasks.py +++ b/installation_and_upgrade/ibex_install_utils/tasks/mysql_tasks.py @@ -35,7 +35,7 @@ MYSQL8_INSTALL_DIR = os.path.join(APPS_BASE_DIR, "MySQL") MYSQL57_INSTALL_DIR = os.path.join("C:\\", "Program Files", "MySQL", "MySQL Server 5.7") -MYSQL_LATEST_VERSION = "8.0.32" +MYSQL_LATEST_VERSION = "8.0.39" MYSQL_ZIP = os.path.join( INST_SHARE_AREA, "kits$", @@ -64,7 +64,7 @@ class MysqlTasks(BaseTasks): Tasks relating to installing or maintaining an installation of MySQL on a machine. """ - def _get_mysql_dir(self): + def _get_mysql_dir(self) -> str: """ Returns the mysql 8 default install directory if it exists, else 5.7. @@ -76,7 +76,7 @@ def _get_mysql_dir(self): return mysql_bin_dir - def _get_mysql_backup_dir(self): + def _get_mysql_backup_dir(self) -> str: mysql_backup_dir = os.path.join( STAGE_DELETED, self._get_machine_name(), @@ -87,14 +87,17 @@ def _get_mysql_backup_dir(self): return mysql_backup_dir @task("Truncate database") - def truncate_database(self): + def truncate_database(self) -> None: """ Truncate the message log, sample and alarms tables """ try: mysql_bin_dir = self._get_mysql_dir() - sql_command = "truncate table msg_log.message;truncate table archive.sample;truncate table alarm.pv" + sql_command = ( + "truncate table msg_log.message;" + "truncate table archive.sample;truncate table alarm.pv" + ) RunProcess( MYSQL_FILES_DIR, @@ -110,7 +113,7 @@ def truncate_database(self): f"Error is {ex}" ) - def _configure_mysql(self): + def _configure_mysql(self) -> None: """ Copy mysql settings and run the MySQL configuration script """ @@ -131,7 +134,7 @@ def _configure_mysql(self): admin_commands.add_command("sc", "start MYSQL80", expected_return_val=None) admin_commands.run_all() - def _remove_old_versions_of_mysql8(self, clean_install): + def _remove_old_versions_of_mysql8(self, clean_install: bool) -> None: if clean_install: self.prompt.prompt_and_raise_if_not_yes( "Warning: this will erase all data held in the MySQL database. " @@ -151,11 +154,11 @@ def _remove_old_versions_of_mysql8(self, clean_install): if clean_install: self._remove_old_mysql_data_dir() - def _remove_old_mysql_data_dir(self): + def _remove_old_mysql_data_dir(self) -> None: if os.path.exists(MYSQL_FILES_DIR): shutil.rmtree(MYSQL_FILES_DIR) - def _create_mysql_binaries(self): + def _create_mysql_binaries(self) -> None: os.makedirs(MYSQL8_INSTALL_DIR) mysql_unzip_temp = os.path.join(APPS_BASE_DIR, "temp-mysql-unzip") @@ -171,7 +174,7 @@ def _create_mysql_binaries(self): shutil.rmtree(mysql_unzip_temp) - def _initialize_mysql_data_area_for_vhd(self): + def _initialize_mysql_data_area_for_vhd(self) -> None: os.makedirs(os.path.join(MYSQL_FILES_DIR, "data")) RunProcess( @@ -187,7 +190,7 @@ def _initialize_mysql_data_area_for_vhd(self): ).run() @contextlib.contextmanager - def temporarily_run_mysql(self, sql_password: str): + def temporarily_run_mysql(self, sql_password: str) -> None: mysqld = os.path.join(MYSQL8_INSTALL_DIR, "bin", "mysqld.exe") # spawn service in background @@ -211,7 +214,7 @@ def temporarily_run_mysql(self, sql_password: str): log_command_args=False, # To make sure password doesn't appear in jenkins log. ).run() - def _setup_database_users_and_tables(self, vhd_install=True): + def _setup_database_users_and_tables(self, vhd_install: bool = True) -> None: sql_password = self.prompt.prompt( "Enter the MySQL root password:", UserPrompt.ANY, @@ -235,8 +238,9 @@ def _setup_database_users_and_tables(self, vhd_install=True): "-u", "root", "-e", - f"ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '{sql_password}';FLUSH " - f"privileges; ", + "ALTER USER 'root'@'localhost' " + f"IDENTIFIED WITH mysql_native_password BY '{sql_password}';FLUSH " + "privileges; ", ], log_command_args=False, # To make sure password doesn't appear in jenkins log. ).run() @@ -249,7 +253,7 @@ def _setup_database_users_and_tables(self, vhd_install=True): log_command_args=False, # To make sure password doesn't appear in jenkins log. ).run() - def _setup_mysql8_service(self): + def _setup_mysql8_service(self) -> None: mysqld = os.path.join(MYSQL8_INSTALL_DIR, "bin", "mysqld.exe") admin_commands = AdminCommandBuilder() @@ -260,8 +264,8 @@ def _setup_mysql8_service(self): ) admin_commands.add_command("sc", "start MYSQL80", expected_return_val=None) - # we use "delayed-auto" for start= as we have some ibex installations where a required disk volume - # doesn't get mounted in time if just "auto" is used + # we use "delayed-auto" for start= as we have some ibex installations + # where a required disk volume doesn't get mounted in time if just "auto" is used admin_commands.add_command("sc", "config MYSQL80 start= delayed-auto") admin_commands.add_command( "sc", "failure MYSQL80 reset= 900 actions= restart/10000/restart/30000/restart/60000" @@ -281,10 +285,10 @@ def _setup_mysql8_service(self): admin_commands.run_all() - def _install_latest_mysql8(self, clean_install): + def _install_latest_mysql8(self, clean_install: bool) -> None: """ - Install the latest mysql. If this is a clean install remove old data directories first and create a new - database + Install the latest mysql. If this is a clean install remove old data + directories first and create a new database Args: clean_install: True to destroy and recreate data directories """ @@ -311,12 +315,14 @@ def _install_latest_mysql8(self, clean_install): self._setup_database_users_and_tables(vhd_install=False) @task("Install latest MySQL for VHD deployment") - def install_mysql_for_vhd(self): + def install_mysql_for_vhd(self) -> None: """ Installs MySQL for the VHD creation build. - Ensure we start from a clean slate. We are creating VHDs so we can assume that no files should exist in - C:\instrument\apps\mysql or c:\instrument\var\mysql and delete them if they do exist. This facilitates + Ensure we start from a clean slate. We are creating VHDs so + we can assume that no files should exist in + C:\instrument\apps\mysql or c:\instrument\var\mysql and + delete them if they do exist. This facilitates developer testing/resuming the script if it failed halfway through """ for path in [MYSQL_FILES_DIR, MYSQL8_INSTALL_DIR]: @@ -338,7 +344,7 @@ def install_mysql_for_vhd(self): self._setup_database_users_and_tables(vhd_install=True) - def _install_vcruntime140(self): + def _install_vcruntime140(self) -> None: if not os.path.exists(VCRUNTIME140): self.prompt.prompt_and_raise_if_not_yes( f"MySQL server 8 requires microsoft visual C++ runtime to be installed.\r\n" @@ -347,7 +353,7 @@ def _install_vcruntime140(self): @version_check(MySQL()) @task("Install latest MySQL") - def install_mysql(self, force=False): + def install_mysql(self, force: bool = False) -> None: """ Install mysql and the ibex database schemas Args: @@ -390,7 +396,7 @@ def install_mysql(self, force=False): self._reload_backup_data() @task("Configure MySQL for VHD post install") - def configure_mysql_for_vhd_post_install(self): + def configure_mysql_for_vhd_post_install(self) -> None: """ configure mysql after vhd is deployed to an instrukent/mdt build """ @@ -399,7 +405,7 @@ def configure_mysql_for_vhd_post_install(self): sleep(5) # Time for service to start @task("Backup database") - def backup_database(self): + def backup_database(self) -> None: """ Backup the database """ @@ -447,7 +453,7 @@ def backup_database(self): f"Dump file '{result_file}' seems to be small is it correct? " ) - def _backup_data(self): + def _backup_data(self) -> None: """ Backup the data for transfer. This dumps just the data not the schema. """ @@ -487,7 +493,7 @@ def _backup_data(self): f"Dump file '{result_file}' seems to be small is it correct? " ) - def _reload_backup_data(self): + def _reload_backup_data(self) -> None: """ Reload backup the data """ @@ -507,6 +513,7 @@ def _reload_backup_data(self): std_in=open(result_file), ).run() self.prompt.prompt_and_raise_if_not_yes( - "Check that there are only 16 errors from mysql. These are 1062 error fail to insert primary key. " + "Check that there are only 16 errors from mysql. " + "These are 1062 error fail to insert primary key. " "These are for constants added by the creation script, e.g. archive severity." ) diff --git a/installation_and_upgrade/summer_update.bat b/installation_and_upgrade/summer_update.bat new file mode 100644 index 0000000..638ee77 --- /dev/null +++ b/installation_and_upgrade/summer_update.bat @@ -0,0 +1,29 @@ +setlocal EnableDelayedExpansion + +set "SOURCE=\\isis.cclrc.ac.uk\inst$\Kits$\CompGroup\ICP\Releases" +call "%~dp0\define_latest_genie_python.bat" +IF %errorlevel% neq 0 EXIT /b %errorlevel% + +git --version + +IF %errorlevel% neq 0 ( + echo No installation of Git found on machine. Please download Git from https://git-scm.com/downloads before proceeding. + EXIT /b %errorlevel% +) + +xcopy /y %SOURCE%\15.0.0\EPICS\utils\logrotate.py c:\instrument\apps\epics\utils +IF %errorlevel% neq 0 goto ERROR + +for %%t in ( truncate_database upgrade_mysql update_calibrations_repository setup_log_rotation ) do ( + call "%LATEST_PYTHON%" "%~dp0IBEX_upgrade.py" --release_dir "%SOURCE%" --release_suffix "%SUFFIX%" --confirm_step %%t + IF !errorlevel! neq 0 goto ERROR +) + +echo Finished +pause +exit /b 0 + +:ERROR +echo ERROR - see messages above and install log +pause +exit /b 1