Create the SSHRetry class

grycap · Jun 12, 2015 · 56c7ac9 · 56c7ac9
1 parent ce75d1e
commit 56c7ac9
Show file tree

Hide file tree

Showing 9 changed files with 180 additions and 3,667 deletions.
diff --git a/IM/ConfManager.py b/IM/ConfManager.py
@@ -224,55 +224,52 @@ def run(self):
 
 				last_step = step
 
-	def launch_ctxt_agent(self, vm, tasks, max_retries = 3):
+	def launch_ctxt_agent(self, vm, tasks):
 		"""
 		Launch the ctxt agent to configure the specified tasks in the specified VM
 		"""
 		pid = None
-		retries = 0
-		while not pid and retries < max_retries:
-			retries += 1
-			try:
-				ip = vm.getPublicIP()
-				if not ip:
-					ip = vm.getPrivateIP()
-				remote_dir = Config.REMOTE_CONF_DIR + "/" + ip + "_" + str(vm.getSSHPort())
-				tmp_dir = tempfile.mkdtemp()
-
-				ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": Create the configuration file for the contextualization agent")
-				conf_file = tmp_dir + "/config.cfg"
-				self.create_vm_conf_file(conf_file, vm.im_id, tasks, remote_dir)
-
-				ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": Copy the contextualization agent config file")
-
-				# Copy the contextualization agent config file
-				ssh = self.inf.vm_master.get_ssh()
-				ssh.sftp_mkdir(remote_dir)
-				ssh.sftp_put(conf_file, remote_dir + "/" + os.path.basename(conf_file))
-
-				shutil.rmtree(tmp_dir, ignore_errors=True)
-
-				(pid, _, _) = ssh.execute("nohup python_ansible " + Config.REMOTE_CONF_DIR + "/ctxt_agent.py " 
-						+ Config.REMOTE_CONF_DIR + "/general_info.cfg "
-						+ remote_dir + "/" + os.path.basename(conf_file) 
-						+ " > " + remote_dir + "/stdout" + " 2> " + remote_dir + "/stderr < /dev/null & echo -n $!")
-
-				ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": Ansible process to configure " + str(vm.im_id) + " launched with pid: " + pid)
-
-				vm.ctxt_pid = pid
-				vm.launch_check_ctxt_process()
-			except:
-				pid = None
-				ConfManager.logger.exception("Inf ID: " + str(self.inf.id) + ": Error (%d/%d) launching the ansible process to configure %s" % (retries, max_retries, str(vm.im_id)))
-				time.sleep(retries*2)
 
-			# If the process is not correctly launched the configuration of this VM fails
-			if pid is None:
-				vm.ctxt_pid = None
-				vm.configured = False
-				vm.cont_out = "Error launching the contextualization agent to configure the VM. Check the SSH connection."
+		try:
+			ip = vm.getPublicIP()
+			if not ip:
+				ip = vm.getPrivateIP()
+			remote_dir = Config.REMOTE_CONF_DIR + "/" + ip + "_" + str(vm.getSSHPort())
+			tmp_dir = tempfile.mkdtemp()
+
+			ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": Create the configuration file for the contextualization agent")
+			conf_file = tmp_dir + "/config.cfg"
+			self.create_vm_conf_file(conf_file, vm.im_id, tasks, remote_dir)
+
+			ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": Copy the contextualization agent config file")
+
+			# Copy the contextualization agent config file
+			ssh = self.inf.vm_master.get_ssh(retry = True)
+			ssh.sftp_mkdir(remote_dir)
+			ssh.sftp_put(conf_file, remote_dir + "/" + os.path.basename(conf_file))
+
+			shutil.rmtree(tmp_dir, ignore_errors=True)
+
+			(pid, _, _) = ssh.execute("nohup python_ansible " + Config.REMOTE_CONF_DIR + "/ctxt_agent.py " 
+					+ Config.REMOTE_CONF_DIR + "/general_info.cfg "
+					+ remote_dir + "/" + os.path.basename(conf_file) 
+					+ " > " + remote_dir + "/stdout" + " 2> " + remote_dir + "/stderr < /dev/null & echo -n $!")
+
+			ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": Ansible process to configure " + str(vm.im_id) + " launched with pid: " + pid)
+
+			vm.ctxt_pid = pid
+			vm.launch_check_ctxt_process()
+		except:
+			pid = None
+			ConfManager.logger.exception("Inf ID: " + str(self.inf.id) + ": Error launching the ansible process to configure %s" % str(vm.im_id))
+
+		# If the process is not correctly launched the configuration of this VM fails
+		if pid is None:
+			vm.ctxt_pid = None
+			vm.configured = False
+			vm.cont_out = "Error launching the contextualization agent to configure the VM. Check the SSH connection."
 
-			return pid
+		return pid
 
 	def generate_inventory(self, tmp_dir):
 		"""
@@ -518,7 +515,7 @@ def configure_master(self):
 			try:
 				ConfManager.logger.info("Inf ID: " + str(self.inf.id) + ": Start the contextualization process.")
 
-				ssh = self.inf.vm_master.get_ssh()
+				ssh = self.inf.vm_master.get_ssh(retry=True)
 				# Activate tty mode to avoid some problems with sudo in REL
 				ssh.tty = True
 
@@ -615,7 +612,7 @@ def wait_master(self):
 				ConfManager.logger.info("Inf ID: " + str(self.inf.id) + ": VMs available.")
 
 				# Check and change if necessary the credentials of the master vm
-				ssh = self.inf.vm_master.get_ssh()
+				ssh = self.inf.vm_master.get_ssh(retry=True)
 				# Activate tty mode to avoid some problems with sudo in REL
 				ssh.tty = True
 				self.change_master_credentials(ssh)
@@ -688,7 +685,7 @@ def generate_playbooks_and_hosts(self):
 			# TODO: Study why it is needed
 			time.sleep(2)
 
-			ssh = self.inf.vm_master.get_ssh()
+			ssh = self.inf.vm_master.get_ssh(retry=True)
 			self.inf.add_cont_msg("Copying YAML, hosts and inventory files.")
 			ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": Copying YAML files.")
 			ssh.sftp_mkdir(remote_dir)

diff --git a/IM/SSHRetry.py b/IM/SSHRetry.py
@@ -0,0 +1,73 @@
+#! /usr/bin/env python
+# IM - Infrastructure Manager
+# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from IM.retry import retry
+from IM.SSH import SSH
+
+class SSHRetry(SSH):
+    """ SSH class decorated to perform a number of retries """
+    TRIES = 3
+    DELAY = 3
+    BACKOFF = 2
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def execute(self, command, timeout = None):
+        return SSH.execute(self, command, timeout)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_get(self, src, dest):
+        return SSH.sftp_get(self, src, dest)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_get_files(self, src, dest):
+        return SSH.sftp_get_files(self, src, dest)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_put_files(self, files):
+        return SSH.sftp_put_files(self, files)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_put(self, src, dest):
+        return SSH.sftp_put(self, src, dest)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_put_dir(self, src, dest):
+        return SSH.sftp_put_dir(self, src, dest)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_put_content(self, content, dest):
+        return SSH.sftp_put_content(self, content, dest)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_mkdir(self, directory):
+        return SSH.sftp_mkdir(self, directory)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_list(self, directory):
+        return SSH.sftp_list(self, directory)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_list_attr(self, directory):
+        return SSH.sftp_list_attr(self, directory)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def getcwd(self):
+        return SSH.getcwd(self)
+
+    @retry(Exception, tries=TRIES, delay=DELAY, backoff=BACKOFF)
+    def sftp_remove(self, path):
+        return SSH.sftp_remove(self, path)
diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py
@@ -18,6 +18,7 @@
 import threading
 from IM.radl.radl import network, RADL
 from IM.SSH import SSH
+from IM.SSHRetry import SSHRetry
 from config import Config
 import shutil
 import string
@@ -497,7 +498,7 @@ def setIps(self,public_ips,private_ips):
 				vm_system.setValue('net_interface.' + str(num_net) + '.ip', str(private_ip))
 				vm_system.setValue('net_interface.' + str(num_net) + '.connection',private_net.id)
 
-	def get_ssh(self):
+	def get_ssh(self, retry = False):
 		"""
 		Get SSH object to connect with this VM
 		"""
@@ -507,7 +508,10 @@ def get_ssh(self):
 			ip = self.getPrivateIP()
 		if ip == None:
 			return None
-		return SSH(ip, user, passwd, private_key, self.getSSHPort())
+		if retry:
+			return SSHRetry(ip, user, passwd, private_key, self.getSSHPort())
+		else:
+			return SSH(ip, user, passwd, private_key, self.getSSHPort())
 
 	def is_ctxt_process_running(self):
 		""" Return the PID of the running process or None if it is not running """
@@ -599,7 +603,7 @@ def is_configured(self):
 				return self.configured
 
 	def get_ctxt_log(self, remote_dir, delete = False):
-		ssh = self.inf.vm_master.get_ssh()
+		ssh = self.inf.vm_master.get_ssh(retry=True)
 		tmp_dir = tempfile.mkdtemp()
 		conf_out = ""
 
@@ -622,7 +626,7 @@ def get_ctxt_log(self, remote_dir, delete = False):
 		return conf_out
 
 	def get_ctxt_output(self, remote_dir, delete = False):
-		ssh = self.inf.vm_master.get_ssh()
+		ssh = self.inf.vm_master.get_ssh(retry=True)
 		tmp_dir = tempfile.mkdtemp()
 
 		# Download the contextualization agent log