Merge branch 'devel'

IM/InfrastructureInfo.py

@@ -362,6 +362,7 @@ def Contextualize(self, auth):
 
 		for vm in self.get_vm_list():
 			vm.cont_out = "" 
+			vm.configured = None
 			tasks = {}
 
 			tasks[0] = ['basic']
@@ -377,9 +378,6 @@ def Contextualize(self, auth):
 
 			for step in tasks.keys():
 				priority = 0
-				# Set more priority to the new VMs to launch the ctxt process first in them
-				if vm.configured is None:
-					priority = -1
 				ctxt_task.append((step,priority,vm,tasks[step]))
 
 		self.add_ctxt_tasks(ctxt_task)

IM/InfrastructureManager.py

@@ -368,7 +368,7 @@ def AddResource(inf_id, radl_data, auth, context = True, failed_clouds = []):
 		# If any deploy is defined, only update definitions.
 		if not radl.deploys:
 			sel_inf.update_radl(radl, [])
-			InfrastructureManager.logger.debug("Infrastructure without any deploy. Exiting.")
+			InfrastructureManager.logger.warn("Infrastructure without any deploy. Exiting.")
 			return []
 
 		for system in radl.systems:

IM/VirtualMachine.py

@@ -23,6 +23,7 @@
 import string
 import json
 import tempfile
+import logging
 
 
 class VirtualMachine:
@@ -38,6 +39,8 @@ class VirtualMachine:
 	UNCONFIGURED = "unconfigured"
 
 	WAIT_TO_PID = "WAIT"
+
+	logger = logging.getLogger('InfrastructureManager')
 
 	def __init__(self, inf, cloud_id, cloud, info, requested_radl, cloud_connector = None):
 		self._lock = threading.Lock()
@@ -57,6 +60,9 @@ def __init__(self, inf, cloud_id, cloud, info, requested_radl, cloud_connector =
 		self.cloud = cloud
 		"""CloudInfo object with the information about the cloud provider"""
 		self.info = info.clone() if info else None
+		# Set the initial state of the VM
+		if info:
+			self.info.systems[0].setValue("state", self.state)
 		"""RADL object with the current information about the VM"""
 		self.requested_radl = requested_radl
 		"""Original RADL requested by the user"""
@@ -382,24 +388,35 @@ def update_status(self, auth):
 		if now - self.last_update > Config.VM_INFO_UPDATE_FREQUENCY:
 			if not self.cloud_connector:
 				self.cloud_connector = self.cloud.getCloudConnector()
-			(success, new_vm) = self.cloud_connector.updateVMInfo(self, auth)
-			if success:
-				state = new_vm.state
-				updated = True
-
-			with self._lock:
-				self.last_update = now
-
-		if state != VirtualMachine.RUNNING:
-			new_state = state
-		elif self.is_configured() is None:
-			new_state = VirtualMachine.RUNNING
-		elif self.is_configured():
-			new_state = VirtualMachine.CONFIGURED
+
+			try:
+				(success, new_vm) = self.cloud_connector.updateVMInfo(self, auth)
+				if success:
+					state = new_vm.state
+					updated = True
+
+				with self._lock:
+					self.last_update = now
+			except:
+				VirtualMachine.logger.exception("Error updating VM status.")
+				updated = False
+
+		# If we have problems to update the VM info too much time, set to unknown
+		if now - self.last_update > Config.VM_INFO_UPDATE_ERROR_GRACE_PERIOD:
+			new_state = VirtualMachine.UNKNOWN
+			VirtualMachine.logger.WARN("Grace period to update VM info passed. Set state to 'unknown'")
 		else:
-			new_state = VirtualMachine.UNCONFIGURED
-
+			if state not in [VirtualMachine.RUNNING, VirtualMachine.CONFIGURED, VirtualMachine.UNCONFIGURED]:
+				new_state = state
+			elif self.is_configured() is None:
+				new_state = VirtualMachine.RUNNING
+			elif self.is_configured():
+				new_state = VirtualMachine.CONFIGURED
+			else:
+				new_state = VirtualMachine.UNCONFIGURED
+
 		with self._lock:
+			self.state = new_state
 			self.info.systems[0].setValue("state", new_state)
 
 		return updated
@@ -513,6 +530,7 @@ def check_ctxt_process(self):
 					try:
 						ssh.execute("kill -9 " + str(self.ctxt_pid))
 					except:
+						VirtualMachine.logger.exception("Error killing ctxt process with pid: " + str(self.ctxt_pid))
 						pass
 
 					self.ctxt_pid = None
@@ -521,9 +539,11 @@ def check_ctxt_process(self):
 					try:
 						(_, _, exit_status) = ssh.execute("ps " + str(self.ctxt_pid))
 					except:
+						VirtualMachine.logger.warn("Error getting status of ctxt process with pid: " + str(self.ctxt_pid))
 						exit_status = 0
 						self.ssh_connect_errors += 1
 						if self.ssh_connect_errors > Config.MAX_SSH_ERRORS:
+							VirtualMachine.logger.error("Too much errors getting status of ctxt process with pid: " + str(self.ctxt_pid) + ". Forget it.")
 							self.ssh_connect_errors = 0
 							self.ctxt_pid = None
 							self.configured = False
@@ -568,6 +588,7 @@ def get_ctxt_output(self, remote_dir):
 
 			ssh.execute("rm -rf " + remote_dir + '/ctxt_agent.log')
 		except Exception, ex:
+			VirtualMachine.logger.exception("Error getting contextualization process output")
 			self.configured = False
 			self.cont_out += "Error getting contextualization process output: " + str(ex)
 
@@ -579,6 +600,7 @@ def get_ctxt_output(self, remote_dir):
 			# And process it
 			self.process_ctxt_agent_out(ctxt_agent_out)
 		except Exception, ex:
+			VirtualMachine.logger.exception("Error getting contextualization agent output")
 			self.configured = False
 			self.cont_out += "Error getting contextualization agent output: "  + str(ex)
 		finally:

IM/__init__.py

@@ -16,6 +16,6 @@
 
 
 __all__ = ['auth','bottle','CloudManager','config','ConfManager','db','ganglia','HTTPHeaderTransport','ImageManager','InfrastructureInfo','InfrastructureManager','parsetab','radl','recipe','request','REST', 'ServiceRequests','SSH','timedcall','uriparse','VMRC','xmlobject']
-__version__ = '1.2.2'
+__version__ = '1.2.3'
 __author__ = 'Miguel Caballer'
 

IM/ansible/ansible_launcher.py

@@ -55,13 +55,10 @@ def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retrie
     options, _ = parser.parse_args([])
 
     sshpass = None
-    sudopass = None
-    options.sudo_user = options.sudo_user or C.DEFAULT_SUDO_USER
     if pk_file:
         options.private_key_file = pk_file
     else:
         sshpass = passwd
-        sudopass = passwd
 
     if user:
         remote_user=user
@@ -105,17 +102,9 @@ def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retrie
             callbacks=playbook_cb,
             runner_callbacks=runner_cb,
             stats=stats,
-            timeout=options.timeout,
-            transport=options.connection,
-            sudo=options.sudo,
-            sudo_user=options.sudo_user,
-            sudo_pass=sudopass,
             extra_vars=extra_vars,
             private_key_file=options.private_key_file,
-            only_tags=['all'],
-            skip_tags=None,
-            check=False,
-            diff=options.diff
+            only_tags=['all']
         )
 
         try:

IM/config.py

@@ -72,6 +72,7 @@ class Config:
 	GANGLIA_INFO_UPDATE_FREQUENCY = 30
 	PLAYBOOK_RETRIES = 1
 	VM_INFO_UPDATE_FREQUENCY = 10
+	VM_INFO_UPDATE_ERROR_GRACE_PERIOD = 120 # This value must be always higher than VM_INFO_UPDATE_FREQUENCY 
 	REMOTE_CONF_DIR = "/tmp/.im"
 	MAX_SSH_ERRORS = 5
 	PRIVATE_NET_AS_PUBLIC = ''

IM/radl/radl_parse.py

@@ -36,7 +36,7 @@ def __init__(self, autodefinevars = True, **kwargs):
 		'RPAREN',
 		'NUMBER',
 		'AND',
-		'OR',
+#		'OR',
 		'EQ',
 		'LT',
 		'GT',
@@ -117,7 +117,7 @@ def t_STRING(self, t):
 		'system' : 'SYSTEM',
 		'soft' : 'SOFT',
 		'and' : 'AND',
-		'or' : 'OR',
+#		'or' : 'OR',
 		'contains' : 'CONTAINS',
 		'deploy' : 'DEPLOY',
 		'configure': 'CONFIGURE',

README

@@ -33,7 +33,7 @@ However, if you install IM from sources you should install:
  * The Python Lex & Yacc library (http://www.dabeaz.com/ply/), typically available
    as the 'python-ply' package.
 
- * The paramiko ssh2 protocol library for python
+ * The paramiko ssh2 protocol library for python version 1.14 or later
 (http://www.lag.net/paramiko/), typically available as the 'python-paramiko' package.
 
  * The YAML library for Python, typically available as the 'python-yaml' or 'PyYAML' package.
@@ -44,27 +44,31 @@ However, if you install IM from sources you should install:
    In particular, Ansible 1.4.2+ must be installed.
    To ensure the functionality the following values must be set in the ansible.cfg file:
 
-		[default]
+		[defaults]
+		transport  = smart
 		host_key_checking = False
-		transport = smart
+		sudo_user = root
+		sudo_exe = sudo
 
 		[paramiko_connection]
-		record_host_keys = False
-
+
+		record_host_keys=False
+
 		[ssh_connection]
-		pipelining=True
-		# Only in systems with OpenSSH support to ControlPersist 
+
+		# Only in systems with OpenSSH support to ControlPersist
 		ssh_args = -o ControlMaster=auto -o ControlPersist=900s
 		# In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) 
-		ssh_args =
+		#ssh_args =
+		pipelining = True
 
 1.2 OPTIONAL PACKAGES
 ---------------------
 
-In case of using the Amazon EC2 plugin the boto library version 2.0 or later
+In case of using the Amazon EC2 plugin the boto library version 2.29 or later
 must be installed (http://boto.readthedocs.org/en/latest/).
 
-In case of using the LibCloud plugin the apache-libcloud library version 0.15 or later
+In case of using the LibCloud plugin the apache-libcloud library version 0.17 or later
 must be installed (http://libcloud.apache.org/).
 
 In case of using the SSL secured version of the XMLRPC API the SpringPython
@@ -82,16 +86,20 @@ framework (http://www.cherrypy.org/) must be installed.
 1.3.1 FROM PIP
 --------------
 
+**WARNING**: The SOAPpy distributed with pip does not work correctly so you must install
+the packages 'python-soappy' or 'SOAPp'y before installing the IM with pip.
+
 **WARNING**: In some GNU/Linux distributions (RHEL 6 or equivalents) you must uninstall
-the packages python-paramiko and python-crypto before installing the IM with pip.**
+the packages python-paramiko and python-crypto before installing the IM with pip.
 
 You only have to install the IM package through the pip tool.
 
 	pip install IM
 
-Pip will install all the pre-requisites needed. So Ansible  1.4.2 or later will
-be installed in the system. In some cases it will need to have installed the GCC
-compiler and the python developer libraries ('python-dev' or 'python-devel'
+Pip will install all the pre-requisites needed. So Ansible  1.4.2 or later will 
+be installed in the system. Yo will also need to install the sshpass command 
+('sshpass' package in main distributions). In some cases it will need to have installed  
+the GCC compiler and the python developer libraries ('python-dev' or 'python-devel' 
 packages in main distributions).
 
 You must also remember to modify the ansible.cfg file setting as specified in the

README.md

@@ -1,8 +1,8 @@
  IM - Infrastructure Manager
 ============================
 
-* Version [![PyPI](https://img.shields.io/pypi/v/im.svg)](https://img.shields.io/pypi/v/im.svg)
-* PyPI [![PypI](https://img.shields.io/pypi/dm/IM.svg)](https://img.shields.io/pypi/dm/IM.svg)
+* Version ![PyPI](https://img.shields.io/pypi/v/im.svg)
+* PyPI ![PypI](https://img.shields.io/pypi/dm/IM.svg)
 
 IM is a tool that deploys complex and customized virtual infrastructures on IaaS
 Cloud deployments (such as AWS, OpenStack, etc.). It eases the access and the
@@ -35,7 +35,7 @@ However, if you install IM from sources you should install:
  + The Python Lex & Yacc library (http://www.dabeaz.com/ply/), typically available
    as the 'python-ply' package.
 
- + The paramiko ssh2 protocol library for python
+ + The paramiko ssh2 protocol library for python version 1.14 or later
 (http://www.lag.net/paramiko/), typically available as the 'python-paramiko' package.
 
  + The YAML library for Python, typically available as the 'python-yaml' or 'PyYAML' package.
@@ -47,28 +47,32 @@ However, if you install IM from sources you should install:
    To ensure the functionality the following values must be set in the ansible.cfg file:
 
 ```
-[default]
+[defaults]
+transport  = smart
 host_key_checking = False
-transport = smart
+sudo_user = root
+sudo_exe = sudo
 
 [paramiko_connection]
-record_host_keys = False
-   
+
+record_host_keys=False
+
 [ssh_connection]
-pipelining=True
-# Only in systems with OpenSSH support to ControlPersist 
+
+# Only in systems with OpenSSH support to ControlPersist
 ssh_args = -o ControlMaster=auto -o ControlPersist=900s
 # In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) 
-ssh_args =
+#ssh_args =
+pipelining = True
 ```
 
 1.2 OPTIONAL PACKAGES
 ---------------------
 
-In case of using the Amazon EC2 plugin the boto library version 2.0 or later
+In case of using the Amazon EC2 plugin the boto library version 2.29 or later
 must be installed (http://boto.readthedocs.org/en/latest/).
 
-In case of using the LibCloud plugin the apache-libcloud library version 0.15 or later
+In case of using the LibCloud plugin the apache-libcloud library version 0.17 or later
 must be installed (http://libcloud.apache.org/).
 
 In case of using the SSL secured version of the XMLRPC API the SpringPython
@@ -85,8 +89,11 @@ framework (http://www.cherrypy.org/) must be installed.
 
 ### 1.3.1 FROM PIP
 
+**WARNING: The SOAPpy distributed with pip does not work correctly so you must install
+the packages 'python-soappy' or 'SOAPp'y before installing the IM with pip.**
+
 **WARNING: In some GNU/Linux distributions (RHEL 6 or equivalents) you must uninstall
-the packages python-paramiko and python-crypto before installing the IM with pip.**
+the packages 'python-paramiko' and 'python-crypto' before installing the IM with pip.**
 
 You only have to install the IM package through the pip tool.
 
@@ -95,8 +102,9 @@ pip install IM
 ```
 
 Pip will install all the pre-requisites needed. So Ansible  1.4.2 or later will
- be installed in the system. In some cases it will need to have installed the GCC
- compiler and the python developer libraries ('python-dev' or 'python-devel'
+ be installed in the system. Yo will also need to install the sshpass command
+ ('sshpass' package in main distributions). In some cases it will need to have installed 
+ the GCC compiler and the python developer libraries ('python-dev' or 'python-devel'
  packages in main distributions).
 
 You must also remember to modify the ansible.cfg file setting as specified in the

changelog

@@ -99,3 +99,9 @@ IM 1.2.2
     * Improve contextualization performance
     * Bugfix in the Ansible installation playbook
     * Change Ansible version to 1.8.4
+
+IM 1.2.3
+    * Bugfix in the Ansible launcher with versions 1.9.X
+    * Bugfix in VirtualMachine update_status function
+    * Add the VM_INFO_UPDATE_ERROR_GRACE_PERIOD to manage errors in the conections with Cloud providers
+    * Bugfix and code improvements in GCE connector