diff --git a/IM/InfrastructureInfo.py b/IM/InfrastructureInfo.py index bab428012..d27aa9684 100644 --- a/IM/InfrastructureInfo.py +++ b/IM/InfrastructureInfo.py @@ -362,6 +362,7 @@ def Contextualize(self, auth): for vm in self.get_vm_list(): vm.cont_out = "" + vm.configured = None tasks = {} tasks[0] = ['basic'] @@ -377,9 +378,6 @@ def Contextualize(self, auth): for step in tasks.keys(): priority = 0 - # Set more priority to the new VMs to launch the ctxt process first in them - if vm.configured is None: - priority = -1 ctxt_task.append((step,priority,vm,tasks[step])) self.add_ctxt_tasks(ctxt_task) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index 3a7234c12..c05acacf4 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -368,7 +368,7 @@ def AddResource(inf_id, radl_data, auth, context = True, failed_clouds = []): # If any deploy is defined, only update definitions. if not radl.deploys: sel_inf.update_radl(radl, []) - InfrastructureManager.logger.debug("Infrastructure without any deploy. Exiting.") + InfrastructureManager.logger.warn("Infrastructure without any deploy. Exiting.") return [] for system in radl.systems: diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index beea4cb7b..4848d1ba6 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -23,6 +23,7 @@ import string import json import tempfile +import logging class VirtualMachine: @@ -38,6 +39,8 @@ class VirtualMachine: UNCONFIGURED = "unconfigured" WAIT_TO_PID = "WAIT" + + logger = logging.getLogger('InfrastructureManager') def __init__(self, inf, cloud_id, cloud, info, requested_radl, cloud_connector = None): self._lock = threading.Lock() @@ -57,6 +60,9 @@ def __init__(self, inf, cloud_id, cloud, info, requested_radl, cloud_connector = self.cloud = cloud """CloudInfo object with the information about the cloud provider""" self.info = info.clone() if info else None + # Set the initial state of the VM + if info: + self.info.systems[0].setValue("state", self.state) """RADL object with the current information about the VM""" self.requested_radl = requested_radl """Original RADL requested by the user""" @@ -382,24 +388,35 @@ def update_status(self, auth): if now - self.last_update > Config.VM_INFO_UPDATE_FREQUENCY: if not self.cloud_connector: self.cloud_connector = self.cloud.getCloudConnector() - (success, new_vm) = self.cloud_connector.updateVMInfo(self, auth) - if success: - state = new_vm.state - updated = True - - with self._lock: - self.last_update = now - - if state != VirtualMachine.RUNNING: - new_state = state - elif self.is_configured() is None: - new_state = VirtualMachine.RUNNING - elif self.is_configured(): - new_state = VirtualMachine.CONFIGURED + + try: + (success, new_vm) = self.cloud_connector.updateVMInfo(self, auth) + if success: + state = new_vm.state + updated = True + + with self._lock: + self.last_update = now + except: + VirtualMachine.logger.exception("Error updating VM status.") + updated = False + + # If we have problems to update the VM info too much time, set to unknown + if now - self.last_update > Config.VM_INFO_UPDATE_ERROR_GRACE_PERIOD: + new_state = VirtualMachine.UNKNOWN + VirtualMachine.logger.WARN("Grace period to update VM info passed. Set state to 'unknown'") else: - new_state = VirtualMachine.UNCONFIGURED - + if state not in [VirtualMachine.RUNNING, VirtualMachine.CONFIGURED, VirtualMachine.UNCONFIGURED]: + new_state = state + elif self.is_configured() is None: + new_state = VirtualMachine.RUNNING + elif self.is_configured(): + new_state = VirtualMachine.CONFIGURED + else: + new_state = VirtualMachine.UNCONFIGURED + with self._lock: + self.state = new_state self.info.systems[0].setValue("state", new_state) return updated @@ -513,6 +530,7 @@ def check_ctxt_process(self): try: ssh.execute("kill -9 " + str(self.ctxt_pid)) except: + VirtualMachine.logger.exception("Error killing ctxt process with pid: " + str(self.ctxt_pid)) pass self.ctxt_pid = None @@ -521,9 +539,11 @@ def check_ctxt_process(self): try: (_, _, exit_status) = ssh.execute("ps " + str(self.ctxt_pid)) except: + VirtualMachine.logger.warn("Error getting status of ctxt process with pid: " + str(self.ctxt_pid)) exit_status = 0 self.ssh_connect_errors += 1 if self.ssh_connect_errors > Config.MAX_SSH_ERRORS: + VirtualMachine.logger.error("Too much errors getting status of ctxt process with pid: " + str(self.ctxt_pid) + ". Forget it.") self.ssh_connect_errors = 0 self.ctxt_pid = None self.configured = False @@ -568,6 +588,7 @@ def get_ctxt_output(self, remote_dir): ssh.execute("rm -rf " + remote_dir + '/ctxt_agent.log') except Exception, ex: + VirtualMachine.logger.exception("Error getting contextualization process output") self.configured = False self.cont_out += "Error getting contextualization process output: " + str(ex) @@ -579,6 +600,7 @@ def get_ctxt_output(self, remote_dir): # And process it self.process_ctxt_agent_out(ctxt_agent_out) except Exception, ex: + VirtualMachine.logger.exception("Error getting contextualization agent output") self.configured = False self.cont_out += "Error getting contextualization agent output: " + str(ex) finally: diff --git a/IM/__init__.py b/IM/__init__.py index d2bc4e257..be3803530 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -16,6 +16,6 @@ __all__ = ['auth','bottle','CloudManager','config','ConfManager','db','ganglia','HTTPHeaderTransport','ImageManager','InfrastructureInfo','InfrastructureManager','parsetab','radl','recipe','request','REST', 'ServiceRequests','SSH','timedcall','uriparse','VMRC','xmlobject'] -__version__ = '1.2.2' +__version__ = '1.2.3' __author__ = 'Miguel Caballer' diff --git a/IM/ansible/ansible_launcher.py b/IM/ansible/ansible_launcher.py index 7263320b7..c2ce50461 100755 --- a/IM/ansible/ansible_launcher.py +++ b/IM/ansible/ansible_launcher.py @@ -55,13 +55,10 @@ def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retrie options, _ = parser.parse_args([]) sshpass = None - sudopass = None - options.sudo_user = options.sudo_user or C.DEFAULT_SUDO_USER if pk_file: options.private_key_file = pk_file else: sshpass = passwd - sudopass = passwd if user: remote_user=user @@ -105,17 +102,9 @@ def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retrie callbacks=playbook_cb, runner_callbacks=runner_cb, stats=stats, - timeout=options.timeout, - transport=options.connection, - sudo=options.sudo, - sudo_user=options.sudo_user, - sudo_pass=sudopass, extra_vars=extra_vars, private_key_file=options.private_key_file, - only_tags=['all'], - skip_tags=None, - check=False, - diff=options.diff + only_tags=['all'] ) try: diff --git a/IM/config.py b/IM/config.py index 4939f592b..170c38c03 100644 --- a/IM/config.py +++ b/IM/config.py @@ -72,6 +72,7 @@ class Config: GANGLIA_INFO_UPDATE_FREQUENCY = 30 PLAYBOOK_RETRIES = 1 VM_INFO_UPDATE_FREQUENCY = 10 + VM_INFO_UPDATE_ERROR_GRACE_PERIOD = 120 # This value must be always higher than VM_INFO_UPDATE_FREQUENCY REMOTE_CONF_DIR = "/tmp/.im" MAX_SSH_ERRORS = 5 PRIVATE_NET_AS_PUBLIC = '' diff --git a/IM/radl/radl_parse.py b/IM/radl/radl_parse.py index acd61ab1a..0394a62eb 100644 --- a/IM/radl/radl_parse.py +++ b/IM/radl/radl_parse.py @@ -36,7 +36,7 @@ def __init__(self, autodefinevars = True, **kwargs): 'RPAREN', 'NUMBER', 'AND', - 'OR', +# 'OR', 'EQ', 'LT', 'GT', @@ -117,7 +117,7 @@ def t_STRING(self, t): 'system' : 'SYSTEM', 'soft' : 'SOFT', 'and' : 'AND', - 'or' : 'OR', +# 'or' : 'OR', 'contains' : 'CONTAINS', 'deploy' : 'DEPLOY', 'configure': 'CONFIGURE', diff --git a/README b/README index 4f2bde598..9cbca5ce2 100644 --- a/README +++ b/README @@ -33,7 +33,7 @@ However, if you install IM from sources you should install: * The Python Lex & Yacc library (http://www.dabeaz.com/ply/), typically available as the 'python-ply' package. - * The paramiko ssh2 protocol library for python + * The paramiko ssh2 protocol library for python version 1.14 or later (http://www.lag.net/paramiko/), typically available as the 'python-paramiko' package. * The YAML library for Python, typically available as the 'python-yaml' or 'PyYAML' package. @@ -44,27 +44,31 @@ However, if you install IM from sources you should install: In particular, Ansible 1.4.2+ must be installed. To ensure the functionality the following values must be set in the ansible.cfg file: - [default] + [defaults] + transport = smart host_key_checking = False - transport = smart + sudo_user = root + sudo_exe = sudo [paramiko_connection] - record_host_keys = False - + + record_host_keys=False + [ssh_connection] - pipelining=True - # Only in systems with OpenSSH support to ControlPersist + + # Only in systems with OpenSSH support to ControlPersist ssh_args = -o ControlMaster=auto -o ControlPersist=900s # In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) - ssh_args = + #ssh_args = + pipelining = True 1.2 OPTIONAL PACKAGES --------------------- -In case of using the Amazon EC2 plugin the boto library version 2.0 or later +In case of using the Amazon EC2 plugin the boto library version 2.29 or later must be installed (http://boto.readthedocs.org/en/latest/). -In case of using the LibCloud plugin the apache-libcloud library version 0.15 or later +In case of using the LibCloud plugin the apache-libcloud library version 0.17 or later must be installed (http://libcloud.apache.org/). In case of using the SSL secured version of the XMLRPC API the SpringPython @@ -82,16 +86,20 @@ framework (http://www.cherrypy.org/) must be installed. 1.3.1 FROM PIP -------------- +**WARNING**: The SOAPpy distributed with pip does not work correctly so you must install +the packages 'python-soappy' or 'SOAPp'y before installing the IM with pip. + **WARNING**: In some GNU/Linux distributions (RHEL 6 or equivalents) you must uninstall -the packages python-paramiko and python-crypto before installing the IM with pip.** +the packages python-paramiko and python-crypto before installing the IM with pip. You only have to install the IM package through the pip tool. pip install IM -Pip will install all the pre-requisites needed. So Ansible 1.4.2 or later will -be installed in the system. In some cases it will need to have installed the GCC -compiler and the python developer libraries ('python-dev' or 'python-devel' +Pip will install all the pre-requisites needed. So Ansible 1.4.2 or later will +be installed in the system. Yo will also need to install the sshpass command +('sshpass' package in main distributions). In some cases it will need to have installed +the GCC compiler and the python developer libraries ('python-dev' or 'python-devel' packages in main distributions). You must also remember to modify the ansible.cfg file setting as specified in the diff --git a/README.md b/README.md index ef49098e8..26034eec4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ IM - Infrastructure Manager ============================ -* Version [![PyPI](https://img.shields.io/pypi/v/im.svg)](https://img.shields.io/pypi/v/im.svg) -* PyPI [![PypI](https://img.shields.io/pypi/dm/IM.svg)](https://img.shields.io/pypi/dm/IM.svg) +* Version ![PyPI](https://img.shields.io/pypi/v/im.svg) +* PyPI ![PypI](https://img.shields.io/pypi/dm/IM.svg) IM is a tool that deploys complex and customized virtual infrastructures on IaaS Cloud deployments (such as AWS, OpenStack, etc.). It eases the access and the @@ -35,7 +35,7 @@ However, if you install IM from sources you should install: + The Python Lex & Yacc library (http://www.dabeaz.com/ply/), typically available as the 'python-ply' package. - + The paramiko ssh2 protocol library for python + + The paramiko ssh2 protocol library for python version 1.14 or later (http://www.lag.net/paramiko/), typically available as the 'python-paramiko' package. + The YAML library for Python, typically available as the 'python-yaml' or 'PyYAML' package. @@ -47,28 +47,32 @@ However, if you install IM from sources you should install: To ensure the functionality the following values must be set in the ansible.cfg file: ``` -[default] +[defaults] +transport = smart host_key_checking = False -transport = smart +sudo_user = root +sudo_exe = sudo [paramiko_connection] -record_host_keys = False - + +record_host_keys=False + [ssh_connection] -pipelining=True -# Only in systems with OpenSSH support to ControlPersist + +# Only in systems with OpenSSH support to ControlPersist ssh_args = -o ControlMaster=auto -o ControlPersist=900s # In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) -ssh_args = +#ssh_args = +pipelining = True ``` 1.2 OPTIONAL PACKAGES --------------------- -In case of using the Amazon EC2 plugin the boto library version 2.0 or later +In case of using the Amazon EC2 plugin the boto library version 2.29 or later must be installed (http://boto.readthedocs.org/en/latest/). -In case of using the LibCloud plugin the apache-libcloud library version 0.15 or later +In case of using the LibCloud plugin the apache-libcloud library version 0.17 or later must be installed (http://libcloud.apache.org/). In case of using the SSL secured version of the XMLRPC API the SpringPython @@ -85,8 +89,11 @@ framework (http://www.cherrypy.org/) must be installed. ### 1.3.1 FROM PIP +**WARNING: The SOAPpy distributed with pip does not work correctly so you must install +the packages 'python-soappy' or 'SOAPp'y before installing the IM with pip.** + **WARNING: In some GNU/Linux distributions (RHEL 6 or equivalents) you must uninstall -the packages python-paramiko and python-crypto before installing the IM with pip.** +the packages 'python-paramiko' and 'python-crypto' before installing the IM with pip.** You only have to install the IM package through the pip tool. @@ -95,8 +102,9 @@ pip install IM ``` Pip will install all the pre-requisites needed. So Ansible 1.4.2 or later will - be installed in the system. In some cases it will need to have installed the GCC - compiler and the python developer libraries ('python-dev' or 'python-devel' + be installed in the system. Yo will also need to install the sshpass command + ('sshpass' package in main distributions). In some cases it will need to have installed + the GCC compiler and the python developer libraries ('python-dev' or 'python-devel' packages in main distributions). You must also remember to modify the ansible.cfg file setting as specified in the diff --git a/changelog b/changelog index 438cd563b..34238b935 100644 --- a/changelog +++ b/changelog @@ -99,3 +99,9 @@ IM 1.2.2 * Improve contextualization performance * Bugfix in the Ansible installation playbook * Change Ansible version to 1.8.4 + +IM 1.2.3 + * Bugfix in the Ansible launcher with versions 1.9.X + * Bugfix in VirtualMachine update_status function + * Add the VM_INFO_UPDATE_ERROR_GRACE_PERIOD to manage errors in the conections with Cloud providers + * Bugfix and code improvements in GCE connector diff --git a/connectors/GCE.py b/connectors/GCE.py index e0d390b5b..59612c422 100644 --- a/connectors/GCE.py +++ b/connectors/GCE.py @@ -18,6 +18,7 @@ import os from CloudConnector import CloudConnector +from libcloud.compute.base import Node from libcloud.compute.types import NodeState, Provider from libcloud.compute.providers import get_driver from IM.uriparse import uriparse @@ -53,9 +54,13 @@ def get_driver(self, auth_data): auth = auth_data.getAuthInfo(self.type) if auth and 'username' in auth[0] and 'password' in auth[0] and 'project' in auth[0]: - cls = get_driver(Provider.GCE) + cls = get_driver(Provider.GCE) + lines = len(auth[0]['password'].replace(" ","").split()) + if lines < 2: + raise Exception("The certificate provided to the GCE plugin has an incorrect format. Check that it has more than one line.") + driver = cls(auth[0]['username'], auth[0]['password'], project=auth[0]['project']) - + self.driver = driver return driver else: @@ -235,10 +240,11 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): args = {'size': instance_type, 'image': image, 'external_ip': 'ephemeral', - 'location': region, - 'name': "%s-%s" % (name.lower().replace("_","-"), int(time.time()*100))} + 'location': region} if self.request_external_ip(radl): + if num_vm: + raise Exception("A fixed IP cannot be specified to a set of nodes (deploy is higher than 1)") fixed_ip = self.request_external_ip(radl) args['external_ip'] = driver.ex_create_address(name="im-" + fixed_ip, region=region, address=fixed_ip) @@ -276,15 +282,17 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): while i < num_vm: self.logger.debug("Creating node") + args['name'] = "%s-%s" % (name.lower().replace("_","-"), int(time.time()*100)) + node = driver.create_node(**args) - + if node: vm = VirtualMachine(inf, node.extra['name'], self.cloud, radl, requested_radl, self) self.logger.debug("Node successfully created.") res.append((True, vm)) else: res.append((False, "Error creating the node")) - + i += 1 return res @@ -316,11 +324,22 @@ def delete_disks(self, node): try: vol_name = os.path.basename(uriparse(disk['source'])[2]) volume = node.driver.ex_get_volume(vol_name) - success = volume.destroy() - if not success: - self.logger.error("Error destroying the volume: " + str(disk.id)) + # First try to detach the volume + if volume: + success = volume.detach() + if not success: + self.logger.error("Error detaching the volume: " + vol_name) + else: + # wait a bit to detach the disk + time.sleep(2) + success = volume.destroy() + if not success: + self.logger.error("Error destroying the volume: " + vol_name) + except ResourceNotFoundError: + self.logger.debug("The volume: " + vol_name + " does not exists. Ignore it.") + success = True except: - self.logger.exception("Error destroying the volume: " + str(disk.id) + " from the node: " + node.id) + self.logger.exception("Error destroying the volume: " + vol_name + " from the node: " + node.id) success = False if not success: @@ -493,4 +512,4 @@ def stop(self, vm, auth_data): return (True, "") def alterVM(self, vm, radl, auth_data): - return (False, "Not supported") \ No newline at end of file + return (False, "Not supported") diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 9cff5f354..9b2bac77c 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -8,7 +8,8 @@ Prerequisites IM needs at least Python 2.6 to run, as well as the next libraries: * `PLY `_, Python Lex & Yacc library for python. -* `paramiko `_, ssh2 protocol library for python. +* `paramiko `_, ssh2 protocol library for python + (version 1.14 or later). * `PyYAML `_, a YAML parser. * `SOAPpy `_, a full-featured SOAP library (we know it is not actively supported by upstream anymore). @@ -29,26 +30,30 @@ Fedora, etc.), do:: Finally, check the next values in the Ansible configuration file :file:`ansible.cfg`, (usually found in :file:`/etc/ansible`):: - [default] + [defaults] + transport = smart host_key_checking = False - transport = smart - + sudo_user = root + sudo_exe = sudo + [paramiko_connection] - record_host_keys = False + + record_host_keys=False [ssh_connection] - pipelining=True - # Only in systems with OpenSSH support to ControlPersist + + # Only in systems with OpenSSH support to ControlPersist ssh_args = -o ControlMaster=auto -o ControlPersist=900s # In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) - ssh_args = + #ssh_args = + pipelining = True Optional Packages ----------------- -* `apache-libcloud `_ 0.16 or later is used in the +* `apache-libcloud `_ 0.17 or later is used in the LibCloud, OpenStack and GCE connectors. -* `boto `_ 2.19.0 or later is used as interface to +* `boto `_ 2.29.0 or later is used as interface to Amazon EC2. It is available as package named ``python-boto`` in Debian based distributions. It can also be downloaded from `boto GitHub repository `_. Download the file and copy the boto subdirectory into the IM install path. @@ -66,16 +71,20 @@ Installation Form Pip ^^^^^^^^ -You only have to call the install command of the pip tool with the IM package:: - - $ pip install IM +**WARNING: The SOAPpy distributed with pip does not work correctly so you must install +the packages 'python-soappy' or 'SOAPpy' before installing the IM with pip.** **WARNING: In some linux distributions (REL 6 or equivalents) you must unistall the packages python-paramiko and python-crypto before installing the IM with pip.** -Pip will install all the pre-requisites needed. So Ansible 1.4.2 or later will be -installed in the system. In some cases it will need to have installed the GCC -compiler and the python developer libraries ('python-dev' or 'python-devel' +You only have to call the install command of the pip tool with the IM package:: + + $ pip install IM + +Pip will install all the pre-requisites needed. So Ansible 1.4.2 or later will +be installed in the system. Yo will also need to install the sshpass command +('sshpass' package in main distributions). In some cases it will need to have installed +the GCC compiler and the python developer libraries ('python-dev' or 'python-devel' packages in main distributions). You must also remember to modify the ansible.cfg file setting as specified in the @@ -182,6 +191,13 @@ Basic Options Maximum frequency to update the VM info (in secs) The default value is 10. + +.. confval:: VM_INFO_UPDATE_ERROR_GRACE_PERIOD + + Maximum time that a VM status maintains the current status in case of connection failure with the + Cloud provider (in secs). If the time is over this value the status is set to 'unknown'. + This value must be always higher than VM_INFO_UPDATE_FREQUENCY. + The default value is 120. .. confval:: WAIT_RUNNING_VM_TIMEOUT diff --git a/doc/source/xmlrpc.rst b/doc/source/xmlrpc.rst index 2f729dd53..36a483100 100644 --- a/doc/source/xmlrpc.rst +++ b/doc/source/xmlrpc.rst @@ -162,7 +162,14 @@ This is the list of method names: ``infId``. The ``deploy`` instructions in the ``radl`` must refer to *systems* already defined. If all the *systems* defined in ``radl`` are new, they will be added. Otherwise the new *systems* defined will be - ignored. + ignored. All the *systems* specified in the ``deploy`` must be specified + in the ``radl``. If they has been already defined only a reference is needed. + This is a simple example to deploy one new VM from an alreay defined system:: + + network public + system node + deploy node 1 + ``RemoveResource`` :parameter 0: ``infId``: integer diff --git a/etc/im.cfg b/etc/im.cfg index 0e08d43ed..112c50410 100644 --- a/etc/im.cfg +++ b/etc/im.cfg @@ -37,6 +37,10 @@ MAX_VM_FAILS = 3 WAIT_RUNNING_VM_TIMEOUT = 1800 # Maximum frequency to update the VM info (in secs) VM_INFO_UPDATE_FREQUENCY = 10 +# Maximum time that a VM status maintains the current status in case of connection failure with the +# Cloud provider (in secs). If the time is over this value the status is set to 'unknown'. +# This value must be always higher than VM_INFO_UPDATE_FREQUENCY. +VM_INFO_UPDATE_ERROR_GRACE_PERIOD = 120 # Log File LOG_LEVEL = DEBUG