From 1a9e2c27e0708e93ef7eda936a06f5f7ab86a6fd Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 May 2015 10:08:33 +0200 Subject: [PATCH 01/17] Check the security of the password in RADL --- IM/radl/radl.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/IM/radl/radl.py b/IM/radl/radl.py index eb8c4bdfd..8340864ac 100644 --- a/IM/radl/radl.py +++ b/IM/radl/radl.py @@ -34,7 +34,26 @@ def UnitToValue(unit): return 1 def is_version(version, _): - return all([num.isdigit() for num in version.getValue().split(".")]) + if version.getValue() == "": + return True + else: + return all([num.isdigit() for num in version.getValue().split(".")]) + +def check_password(password, _): + passwd = password.value + # Al least 6 chars + if len(passwd) <= 6: + return False + # At least one Upper leter + if passwd.lower() == passwd: + return False + # At least one digit + if len([x for x in passwd if x.isdigit()]) == 0: + return False + # At least one special char + if len([x for x in passwd if not x.isalnum()]) == 0: + return False + return True def check_outports_format(outports, _): """ @@ -1010,6 +1029,7 @@ def positive(f, _): "cpu.arch": (str, ['I386', 'X86_64']), "cpu.performance": ((int,float), positive, ["ECU", "GCEU", "HRZ"]), "memory.size": (int, positive, mem_units), + "disk.0.os.credentials.new.password": (str, check_password), SoftFeatures.SOFT: (SoftFeatures, lambda x, r: x.check(r)) } self.check_simple(SIMPLE_FEATURES, radl) From 94184be6557f51a631e47da7568b5a44e4b50962 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 May 2015 10:09:19 +0200 Subject: [PATCH 02/17] Bugfix when reusing HTTP connections --- connectors/Azure.py | 30 +++++++++---------- connectors/Docker.py | 69 +++++++++++++++++++++++++++++--------------- connectors/OCCI.py | 34 ++++++++++------------ 3 files changed, 77 insertions(+), 56 deletions(-) diff --git a/connectors/Azure.py b/connectors/Azure.py index 6a63d2166..d1d980295 100644 --- a/connectors/Azure.py +++ b/connectors/Azure.py @@ -130,9 +130,8 @@ class AzureCloudConnector(CloudConnector): } def __init__(self, cloud_info): - self.cert_file = None - self.key_file = None - self.connection = None + self.cert_file = '' + self.key_file = '' CloudConnector.__init__(self, cloud_info) def concreteSystem(self, radl_system, auth_data): @@ -361,23 +360,24 @@ def get_user_subscription_id(self, auth_data): def get_connection(self, auth_data): # We check if the cert and key files exist - if self.connection and os.path.isfile(self.cert_file) and os.path.isfile(self.key_file): - return self.connection + subscription_id = self.get_user_subscription_id(auth_data) + if subscription_id is None: + return None + + if os.path.isfile(self.cert_file) and os.path.isfile(self.key_file): + cert_file = self.cert_file + key_file = self.key_file else: - subscription_id = self.get_user_subscription_id(auth_data) auth = self.get_user_cert_data(auth_data) - - if auth is None or subscription_id is None: + if auth is None: return None - else: - cert_file, key_file = auth - - conn = httplib.HTTPSConnection(self.AZURE_SERVER, self.AZURE_PORT, cert_file=cert_file, key_file=key_file) + cert_file, key_file = auth self.cert_file = cert_file self.key_file = key_file - self.connection = conn - - return conn + + conn = httplib.HTTPSConnection(self.AZURE_SERVER, self.AZURE_PORT, cert_file=cert_file, key_file=key_file) + + return conn def get_user_cert_data(self, auth_data): """ diff --git a/connectors/Docker.py b/connectors/Docker.py index 3c9f94119..520bcd1b9 100644 --- a/connectors/Docker.py +++ b/connectors/Docker.py @@ -15,6 +15,7 @@ # along with this program. If not, see . import os +import tempfile import json import socket import httplib @@ -39,9 +40,8 @@ class DockerCloudConnector(CloudConnector): """ Counter to assign SSH port on Docker server host.""" def __init__(self, cloud_info): - self.cert_file = None - self.key_file = None - self.connection = None + self.cert_file = '' + self.key_file = '' CloudConnector.__init__(self, cloud_info) def get_http_connection(self, auth_data): @@ -53,29 +53,52 @@ def get_http_connection(self, auth_data): Returns(HTTPConnection or HTTPSConnection): HTTPConnection connection object """ - if self.connection and (not self.cert_file or os.path.isfile(self.cert_file)) and (not self.key_file or os.path.isfile(self.key_file)): - return self.connection - else: - auth = auth_data.getAuthInfo(DockerCloudConnector.type) - url = uriparse(self.cloud.server) + self.cert_file or os.path.isfile(self.cert_file) + - if url[0] == 'unix': - socket_path = "/" + url[1] + url[2] - conn = UnixHTTPConnection.UnixHTTPConnection(socket_path) - elif url[0] == 'https': - if auth and 'cert' in auth[0] and 'key' in auth[0]: - cert = auth[0]['cert'] - key = auth[0]['cert'] - conn = httplib.HTTPSConnection(url[1], self.cloud.port, cert_file = cert, key_file = key) + auth = auth_data.getAuthInfo(DockerCloudConnector.type) + url = uriparse(self.cloud.server) + + if url[0] == 'unix': + socket_path = "/" + url[1] + url[2] + conn = UnixHTTPConnection.UnixHTTPConnection(socket_path) + elif url[0] == 'https': + if auth and 'cert' in auth[0] and 'key' in auth[0]: + if os.path.isfile(self.cert_file) and os.path.isfile(self.key_file): + cert_file = self.cert_file + key_file = self.key_file else: - conn = httplib.HTTPSConnection(url[1], self.cloud.port) - elif url[0] == 'http': - self.logger.warn("Using a unsecure connection to docker API!") - conn = httplib.HTTPConnection(url[1], self.cloud.port) - - self.connection = conn - return conn + cert_file, key_file = self.get_user_cert_data(auth) + self.cert_file = cert_file + self.key_file = key_file + conn = httplib.HTTPSConnection(url[1], self.cloud.port, cert_file = cert_file, key_file = key_file) + else: + conn = httplib.HTTPSConnection(url[1], self.cloud.port) + elif url[0] == 'http': + self.logger.warn("Using a unsecure connection to docker API!") + conn = httplib.HTTPConnection(url[1], self.cloud.port) + + return conn + + def get_user_cert_data(self, auth): + """ + Get the Docker private_key and public_key files from the auth data + """ + certificate = auth[0]['cert'] + fd, cert_file = tempfile.mkstemp() + os.write(fd, certificate) + os.close(fd) + os.chmod(cert_file,0644) + private_key = auth[0]['key'] + fd, key_file = tempfile.mkstemp() + os.write(fd, private_key) + os.close(fd) + os.chmod(key_file,0600) + + return (cert_file, key_file) + + def concreteSystem(self, radl_system, auth_data): if radl_system.getValue("disk.0.image.url"): url = uriparse(radl_system.getValue("disk.0.image.url")) diff --git a/connectors/OCCI.py b/connectors/OCCI.py index 736cbd0fa..54020991f 100644 --- a/connectors/OCCI.py +++ b/connectors/OCCI.py @@ -42,13 +42,13 @@ class OCCICloudConnector(CloudConnector): 'waiting': VirtualMachine.PENDING, 'active': VirtualMachine.RUNNING, 'inactive': VirtualMachine.OFF, + 'error': VirtualMachine.FAILED, 'suspended': VirtualMachine.OFF } """Dictionary with a map with the OCCI VM states to the IM states.""" def __init__(self, cloud_info): self.proxy_filename = None - self.connection = None CloudConnector.__init__(self, cloud_info) def get_https_connection(self, auth, server, port): @@ -57,12 +57,15 @@ def get_https_connection(self, auth, server, port): It uses a proxy file if it has been specified in the auth credentials """ if 'proxy' in auth[0]: - proxy = auth[0]['proxy'] - - (fproxy, proxy_filename) = tempfile.mkstemp() - os.write(fproxy, proxy) - os.close(fproxy) - self.proxy_filename = proxy_filename + if self.proxy_filename and os.path.isfile(self.proxy_filename): + proxy_filename = self.proxy_filename + else: + proxy = auth[0]['proxy'] + + (fproxy, proxy_filename) = tempfile.mkstemp() + os.write(fproxy, proxy) + os.close(fproxy) + self.proxy_filename = proxy_filename return httplib.HTTPSConnection(server, port, cert_file = proxy_filename) else: @@ -73,18 +76,13 @@ def get_http_connection(self, auth_data): Get the HTTP connection to contact the OCCI server """ # We check if the proxy file exists - if self.connection and (self.proxy_filename is None or os.path.isfile(self.proxy_filename)): - return self.connection + auth = auth_data.getAuthInfo(OCCICloudConnector.type) + url = uriparse(self.cloud.server) + + if url[0] == 'https': + conn = self.get_https_connection(auth, url[1], self.cloud.port) else: - auth = auth_data.getAuthInfo(OCCICloudConnector.type) - url = uriparse(self.cloud.server) - - if url[0] == 'https': - conn = self.get_https_connection(auth, url[1], self.cloud.port) - else: - conn = httplib.HTTPConnection(url[1], self.cloud.port) - - self.connection = conn + conn = httplib.HTTPConnection(url[1], self.cloud.port) return conn From ec9ad270d99cb57f49ac136187c1c7b050e247e4 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 May 2015 10:09:42 +0200 Subject: [PATCH 03/17] Check the security of the password in RADL --- test/TestRADL.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/TestRADL.py b/test/TestRADL.py index bdf6c54f0..37e5e23ed 100755 --- a/test/TestRADL.py +++ b/test/TestRADL.py @@ -143,6 +143,26 @@ def test_outports(self): with self.assertRaises(RADLParseException): self.radl_check(r) + def test_check_password(self): + + radl = """ +network publica () + +system main ( +disk.0.os.credentials.new.password = 'verysimple' +) """ + r = parse_radl(radl) + with self.assertRaises(RADLParseException): + r.check() + + radl = """ +network publica () + +system main ( +disk.0.os.credentials.new.password = 'NotS0simple+' +) """ + r = parse_radl(radl) + r.check() if __name__ == "__main__": unittest.main() From 7b16f15a9bab70be6a1dbad85fe3ae38dea8efab Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 May 2015 10:11:37 +0200 Subject: [PATCH 04/17] Set new version number --- IM/__init__.py | 2 +- changelog | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/IM/__init__.py b/IM/__init__.py index be3803530..d829a0fcd 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -16,6 +16,6 @@ __all__ = ['auth','bottle','CloudManager','config','ConfManager','db','ganglia','HTTPHeaderTransport','ImageManager','InfrastructureInfo','InfrastructureManager','parsetab','radl','recipe','request','REST', 'ServiceRequests','SSH','timedcall','uriparse','VMRC','xmlobject'] -__version__ = '1.2.3' +__version__ = '1.2.4' __author__ = 'Miguel Caballer' diff --git a/changelog b/changelog index 34238b935..3a3f99c43 100644 --- a/changelog +++ b/changelog @@ -105,3 +105,7 @@ IM 1.2.3 * Bugfix in VirtualMachine update_status function * Add the VM_INFO_UPDATE_ERROR_GRACE_PERIOD to manage errors in the conections with Cloud providers * Bugfix and code improvements in GCE connector + +IM 1.2.4 + * Bugfix in OCCI, Azure and Docker connectors when reusing HTTP connections + * Add a password check in the RADL parser From f46136122864f3e75ead57ba3c682cbc3253fd7f Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 May 2015 12:38:23 +0200 Subject: [PATCH 05/17] Bugfixes en case of multiple auth of same type --- IM/InfrastructureManager.py | 1 - IM/auth.py | 10 +++++++--- connectors/OCCI.py | 26 ++++++++++++++++++-------- connectors/OpenNebula.py | 13 +++++++++---- connectors/OpenStack.py | 18 +++++++++++------- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index c05acacf4..aa190e084 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -702,7 +702,6 @@ def GetInfrastructureRADL(inf_id, auth): sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) - InfrastructureManager.logger.info("RADL obtained successfully") # remove the F0000__FAKE_SYSTEM__ deploys # TODO: Do in a better way radl = sel_inf.radl.clone() diff --git a/IM/auth.py b/IM/auth.py index 240042d8f..c4ee96420 100644 --- a/IM/auth.py +++ b/IM/auth.py @@ -38,7 +38,7 @@ def __init__(self, auth_data): else: self.auth_list = auth_data - def getAuthInfo(self, auth_type): + def getAuthInfo(self, auth_type, host = None): """ Get the auth data of the specified type @@ -50,8 +50,12 @@ def getAuthInfo(self, auth_type): res = [] for auth in self.auth_list: if auth['type'] == auth_type: - res.append(auth) - break + if host: + if 'host' in auth and auth['host'].find(host) != -1: + res.append(auth) + else: + res.append(auth) + return res def getAuthInfoByID(self, auth_id): diff --git a/connectors/OCCI.py b/connectors/OCCI.py index 54020991f..315212a00 100644 --- a/connectors/OCCI.py +++ b/connectors/OCCI.py @@ -56,11 +56,11 @@ def get_https_connection(self, auth, server, port): Get a HTTPS connection with the specified server. It uses a proxy file if it has been specified in the auth credentials """ - if 'proxy' in auth[0]: + if 'proxy' in auth: if self.proxy_filename and os.path.isfile(self.proxy_filename): proxy_filename = self.proxy_filename else: - proxy = auth[0]['proxy'] + proxy = auth['proxy'] (fproxy, proxy_filename) = tempfile.mkstemp() os.write(fproxy, proxy) @@ -75,8 +75,12 @@ def get_http_connection(self, auth_data): """ Get the HTTP connection to contact the OCCI server """ - # We check if the proxy file exists - auth = auth_data.getAuthInfo(OCCICloudConnector.type) + auths = auth_data.getAuthInfo(self.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to OCCI.") + else: + auth = auths[0] + url = uriparse(self.cloud.server) if url[0] == 'https': @@ -91,8 +95,14 @@ def get_auth_header(self, auth_data): Generate the auth header needed to contact with the OCCI server. I supports Keystone tokens and basic auth. """ + auths = auth_data.getAuthInfo(self.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to OCCI.") + return None + else: + auth = auths[0] + auth_header = None - auth = auth_data.getAuthInfo(OCCICloudConnector.type) keystone_uri = KeyStoneAuth.get_keystone_uri(self, auth_data) if keystone_uri: @@ -100,9 +110,9 @@ def get_auth_header(self, auth_data): keystone_token = KeyStoneAuth.get_keystone_token(self, keystone_uri, auth) auth_header = {'X-Auth-Token' : keystone_token} else: - if auth and 'username' in auth[0] and 'password' in auth[0]: - passwd = auth[0]['password'] - user = auth[0]['username'] + if 'username' in auth and 'password' in auth: + passwd = auth['password'] + user = auth['username'] auth_header = { 'Authorization' : 'Basic ' + string.strip(base64.encodestring(user + ':' + passwd))} return auth_header diff --git a/connectors/OpenNebula.py b/connectors/OpenNebula.py index fe69e6b0a..b9d052538 100644 --- a/connectors/OpenNebula.py +++ b/connectors/OpenNebula.py @@ -149,9 +149,14 @@ def getSessionID(self, auth_data, hash_password = None): if self.session_id: return self.session_id else: - auth = auth_data.getAuthInfo(OpenNebulaCloudConnector.type) - if auth and 'username' in auth[0] and 'password' in auth[0]: - passwd = auth[0]['password'] + auths = auth_data.getAuthInfo(self.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to OpenNebula.") + else: + auth = auths[0] + + if 'username' in auth and 'password' in auth: + passwd = auth['password'] if hash_password is None: one_ver = self.getONEVersion(auth_data) if one_ver == "2.0.0" or one_ver == "3.0.0": @@ -159,7 +164,7 @@ def getSessionID(self, auth_data, hash_password = None): if hash_password: passwd = hashlib.sha1(passwd.strip()).hexdigest() - self.session_id = auth[0]['username'] + ":" + passwd + self.session_id = auth['username'] + ":" + passwd return self.session_id else: self.logger.error("No correct auth data has been specified to OpenNebula: username and password") diff --git a/connectors/OpenStack.py b/connectors/OpenStack.py index 80f09e3be..4d78359c2 100644 --- a/connectors/OpenStack.py +++ b/connectors/OpenStack.py @@ -41,9 +41,13 @@ def get_driver(self, auth_data): if self.driver: return self.driver else: - auth = auth_data.getAuthInfo(self.type) - - if auth and 'username' in auth[0] and 'password' in auth[0] and 'tenant' in auth[0]: + auths = auth_data.getAuthInfo(self.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to OpenStack.") + else: + auth = auths[0] + + if 'username' in auth and 'password' in auth and 'tenant' in auth: parameters = {"auth_version":'2.0_password', "auth_url":"http://" + self.cloud.server + ":" + str(self.cloud.port), "auth_token":None, @@ -53,15 +57,15 @@ def get_driver(self, auth_data): "base_url":None} for param in parameters: - if param in auth[0]: - parameters[param] = auth[0][param] + if param in auth: + parameters[param] = auth[param] else: self.logger.error("No correct auth data has been specified to OpenStack: username, password and tenant") return None cls = get_driver(Provider.OPENSTACK) - driver = cls(auth[0]['username'], auth[0]['password'], - ex_tenant_name=auth[0]['tenant'], + driver = cls(auth['username'], auth['password'], + ex_tenant_name=auth['tenant'], ex_force_auth_url=parameters["auth_url"], ex_force_auth_version=parameters["auth_version"], ex_force_service_region=parameters["service_region"], From 67c249e09adfec040f8fc9ee556b2fba131d5f69 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 May 2015 12:39:18 +0200 Subject: [PATCH 06/17] Bugfixes en case of multiple auth of same type --- changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog b/changelog index 3a3f99c43..31b1dcaf7 100644 --- a/changelog +++ b/changelog @@ -108,4 +108,5 @@ IM 1.2.3 IM 1.2.4 * Bugfix in OCCI, Azure and Docker connectors when reusing HTTP connections + * Bugfix in OpenNebula, OCCI and OpenStack connectors when using mutiple auth of same type * Add a password check in the RADL parser From 73f250e6d0c965fe4a1d3687c1aaa46b876edc47 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 May 2015 16:10:46 +0200 Subject: [PATCH 07/17] Bugfix in GetInfrastructureRADL function --- IM/InfrastructureManager.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index aa190e084..86da87701 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -710,9 +710,17 @@ def GetInfrastructureRADL(inf_id, auth): if not deploy.id.startswith("F0000__FAKE_SYSTEM_"): deploys.append(deploy) radl.deploys = deploys + + # remove the F0000__FAKE_SYSTEM__ deploys + # TODO: Do in a better way + systems = [] + for system in radl.systems: + if not system.name.startswith("F0000__FAKE_SYSTEM_"): + systems.append(system) + radl.systems = systems InfrastructureManager.logger.debug(str(radl)) - return str(sel_inf.radl) + return str(radl) @staticmethod def GetInfrastructureInfo(inf_id, auth): From 327d5de407d8e8c783577dcbbb5f6738fad26de8 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 May 2015 17:47:05 +0200 Subject: [PATCH 08/17] Bugfix in the GetInfrastructureRADL function --- IM/InfrastructureInfo.py | 28 ++++++++++++++++++++++++++-- IM/InfrastructureManager.py | 22 +++------------------- test/test.radl | 8 ++++---- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/IM/InfrastructureInfo.py b/IM/InfrastructureInfo.py index d27aa9684..393f868c2 100644 --- a/IM/InfrastructureInfo.py +++ b/IM/InfrastructureInfo.py @@ -47,6 +47,8 @@ class InfrastructureInfo: logger = logging.getLogger('InfrastructureManager') """Logger object.""" + FAKE_SYSTEM = "F0000__FAKE_SYSTEM__" + def __init__(self): self._lock = threading.Lock() """Threading Lock to avoid concurrency problems.""" @@ -239,10 +241,10 @@ def complete_radl(self, radl): radl.add(aspect.clone(), "replace") # Add fake deploys to indicate the cloud provider associated to a private network. - FAKE_SYSTEM, system_counter = "F0000__FAKE_SYSTEM__%s", 0 + system_counter = 0 for n in radl.networks: if n.id in self.private_networks: - system_id = FAKE_SYSTEM % system_counter + system_id = self.FAKE_SYSTEM + str(system_counter) system_counter += 1 radl.add(system(system_id, [Feature("net_interface.0.connection", "=", n.id)])) radl.add(deploy(system_id, 0, self.private_networks[n.id])) @@ -250,6 +252,28 @@ def complete_radl(self, radl): # Check the RADL radl.check(); + def get_radl(self): + """ + Get the RADL of this Infrastructure + """ + # remove the F0000__FAKE_SYSTEM__ deploys + # TODO: Do in a better way + radl = self.radl.clone() + deploys = [] + for deploy in radl.deploys: + if not deploy.id.startswith(self.FAKE_SYSTEM): + deploys.append(deploy) + radl.deploys = deploys + + # remove the F0000__FAKE_SYSTEM__ deploys + # TODO: Do in a better way + systems = [] + for system in radl.systems: + if not system.name.startswith(self.FAKE_SYSTEM): + systems.append(system) + radl.systems = systems + + return radl def select_vm_master(self): """ Select the VM master of the infrastructure. diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index 86da87701..d10341ee6 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -702,25 +702,9 @@ def GetInfrastructureRADL(inf_id, auth): sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) - # remove the F0000__FAKE_SYSTEM__ deploys - # TODO: Do in a better way - radl = sel_inf.radl.clone() - deploys = [] - for deploy in radl.deploys: - if not deploy.id.startswith("F0000__FAKE_SYSTEM_"): - deploys.append(deploy) - radl.deploys = deploys - - # remove the F0000__FAKE_SYSTEM__ deploys - # TODO: Do in a better way - systems = [] - for system in radl.systems: - if not system.name.startswith("F0000__FAKE_SYSTEM_"): - systems.append(system) - radl.systems = systems - - InfrastructureManager.logger.debug(str(radl)) - return str(radl) + radl = str(sel_inf.get_radl()) + InfrastructureManager.logger.debug(radl) + return radl @staticmethod def GetInfrastructureInfo(inf_id, auth): diff --git a/test/test.radl b/test/test.radl index 2c2786b0f..5bfa6ea77 100644 --- a/test/test.radl +++ b/test/test.radl @@ -23,13 +23,13 @@ cpu.count>=1 and memory.size>=512m and net_interface.0.connection = 'privada' and disk.0.os.name='linux' and -disk.0.image.url = 'one://ramses.i3m.upv.es/73' and +disk.0.image.url = 'one://ramses.i3m.upv.es/95' and disk.0.os.credentials.username = 'ubuntu' and disk.0.os.credentials.password = 'yoyoyo' and -#disk.0.image.url = 'one://ramses.i3m.upv.es/28' and +#disk.0.image.url = 'one://ramses.i3m.upv.es/94' and #disk.0.os.credentials.username = 'root' and #disk.0.os.credentials.password = 'grycap01' and -disk.0.os.credentials.new.password = 'tututu' and +disk.0.os.credentials.new.password = 'Tututu+01' and disk.0.applications contains (name='ganglia') and disk.1.size=1GB and disk.1.device='hdb' @@ -71,4 +71,4 @@ contextualize ( system front configure hadoop step 1 system front configure hd step 1 system wn configure hd step 1 -) \ No newline at end of file +) From 4c765347ec46fa80563fea90ae5cc63faadced33 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 May 2015 10:50:00 +0200 Subject: [PATCH 09/17] Enable to show the Ansible log messages while the process is running --- IM/ConfManager.py | 10 ++++++---- IM/ansible/ansible_callbacks.py | 6 +++++- IM/ansible/ansible_launcher.py | 18 ++++++++---------- contextualization/ctxt_agent.py | 28 ++++++++++++++++------------ 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/IM/ConfManager.py b/IM/ConfManager.py index 0c94e5224..e4657dcf7 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -23,6 +23,7 @@ import shutil import json import copy +from StringIO import StringIO from IM.ansible.ansible_launcher import AnsibleThread @@ -894,16 +895,17 @@ def call_ansible(self, tmp_dir, inventory, playbook, ssh): os.symlink(os.path.abspath(Config.RECIPES_DIR + "/utils"), tmp_dir + "/utils") ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": " + 'Lanzamos ansible.') - t = AnsibleThread(tmp_dir + "/" + playbook, None, 2, gen_pk_file, ssh.password, 1, tmp_dir + "/" + inventory, ssh.username) + output = StringIO() + t = AnsibleThread(output, tmp_dir + "/" + playbook, None, 2, gen_pk_file, ssh.password, 1, tmp_dir + "/" + inventory, ssh.username) t.daemon = True t.start() t.join() - (return_code, output, _) = t.results + (return_code, _) = t.results if return_code == 0: - return (True, output) + return (True, output.getvalue()) else: - return (False, output) + return (False, output.getvalue()) def add_ansible_header(self, host, os): """ diff --git a/IM/ansible/ansible_callbacks.py b/IM/ansible/ansible_callbacks.py index 9ea15c83d..119eec7aa 100644 --- a/IM/ansible/ansible_callbacks.py +++ b/IM/ansible/ansible_callbacks.py @@ -22,11 +22,15 @@ import getpass import fnmatch import datetime +import logging def display(msg, color=None, stderr=False, screen_only=False, log_only=False, runner=None, output=sys.stdout): if not log_only: msg2 = msg - print >>output, msg2 + if isinstance(output, logging.Logger): + output.info(msg2) + else: + print >>output, msg2 class AggregateStats(object): ''' holds stats about per-host activity during playbook runs ''' diff --git a/IM/ansible/ansible_launcher.py b/IM/ansible/ansible_launcher.py index c2ce50461..532c4c51f 100755 --- a/IM/ansible/ansible_launcher.py +++ b/IM/ansible/ansible_launcher.py @@ -22,7 +22,6 @@ import time import os import threading -from StringIO import StringIO import ansible.playbook import ansible.inventory import ansible.constants as C @@ -38,7 +37,7 @@ def colorize(lead, num, color): def hostcolor(host, stats, color=True): return "%-26s" % host -def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retries = 1, inventory_file=None, user=None, extra_vars={}): +def launch_playbook(output, playbook_file, host, passwd, threads, pk_file = None, retries = 1, inventory_file=None, user=None, extra_vars={}): ''' run ansible-playbook operations ''' # create parser for CLI options @@ -70,7 +69,6 @@ def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retrie if not os.path.isfile(playbook_file): raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook_file) - output = StringIO() num_retries = 0 return_code = 4 hosts_with_errors = [] @@ -161,14 +159,14 @@ def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retrie if return_code != 0: display("ERROR executing playbook (%s/%s)" % (num_retries, retries), color='red', output=output) - return (return_code, output.getvalue(), hosts_with_errors) + return (return_code, hosts_with_errors) class AnsibleThread(threading.Thread): """ Class to call the ansible playbooks in a Thread """ - def __init__(self, playbook_file, host = None, threads = 1, pk_file = None, passwd = None, retries = 1, inventory_file=None, user=None, extra_vars={}): + def __init__(self, output, playbook_file, host = None, threads = 1, pk_file = None, passwd = None, retries = 1, inventory_file=None, user=None, extra_vars={}): threading.Thread.__init__(self) self.playbook_file = playbook_file @@ -180,12 +178,12 @@ def __init__(self, playbook_file, host = None, threads = 1, pk_file = None, pass self.inventory_file = inventory_file self.user = user self.extra_vars=extra_vars - self.results = (None, None, None) + self.output = output + self.results = (None, None) def run(self): try: - self.results = launch_playbook(self.playbook_file, self.host, self.passwd, self.threads, self.pk_file, self.retries, self.inventory_file, self.user, self.extra_vars) + self.results = launch_playbook(self.output, self.playbook_file, self.host, self.passwd, self.threads, self.pk_file, self.retries, self.inventory_file, self.user, self.extra_vars) except errors.AnsibleError, e: - output = StringIO() - display("ERROR: %s" % e, color='red', stderr=True, output=output) - self.results = (1, output.getvalue(), []) + display("ERROR: %s" % e, color='red', stderr=True, output=self.output) + self.results = (1, []) diff --git a/contextualization/ctxt_agent.py b/contextualization/ctxt_agent.py index 9039bdb16..28cfeab4b 100755 --- a/contextualization/ctxt_agent.py +++ b/contextualization/ctxt_agent.py @@ -23,6 +23,7 @@ import getpass import json import threading +from StringIO import StringIO from SSH import SSH, AuthenticationException from ansible_launcher import AnsibleThread @@ -108,21 +109,22 @@ def run_command(command, timeout = None, poll_delay = 5): except Exception, ex: return "ERROR: Exception msg: " + str(ex) -def wait_thread(thread): +def wait_thread(thread, output = None): """ Wait for a thread to finish """ thread.join() - (return_code, output, hosts_with_errors) = thread.results + (return_code, hosts_with_errors) = thread.results - if return_code==0: - logger.debug(output) - else: - logger.error(output) + if output: + if return_code==0: + logger.debug(output) + else: + logger.error(output) return (return_code==0, hosts_with_errors) -def LaunchAnsiblePlaybook(playbook_file, vm, threads, inventory_file, pk_file, retries, change_pass_ok): +def LaunchAnsiblePlaybook(output, playbook_file, vm, threads, inventory_file, pk_file, retries, change_pass_ok): logger.debug('Call Ansible') passwd = None @@ -143,7 +145,7 @@ def LaunchAnsiblePlaybook(playbook_file, vm, threads, inventory_file, pk_file, r if 'new_passwd' in vm and vm['new_passwd'] and change_pass_ok: passwd = vm['new_passwd'] - t = AnsibleThread(playbook_file, None, threads, gen_pk_file, passwd, retries, inventory_file, None, {'IM_HOST': vm['ip'] + ":" + str(vm['ssh_port'])}) + t = AnsibleThread(output, playbook_file, None, threads, gen_pk_file, passwd, retries, inventory_file, None, {'IM_HOST': vm['ip'] + ":" + str(vm['ssh_port'])}) t.start() return t @@ -249,7 +251,7 @@ def contextualize_vm(general_conf_data, vm_conf_data): pk_file = None if cred_used == "pk_file": pk_file = PK_FILE - ansible_thread = LaunchAnsiblePlaybook(playbook, ctxt_vm, 2, inventory_file, pk_file, INTERNAL_PLAYBOOK_RETRIES, change_creds) + ansible_thread = LaunchAnsiblePlaybook(logger, playbook, ctxt_vm, 2, inventory_file, pk_file, INTERNAL_PLAYBOOK_RETRIES, change_creds) else: # In some strange cases the pk_file disappears. So test it and remake basic recipe success = False @@ -262,11 +264,12 @@ def contextualize_vm(general_conf_data, vm_conf_data): if not success: logger.warn("Error connecting with SSH using the ansible key with: " + ctxt_vm['ip'] + ". Call the basic playbook again.") basic_playbook = general_conf_data['conf_dir'] + "/basic_task_all.yml" - ansible_thread = LaunchAnsiblePlaybook(basic_playbook, ctxt_vm, 2, inventory_file, None, INTERNAL_PLAYBOOK_RETRIES, True) + output_basic = StringIO() + ansible_thread = LaunchAnsiblePlaybook(output_basic, basic_playbook, ctxt_vm, 2, inventory_file, None, INTERNAL_PLAYBOOK_RETRIES, True) ansible_thread.join() # in the other tasks pk_file can be used - ansible_thread = LaunchAnsiblePlaybook(playbook, ctxt_vm, 2, inventory_file, PK_FILE, INTERNAL_PLAYBOOK_RETRIES, True) + ansible_thread = LaunchAnsiblePlaybook(logger, playbook, ctxt_vm, 2, inventory_file, PK_FILE, INTERNAL_PLAYBOOK_RETRIES, True) (task_ok, _) = wait_thread(ansible_thread) if not task_ok: @@ -298,7 +301,8 @@ def contextualize_vm(general_conf_data, vm_conf_data): # Root logger: is used by paramiko logging.basicConfig(filename=vm_conf_data['remote_dir'] +"/ctxt_agent.log", level=logging.WARNING, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + #format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + format='%(message)s', datefmt='%m-%d-%Y %H:%M:%S') # ctxt_agent logger logger = logging.getLogger('ctxt_agent') From 0425eb7ad2a2b43de9825eaecc5a3cb7570b53a6 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 May 2015 12:29:06 +0200 Subject: [PATCH 10/17] Dynamically refresh the Ctxt output --- IM/SSH.py | 16 +++++++++++++++ IM/VirtualMachine.py | 45 +++++++++++++++++++++++++++++++------------ IM/config.py | 1 + changelog | 2 ++ doc/source/manual.rst | 9 +++++++-- etc/im.cfg | 2 ++ 6 files changed, 61 insertions(+), 14 deletions(-) diff --git a/IM/SSH.py b/IM/SSH.py index 76647652c..12a3992cc 100755 --- a/IM/SSH.py +++ b/IM/SSH.py @@ -394,3 +394,19 @@ def execute_timeout(self, command, timeout, retry = 1, kill_command = None): return res raise TimeOutException("Error: Timeout") + + def sftp_remove(self, path): + """ Delete a file, if possible. + + Arguments: + - path: Name of the file in the remote server to delete. + + Returns: True if the file is deleted or False if it exists. + """ + client = self.connect() + transport = client.get_transport() + sftp = paramiko.SFTPClient.from_transport(transport) + res = sftp.remove(path) + sftp.close() + transport.close() + return res \ No newline at end of file diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index 4848d1ba6..145f1eff8 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -522,6 +522,12 @@ def check_ctxt_process(self): self.ctxt_pid = None self.configured = False + ip = self.getPublicIP() + if not ip: + ip = ip = self.getPrivateIP() + remote_dir = Config.REMOTE_CONF_DIR + "/" + ip + "_" + str(self.getSSHPort()) + + wait = 0 while self.ctxt_pid: if self.ctxt_pid != self.WAIT_TO_PID: ssh = self.inf.vm_master.get_ssh() @@ -551,12 +557,19 @@ def check_ctxt_process(self): if exit_status != 0: self.ctxt_pid = None # The process has finished, get the outputs - ip = self.getPublicIP() - if not ip: - ip = ip = self.getPrivateIP() - remote_dir = Config.REMOTE_CONF_DIR + "/" + ip + "_" + str(self.getSSHPort()) - self.get_ctxt_output(remote_dir) + self.get_ctxt_log(remote_dir, True) + self.get_ctxt_output(remote_dir, True) + else: + # Get the log of the process to update the cont_out dynamically + if Config.UPDATE_CTXT_LOG_INTERVAL > 0 and wait > Config.UPDATE_CTXT_LOG_INTERVAL: + wait = 0 + VirtualMachine.logger.debug("Get the log of the ctxt process with pid: "+ str(self.ctxt_pid)) + self.get_ctxt_log(remote_dir) + # The process is still running, wait + time.sleep(Config.CHECK_CTXT_PROCESS_INTERVAL) + wait += Config.CHECK_CTXT_PROCESS_INTERVAL else: + # We are waiting the PID, sleep time.sleep(Config.CHECK_CTXT_PROCESS_INTERVAL) return self.ctxt_pid @@ -572,11 +585,11 @@ def is_configured(self): # Otherwise return the value of configured return self.configured - def get_ctxt_output(self, remote_dir): + def get_ctxt_log(self, remote_dir, delete = False): ssh = self.inf.vm_master.get_ssh() tmp_dir = tempfile.mkdtemp() - # Donwload the contextualization agent log + # Download the contextualization agent log try: # Get the messages of the contextualization process ssh.sftp_get(remote_dir + '/ctxt_agent.log', tmp_dir + '/ctxt_agent.log') @@ -585,18 +598,26 @@ def get_ctxt_output(self, remote_dir): # Remove problematic chars conf_out = filter(lambda x: x in string.printable, conf_out) self.cont_out += conf_out.encode("ascii", "replace") - - ssh.execute("rm -rf " + remote_dir + '/ctxt_agent.log') + if delete: + ssh.sftp_remove(remote_dir + '/ctxt_agent.log') except Exception, ex: - VirtualMachine.logger.exception("Error getting contextualization process output") + VirtualMachine.logger.exception("Error getting contextualization process log") self.configured = False - self.cont_out += "Error getting contextualization process output: " + str(ex) + self.cont_out += "Error getting contextualization process log: " + str(ex) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + def get_ctxt_output(self, remote_dir, delete = False): + ssh = self.inf.vm_master.get_ssh() + tmp_dir = tempfile.mkdtemp() - # Donwload the contextualization agent log + # Download the contextualization agent log try: # Get the JSON output of the ctxt_agent ssh.sftp_get(remote_dir + '/ctxt_agent.out', tmp_dir + '/ctxt_agent.out') with open(tmp_dir + '/ctxt_agent.out') as f: ctxt_agent_out = json.load(f) + if delete: + ssh.sftp_remove(remote_dir + '/ctxt_agent.out') # And process it self.process_ctxt_agent_out(ctxt_agent_out) except Exception, ex: diff --git a/IM/config.py b/IM/config.py index 170c38c03..812106ea6 100644 --- a/IM/config.py +++ b/IM/config.py @@ -78,6 +78,7 @@ class Config: PRIVATE_NET_AS_PUBLIC = '' CHECK_CTXT_PROCESS_INTERVAL = 5 CONFMAMAGER_CHECK_STATE_INTERVAL = 5 + UPDATE_CTXT_LOG_INTERVAL = 20 config = ConfigParser.ConfigParser() config.read([Config.IM_PATH + '/../im.cfg', Config.IM_PATH + '/../etc/im.cfg', '/etc/im/im.cfg']) diff --git a/changelog b/changelog index 31b1dcaf7..1b5c16250 100644 --- a/changelog +++ b/changelog @@ -110,3 +110,5 @@ IM 1.2.4 * Bugfix in OCCI, Azure and Docker connectors when reusing HTTP connections * Bugfix in OpenNebula, OCCI and OpenStack connectors when using mutiple auth of same type * Add a password check in the RADL parser + * Dynamically refresh the Ctxt output + diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 4f26045d8..ad62fdcdc 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -287,20 +287,25 @@ Contextualization Number of retries of the Ansible playbooks in case of failure. The default value is 1. -.. confval:: CHECK_CTXT_PROCESS_INTERVAL = 5 +.. confval:: CHECK_CTXT_PROCESS_INTERVAL Interval to update the state of the contextualization process in the VMs (in secs). Reducing this time the load of the IM service will decrease in contextualization steps, but may introduce some overhead time. The default value is 5. -.. confval:: CONFMAMAGER_CHECK_STATE_INTERVAL = 5 +.. confval:: CONFMAMAGER_CHECK_STATE_INTERVAL Interval to update the state of the processes of the ConfManager (in secs). Reducing this time the load of the IM service will decrease in contextualization steps, but may introduce some overhead time. The default value is 5. +.. confval:: UPDATE_CTXT_LOG_INTERVAL + + Interval to update the log output of the contextualization process in the VMs (in secs). + The default value is 20. + .. _options-xmlrpc: XML-RPC API diff --git a/etc/im.cfg b/etc/im.cfg index 112c50410..70caa53aa 100644 --- a/etc/im.cfg +++ b/etc/im.cfg @@ -68,6 +68,8 @@ MAX_CONTEXTUALIZATION_TIME = 7200 REMOTE_CONF_DIR = /tmp/.im # Interval to update the state of the contextualization process in the VMs (in secs) CHECK_CTXT_PROCESS_INTERVAL = 5 +# Interval to update the log output of the contextualization process in the VMs (in secs) +UPDATE_CTXT_LOG_INTERVAL = 20 # Interval to update the state of the processes of the ConfManager (in secs) CONFMAMAGER_CHECK_STATE_INTERVAL = 5 From 0f012d654136372f8c1ec1234eec0124364874f9 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 May 2015 16:32:04 +0200 Subject: [PATCH 11/17] Minor bugfix in EC2 connector when deleting a non existing instance --- connectors/EC2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connectors/EC2.py b/connectors/EC2.py index 9a504a432..392d9e153 100644 --- a/connectors/EC2.py +++ b/connectors/EC2.py @@ -930,7 +930,7 @@ def finalize(self, vm, auth_data): self.cancel_spot_requests(conn, vm) # Delete the EBS volumes - self.delete_volumes(conn, volumes, instance.id) + self.delete_volumes(conn, volumes, instance_id) # Delete the SG if this is the last VM self.delete_security_group(conn, vm.inf) From 6965db42f06a958e68162fb512c9c4dfefd8aa0d Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 May 2015 16:32:39 +0200 Subject: [PATCH 12/17] Minor bugfix in EC2 connector when deleting a non existing instance --- changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog b/changelog index 1b5c16250..4b6e3e564 100644 --- a/changelog +++ b/changelog @@ -111,4 +111,5 @@ IM 1.2.4 * Bugfix in OpenNebula, OCCI and OpenStack connectors when using mutiple auth of same type * Add a password check in the RADL parser * Dynamically refresh the Ctxt output + * Minor bugfix in EC2 connector when deleting a non existing instance From 8b04d42dd2d951394fadd4ba87961fd15e05ddf0 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 May 2015 16:57:10 +0200 Subject: [PATCH 13/17] Bugfix in dynamically refresh the Ctxt output --- IM/VirtualMachine.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index 145f1eff8..0ffefeea7 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -527,6 +527,7 @@ def check_ctxt_process(self): ip = ip = self.getPrivateIP() remote_dir = Config.REMOTE_CONF_DIR + "/" + ip + "_" + str(self.getSSHPort()) + initial_count_out = self.cont_out wait = 0 while self.ctxt_pid: if self.ctxt_pid != self.WAIT_TO_PID: @@ -557,14 +558,19 @@ def check_ctxt_process(self): if exit_status != 0: self.ctxt_pid = None # The process has finished, get the outputs - self.get_ctxt_log(remote_dir, True) + ctxt_log = self.get_ctxt_log(remote_dir, True) self.get_ctxt_output(remote_dir, True) + if ctxt_log: + self.cont_out = initial_count_out + ctxt_log + else: + self.cont_out = initial_count_out + "Error getting contextualization process log." else: # Get the log of the process to update the cont_out dynamically if Config.UPDATE_CTXT_LOG_INTERVAL > 0 and wait > Config.UPDATE_CTXT_LOG_INTERVAL: wait = 0 VirtualMachine.logger.debug("Get the log of the ctxt process with pid: "+ str(self.ctxt_pid)) - self.get_ctxt_log(remote_dir) + ctxt_log = self.get_ctxt_log(remote_dir) + self.cont_out = initial_count_out + ctxt_log # The process is still running, wait time.sleep(Config.CHECK_CTXT_PROCESS_INTERVAL) wait += Config.CHECK_CTXT_PROCESS_INTERVAL @@ -588,7 +594,8 @@ def is_configured(self): def get_ctxt_log(self, remote_dir, delete = False): ssh = self.inf.vm_master.get_ssh() tmp_dir = tempfile.mkdtemp() - + conf_out = "" + # Download the contextualization agent log try: # Get the messages of the contextualization process @@ -596,16 +603,16 @@ def get_ctxt_log(self, remote_dir, delete = False): with open(tmp_dir + '/ctxt_agent.log') as f: conf_out = f.read() # Remove problematic chars - conf_out = filter(lambda x: x in string.printable, conf_out) - self.cont_out += conf_out.encode("ascii", "replace") + conf_out = filter(lambda x: x in string.printable, conf_out).encode("ascii", "replace") if delete: ssh.sftp_remove(remote_dir + '/ctxt_agent.log') - except Exception, ex: + except Exception: VirtualMachine.logger.exception("Error getting contextualization process log") self.configured = False - self.cont_out += "Error getting contextualization process log: " + str(ex) finally: shutil.rmtree(tmp_dir, ignore_errors=True) + + return conf_out def get_ctxt_output(self, remote_dir, delete = False): ssh = self.inf.vm_master.get_ssh() From 682955b3a510ff2875922cad888a70f463628a2f Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 May 2015 18:13:52 +0200 Subject: [PATCH 14/17] Code improvements --- IM/ConfManager.py | 6 ++++++ IM/VirtualMachine.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/IM/ConfManager.py b/IM/ConfManager.py index e4657dcf7..0705c0858 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -115,6 +115,12 @@ def check_vm_ips(self, timeout = Config.WAIT_RUNNING_VM_TIMEOUT): # If the IP is not Available try to update the info vm.update_status(self.auth) + # If the VM is not in a "running" state, return false + if vm.state in [VirtualMachine.OFF, VirtualMachine.FAILED, VirtualMachine.STOPPED]: + ConfManager.logger.warn("Inf ID: " + str(self.inf.id) + ": Error waiting all the VMs to have a correct IP. VM ID: " + str(vm.id) + " is not running.") + self.inf.set_configured(False) + return False + if vm.hasPublicNet(): ip = vm.getPublicIP() else: diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index 0ffefeea7..b291a304b 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -49,7 +49,7 @@ def __init__(self, inf, cloud_id, cloud, info, requested_radl, cloud_connector = """Last update of the VM info""" self.destroy = False """Flag to specify that this VM has been destroyed""" - self.state = self.UNKNOWN + self.state = self.PENDING """VM State""" self.inf = inf """Infrastructure which this VM is part of""" @@ -533,7 +533,7 @@ def check_ctxt_process(self): if self.ctxt_pid != self.WAIT_TO_PID: ssh = self.inf.vm_master.get_ssh() - if self.state in [VirtualMachine.OFF, VirtualMachine.FAILED]: + if self.state in [VirtualMachine.OFF, VirtualMachine.FAILED, VirtualMachine.STOPPED]: try: ssh.execute("kill -9 " + str(self.ctxt_pid)) except: From ac14f2bec6c09be52121e2d994fb988b4852494a Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 22 May 2015 09:42:34 +0200 Subject: [PATCH 15/17] Bugfix with logger WARN instead of warn --- IM/VirtualMachine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index b291a304b..4e663156f 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -404,7 +404,7 @@ def update_status(self, auth): # If we have problems to update the VM info too much time, set to unknown if now - self.last_update > Config.VM_INFO_UPDATE_ERROR_GRACE_PERIOD: new_state = VirtualMachine.UNKNOWN - VirtualMachine.logger.WARN("Grace period to update VM info passed. Set state to 'unknown'") + VirtualMachine.logger.warn("Grace period to update VM info passed. Set state to 'unknown'") else: if state not in [VirtualMachine.RUNNING, VirtualMachine.CONFIGURED, VirtualMachine.UNCONFIGURED]: new_state = state From de04701c18f83e2db34f8ab1125dddbb3b836577 Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 22 May 2015 13:00:38 +0200 Subject: [PATCH 16/17] Set the VM number in the Inf Cont Msg --- IM/InfrastructureManager.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index d10341ee6..07434b879 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -746,7 +746,12 @@ def GetInfrastructureContMsg(inf_id, auth): InfrastructureManager.logger.info("Getting cont msg of the inf: " + str(inf_id)) sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) - res = sel_inf.cont_out + "\n\n".join([vm.cont_out for vm in sel_inf.get_vm_list() if vm.cont_out]) + res = sel_inf.cont_out + + for vm in sel_inf.get_vm_list(): + if vm.cont_out: + res += "VM " + str(vm.id) + ":\n" + vm.cont_out + "\n" + res += "***************************************************************************\n" InfrastructureManager.logger.debug(res) return res From 7b9234074a5ee7972657971f5edab4a2aea62370 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 25 May 2015 18:43:22 +0200 Subject: [PATCH 17/17] Encode to utf-8 the RADL recipes to enable special chars in them --- IM/radl/radl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/IM/radl/radl.py b/IM/radl/radl.py index 8340864ac..0878cd207 100644 --- a/IM/radl/radl.py +++ b/IM/radl/radl.py @@ -596,9 +596,10 @@ class configure(Aspect): """Store a RADL ``configure``.""" def __init__(self, name, recipe="", reference=False, line=None): - self.recipes = recipe + # encode the recipe to enable to set special chars in the recipes + self.recipes = str(recipe.encode('utf-8', 'ignore')) """Recipe content.""" - self.name = name + self.name = str(name.encode('utf-8', 'ignore')) """Configure id.""" self.reference = reference """True if it is only a reference and it isn't a definition."""