diff --git a/IM/ConfManager.py b/IM/ConfManager.py index 0c94e5224..0705c0858 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -23,6 +23,7 @@ import shutil import json import copy +from StringIO import StringIO from IM.ansible.ansible_launcher import AnsibleThread @@ -114,6 +115,12 @@ def check_vm_ips(self, timeout = Config.WAIT_RUNNING_VM_TIMEOUT): # If the IP is not Available try to update the info vm.update_status(self.auth) + # If the VM is not in a "running" state, return false + if vm.state in [VirtualMachine.OFF, VirtualMachine.FAILED, VirtualMachine.STOPPED]: + ConfManager.logger.warn("Inf ID: " + str(self.inf.id) + ": Error waiting all the VMs to have a correct IP. VM ID: " + str(vm.id) + " is not running.") + self.inf.set_configured(False) + return False + if vm.hasPublicNet(): ip = vm.getPublicIP() else: @@ -894,16 +901,17 @@ def call_ansible(self, tmp_dir, inventory, playbook, ssh): os.symlink(os.path.abspath(Config.RECIPES_DIR + "/utils"), tmp_dir + "/utils") ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": " + 'Lanzamos ansible.') - t = AnsibleThread(tmp_dir + "/" + playbook, None, 2, gen_pk_file, ssh.password, 1, tmp_dir + "/" + inventory, ssh.username) + output = StringIO() + t = AnsibleThread(output, tmp_dir + "/" + playbook, None, 2, gen_pk_file, ssh.password, 1, tmp_dir + "/" + inventory, ssh.username) t.daemon = True t.start() t.join() - (return_code, output, _) = t.results + (return_code, _) = t.results if return_code == 0: - return (True, output) + return (True, output.getvalue()) else: - return (False, output) + return (False, output.getvalue()) def add_ansible_header(self, host, os): """ diff --git a/IM/InfrastructureInfo.py b/IM/InfrastructureInfo.py index d27aa9684..393f868c2 100644 --- a/IM/InfrastructureInfo.py +++ b/IM/InfrastructureInfo.py @@ -47,6 +47,8 @@ class InfrastructureInfo: logger = logging.getLogger('InfrastructureManager') """Logger object.""" + FAKE_SYSTEM = "F0000__FAKE_SYSTEM__" + def __init__(self): self._lock = threading.Lock() """Threading Lock to avoid concurrency problems.""" @@ -239,10 +241,10 @@ def complete_radl(self, radl): radl.add(aspect.clone(), "replace") # Add fake deploys to indicate the cloud provider associated to a private network. - FAKE_SYSTEM, system_counter = "F0000__FAKE_SYSTEM__%s", 0 + system_counter = 0 for n in radl.networks: if n.id in self.private_networks: - system_id = FAKE_SYSTEM % system_counter + system_id = self.FAKE_SYSTEM + str(system_counter) system_counter += 1 radl.add(system(system_id, [Feature("net_interface.0.connection", "=", n.id)])) radl.add(deploy(system_id, 0, self.private_networks[n.id])) @@ -250,6 +252,28 @@ def complete_radl(self, radl): # Check the RADL radl.check(); + def get_radl(self): + """ + Get the RADL of this Infrastructure + """ + # remove the F0000__FAKE_SYSTEM__ deploys + # TODO: Do in a better way + radl = self.radl.clone() + deploys = [] + for deploy in radl.deploys: + if not deploy.id.startswith(self.FAKE_SYSTEM): + deploys.append(deploy) + radl.deploys = deploys + + # remove the F0000__FAKE_SYSTEM__ deploys + # TODO: Do in a better way + systems = [] + for system in radl.systems: + if not system.name.startswith(self.FAKE_SYSTEM): + systems.append(system) + radl.systems = systems + + return radl def select_vm_master(self): """ Select the VM master of the infrastructure. diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index c05acacf4..07434b879 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -702,18 +702,9 @@ def GetInfrastructureRADL(inf_id, auth): sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) - InfrastructureManager.logger.info("RADL obtained successfully") - # remove the F0000__FAKE_SYSTEM__ deploys - # TODO: Do in a better way - radl = sel_inf.radl.clone() - deploys = [] - for deploy in radl.deploys: - if not deploy.id.startswith("F0000__FAKE_SYSTEM_"): - deploys.append(deploy) - radl.deploys = deploys - - InfrastructureManager.logger.debug(str(radl)) - return str(sel_inf.radl) + radl = str(sel_inf.get_radl()) + InfrastructureManager.logger.debug(radl) + return radl @staticmethod def GetInfrastructureInfo(inf_id, auth): @@ -755,7 +746,12 @@ def GetInfrastructureContMsg(inf_id, auth): InfrastructureManager.logger.info("Getting cont msg of the inf: " + str(inf_id)) sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) - res = sel_inf.cont_out + "\n\n".join([vm.cont_out for vm in sel_inf.get_vm_list() if vm.cont_out]) + res = sel_inf.cont_out + + for vm in sel_inf.get_vm_list(): + if vm.cont_out: + res += "VM " + str(vm.id) + ":\n" + vm.cont_out + "\n" + res += "***************************************************************************\n" InfrastructureManager.logger.debug(res) return res diff --git a/IM/SSH.py b/IM/SSH.py index 76647652c..12a3992cc 100755 --- a/IM/SSH.py +++ b/IM/SSH.py @@ -394,3 +394,19 @@ def execute_timeout(self, command, timeout, retry = 1, kill_command = None): return res raise TimeOutException("Error: Timeout") + + def sftp_remove(self, path): + """ Delete a file, if possible. + + Arguments: + - path: Name of the file in the remote server to delete. + + Returns: True if the file is deleted or False if it exists. + """ + client = self.connect() + transport = client.get_transport() + sftp = paramiko.SFTPClient.from_transport(transport) + res = sftp.remove(path) + sftp.close() + transport.close() + return res \ No newline at end of file diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index 4848d1ba6..4e663156f 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -49,7 +49,7 @@ def __init__(self, inf, cloud_id, cloud, info, requested_radl, cloud_connector = """Last update of the VM info""" self.destroy = False """Flag to specify that this VM has been destroyed""" - self.state = self.UNKNOWN + self.state = self.PENDING """VM State""" self.inf = inf """Infrastructure which this VM is part of""" @@ -404,7 +404,7 @@ def update_status(self, auth): # If we have problems to update the VM info too much time, set to unknown if now - self.last_update > Config.VM_INFO_UPDATE_ERROR_GRACE_PERIOD: new_state = VirtualMachine.UNKNOWN - VirtualMachine.logger.WARN("Grace period to update VM info passed. Set state to 'unknown'") + VirtualMachine.logger.warn("Grace period to update VM info passed. Set state to 'unknown'") else: if state not in [VirtualMachine.RUNNING, VirtualMachine.CONFIGURED, VirtualMachine.UNCONFIGURED]: new_state = state @@ -522,11 +522,18 @@ def check_ctxt_process(self): self.ctxt_pid = None self.configured = False + ip = self.getPublicIP() + if not ip: + ip = ip = self.getPrivateIP() + remote_dir = Config.REMOTE_CONF_DIR + "/" + ip + "_" + str(self.getSSHPort()) + + initial_count_out = self.cont_out + wait = 0 while self.ctxt_pid: if self.ctxt_pid != self.WAIT_TO_PID: ssh = self.inf.vm_master.get_ssh() - if self.state in [VirtualMachine.OFF, VirtualMachine.FAILED]: + if self.state in [VirtualMachine.OFF, VirtualMachine.FAILED, VirtualMachine.STOPPED]: try: ssh.execute("kill -9 " + str(self.ctxt_pid)) except: @@ -551,12 +558,24 @@ def check_ctxt_process(self): if exit_status != 0: self.ctxt_pid = None # The process has finished, get the outputs - ip = self.getPublicIP() - if not ip: - ip = ip = self.getPrivateIP() - remote_dir = Config.REMOTE_CONF_DIR + "/" + ip + "_" + str(self.getSSHPort()) - self.get_ctxt_output(remote_dir) + ctxt_log = self.get_ctxt_log(remote_dir, True) + self.get_ctxt_output(remote_dir, True) + if ctxt_log: + self.cont_out = initial_count_out + ctxt_log + else: + self.cont_out = initial_count_out + "Error getting contextualization process log." + else: + # Get the log of the process to update the cont_out dynamically + if Config.UPDATE_CTXT_LOG_INTERVAL > 0 and wait > Config.UPDATE_CTXT_LOG_INTERVAL: + wait = 0 + VirtualMachine.logger.debug("Get the log of the ctxt process with pid: "+ str(self.ctxt_pid)) + ctxt_log = self.get_ctxt_log(remote_dir) + self.cont_out = initial_count_out + ctxt_log + # The process is still running, wait + time.sleep(Config.CHECK_CTXT_PROCESS_INTERVAL) + wait += Config.CHECK_CTXT_PROCESS_INTERVAL else: + # We are waiting the PID, sleep time.sleep(Config.CHECK_CTXT_PROCESS_INTERVAL) return self.ctxt_pid @@ -572,31 +591,40 @@ def is_configured(self): # Otherwise return the value of configured return self.configured - def get_ctxt_output(self, remote_dir): + def get_ctxt_log(self, remote_dir, delete = False): ssh = self.inf.vm_master.get_ssh() tmp_dir = tempfile.mkdtemp() - - # Donwload the contextualization agent log + conf_out = "" + + # Download the contextualization agent log try: # Get the messages of the contextualization process ssh.sftp_get(remote_dir + '/ctxt_agent.log', tmp_dir + '/ctxt_agent.log') with open(tmp_dir + '/ctxt_agent.log') as f: conf_out = f.read() # Remove problematic chars - conf_out = filter(lambda x: x in string.printable, conf_out) - self.cont_out += conf_out.encode("ascii", "replace") - - ssh.execute("rm -rf " + remote_dir + '/ctxt_agent.log') - except Exception, ex: - VirtualMachine.logger.exception("Error getting contextualization process output") + conf_out = filter(lambda x: x in string.printable, conf_out).encode("ascii", "replace") + if delete: + ssh.sftp_remove(remote_dir + '/ctxt_agent.log') + except Exception: + VirtualMachine.logger.exception("Error getting contextualization process log") self.configured = False - self.cont_out += "Error getting contextualization process output: " + str(ex) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + return conf_out + + def get_ctxt_output(self, remote_dir, delete = False): + ssh = self.inf.vm_master.get_ssh() + tmp_dir = tempfile.mkdtemp() - # Donwload the contextualization agent log + # Download the contextualization agent log try: # Get the JSON output of the ctxt_agent ssh.sftp_get(remote_dir + '/ctxt_agent.out', tmp_dir + '/ctxt_agent.out') with open(tmp_dir + '/ctxt_agent.out') as f: ctxt_agent_out = json.load(f) + if delete: + ssh.sftp_remove(remote_dir + '/ctxt_agent.out') # And process it self.process_ctxt_agent_out(ctxt_agent_out) except Exception, ex: diff --git a/IM/__init__.py b/IM/__init__.py index be3803530..d829a0fcd 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -16,6 +16,6 @@ __all__ = ['auth','bottle','CloudManager','config','ConfManager','db','ganglia','HTTPHeaderTransport','ImageManager','InfrastructureInfo','InfrastructureManager','parsetab','radl','recipe','request','REST', 'ServiceRequests','SSH','timedcall','uriparse','VMRC','xmlobject'] -__version__ = '1.2.3' +__version__ = '1.2.4' __author__ = 'Miguel Caballer' diff --git a/IM/ansible/ansible_callbacks.py b/IM/ansible/ansible_callbacks.py index 9ea15c83d..119eec7aa 100644 --- a/IM/ansible/ansible_callbacks.py +++ b/IM/ansible/ansible_callbacks.py @@ -22,11 +22,15 @@ import getpass import fnmatch import datetime +import logging def display(msg, color=None, stderr=False, screen_only=False, log_only=False, runner=None, output=sys.stdout): if not log_only: msg2 = msg - print >>output, msg2 + if isinstance(output, logging.Logger): + output.info(msg2) + else: + print >>output, msg2 class AggregateStats(object): ''' holds stats about per-host activity during playbook runs ''' diff --git a/IM/ansible/ansible_launcher.py b/IM/ansible/ansible_launcher.py index c2ce50461..532c4c51f 100755 --- a/IM/ansible/ansible_launcher.py +++ b/IM/ansible/ansible_launcher.py @@ -22,7 +22,6 @@ import time import os import threading -from StringIO import StringIO import ansible.playbook import ansible.inventory import ansible.constants as C @@ -38,7 +37,7 @@ def colorize(lead, num, color): def hostcolor(host, stats, color=True): return "%-26s" % host -def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retries = 1, inventory_file=None, user=None, extra_vars={}): +def launch_playbook(output, playbook_file, host, passwd, threads, pk_file = None, retries = 1, inventory_file=None, user=None, extra_vars={}): ''' run ansible-playbook operations ''' # create parser for CLI options @@ -70,7 +69,6 @@ def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retrie if not os.path.isfile(playbook_file): raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook_file) - output = StringIO() num_retries = 0 return_code = 4 hosts_with_errors = [] @@ -161,14 +159,14 @@ def launch_playbook(playbook_file, host, passwd, threads, pk_file = None, retrie if return_code != 0: display("ERROR executing playbook (%s/%s)" % (num_retries, retries), color='red', output=output) - return (return_code, output.getvalue(), hosts_with_errors) + return (return_code, hosts_with_errors) class AnsibleThread(threading.Thread): """ Class to call the ansible playbooks in a Thread """ - def __init__(self, playbook_file, host = None, threads = 1, pk_file = None, passwd = None, retries = 1, inventory_file=None, user=None, extra_vars={}): + def __init__(self, output, playbook_file, host = None, threads = 1, pk_file = None, passwd = None, retries = 1, inventory_file=None, user=None, extra_vars={}): threading.Thread.__init__(self) self.playbook_file = playbook_file @@ -180,12 +178,12 @@ def __init__(self, playbook_file, host = None, threads = 1, pk_file = None, pass self.inventory_file = inventory_file self.user = user self.extra_vars=extra_vars - self.results = (None, None, None) + self.output = output + self.results = (None, None) def run(self): try: - self.results = launch_playbook(self.playbook_file, self.host, self.passwd, self.threads, self.pk_file, self.retries, self.inventory_file, self.user, self.extra_vars) + self.results = launch_playbook(self.output, self.playbook_file, self.host, self.passwd, self.threads, self.pk_file, self.retries, self.inventory_file, self.user, self.extra_vars) except errors.AnsibleError, e: - output = StringIO() - display("ERROR: %s" % e, color='red', stderr=True, output=output) - self.results = (1, output.getvalue(), []) + display("ERROR: %s" % e, color='red', stderr=True, output=self.output) + self.results = (1, []) diff --git a/IM/auth.py b/IM/auth.py index 240042d8f..c4ee96420 100644 --- a/IM/auth.py +++ b/IM/auth.py @@ -38,7 +38,7 @@ def __init__(self, auth_data): else: self.auth_list = auth_data - def getAuthInfo(self, auth_type): + def getAuthInfo(self, auth_type, host = None): """ Get the auth data of the specified type @@ -50,8 +50,12 @@ def getAuthInfo(self, auth_type): res = [] for auth in self.auth_list: if auth['type'] == auth_type: - res.append(auth) - break + if host: + if 'host' in auth and auth['host'].find(host) != -1: + res.append(auth) + else: + res.append(auth) + return res def getAuthInfoByID(self, auth_id): diff --git a/IM/config.py b/IM/config.py index 170c38c03..812106ea6 100644 --- a/IM/config.py +++ b/IM/config.py @@ -78,6 +78,7 @@ class Config: PRIVATE_NET_AS_PUBLIC = '' CHECK_CTXT_PROCESS_INTERVAL = 5 CONFMAMAGER_CHECK_STATE_INTERVAL = 5 + UPDATE_CTXT_LOG_INTERVAL = 20 config = ConfigParser.ConfigParser() config.read([Config.IM_PATH + '/../im.cfg', Config.IM_PATH + '/../etc/im.cfg', '/etc/im/im.cfg']) diff --git a/IM/radl/radl.py b/IM/radl/radl.py index eb8c4bdfd..0878cd207 100644 --- a/IM/radl/radl.py +++ b/IM/radl/radl.py @@ -34,7 +34,26 @@ def UnitToValue(unit): return 1 def is_version(version, _): - return all([num.isdigit() for num in version.getValue().split(".")]) + if version.getValue() == "": + return True + else: + return all([num.isdigit() for num in version.getValue().split(".")]) + +def check_password(password, _): + passwd = password.value + # Al least 6 chars + if len(passwd) <= 6: + return False + # At least one Upper leter + if passwd.lower() == passwd: + return False + # At least one digit + if len([x for x in passwd if x.isdigit()]) == 0: + return False + # At least one special char + if len([x for x in passwd if not x.isalnum()]) == 0: + return False + return True def check_outports_format(outports, _): """ @@ -577,9 +596,10 @@ class configure(Aspect): """Store a RADL ``configure``.""" def __init__(self, name, recipe="", reference=False, line=None): - self.recipes = recipe + # encode the recipe to enable to set special chars in the recipes + self.recipes = str(recipe.encode('utf-8', 'ignore')) """Recipe content.""" - self.name = name + self.name = str(name.encode('utf-8', 'ignore')) """Configure id.""" self.reference = reference """True if it is only a reference and it isn't a definition.""" @@ -1010,6 +1030,7 @@ def positive(f, _): "cpu.arch": (str, ['I386', 'X86_64']), "cpu.performance": ((int,float), positive, ["ECU", "GCEU", "HRZ"]), "memory.size": (int, positive, mem_units), + "disk.0.os.credentials.new.password": (str, check_password), SoftFeatures.SOFT: (SoftFeatures, lambda x, r: x.check(r)) } self.check_simple(SIMPLE_FEATURES, radl) diff --git a/changelog b/changelog index 34238b935..4b6e3e564 100644 --- a/changelog +++ b/changelog @@ -105,3 +105,11 @@ IM 1.2.3 * Bugfix in VirtualMachine update_status function * Add the VM_INFO_UPDATE_ERROR_GRACE_PERIOD to manage errors in the conections with Cloud providers * Bugfix and code improvements in GCE connector + +IM 1.2.4 + * Bugfix in OCCI, Azure and Docker connectors when reusing HTTP connections + * Bugfix in OpenNebula, OCCI and OpenStack connectors when using mutiple auth of same type + * Add a password check in the RADL parser + * Dynamically refresh the Ctxt output + * Minor bugfix in EC2 connector when deleting a non existing instance + diff --git a/connectors/Azure.py b/connectors/Azure.py index 6a63d2166..d1d980295 100644 --- a/connectors/Azure.py +++ b/connectors/Azure.py @@ -130,9 +130,8 @@ class AzureCloudConnector(CloudConnector): } def __init__(self, cloud_info): - self.cert_file = None - self.key_file = None - self.connection = None + self.cert_file = '' + self.key_file = '' CloudConnector.__init__(self, cloud_info) def concreteSystem(self, radl_system, auth_data): @@ -361,23 +360,24 @@ def get_user_subscription_id(self, auth_data): def get_connection(self, auth_data): # We check if the cert and key files exist - if self.connection and os.path.isfile(self.cert_file) and os.path.isfile(self.key_file): - return self.connection + subscription_id = self.get_user_subscription_id(auth_data) + if subscription_id is None: + return None + + if os.path.isfile(self.cert_file) and os.path.isfile(self.key_file): + cert_file = self.cert_file + key_file = self.key_file else: - subscription_id = self.get_user_subscription_id(auth_data) auth = self.get_user_cert_data(auth_data) - - if auth is None or subscription_id is None: + if auth is None: return None - else: - cert_file, key_file = auth - - conn = httplib.HTTPSConnection(self.AZURE_SERVER, self.AZURE_PORT, cert_file=cert_file, key_file=key_file) + cert_file, key_file = auth self.cert_file = cert_file self.key_file = key_file - self.connection = conn - - return conn + + conn = httplib.HTTPSConnection(self.AZURE_SERVER, self.AZURE_PORT, cert_file=cert_file, key_file=key_file) + + return conn def get_user_cert_data(self, auth_data): """ diff --git a/connectors/Docker.py b/connectors/Docker.py index 3c9f94119..520bcd1b9 100644 --- a/connectors/Docker.py +++ b/connectors/Docker.py @@ -15,6 +15,7 @@ # along with this program. If not, see . import os +import tempfile import json import socket import httplib @@ -39,9 +40,8 @@ class DockerCloudConnector(CloudConnector): """ Counter to assign SSH port on Docker server host.""" def __init__(self, cloud_info): - self.cert_file = None - self.key_file = None - self.connection = None + self.cert_file = '' + self.key_file = '' CloudConnector.__init__(self, cloud_info) def get_http_connection(self, auth_data): @@ -53,29 +53,52 @@ def get_http_connection(self, auth_data): Returns(HTTPConnection or HTTPSConnection): HTTPConnection connection object """ - if self.connection and (not self.cert_file or os.path.isfile(self.cert_file)) and (not self.key_file or os.path.isfile(self.key_file)): - return self.connection - else: - auth = auth_data.getAuthInfo(DockerCloudConnector.type) - url = uriparse(self.cloud.server) + self.cert_file or os.path.isfile(self.cert_file) + - if url[0] == 'unix': - socket_path = "/" + url[1] + url[2] - conn = UnixHTTPConnection.UnixHTTPConnection(socket_path) - elif url[0] == 'https': - if auth and 'cert' in auth[0] and 'key' in auth[0]: - cert = auth[0]['cert'] - key = auth[0]['cert'] - conn = httplib.HTTPSConnection(url[1], self.cloud.port, cert_file = cert, key_file = key) + auth = auth_data.getAuthInfo(DockerCloudConnector.type) + url = uriparse(self.cloud.server) + + if url[0] == 'unix': + socket_path = "/" + url[1] + url[2] + conn = UnixHTTPConnection.UnixHTTPConnection(socket_path) + elif url[0] == 'https': + if auth and 'cert' in auth[0] and 'key' in auth[0]: + if os.path.isfile(self.cert_file) and os.path.isfile(self.key_file): + cert_file = self.cert_file + key_file = self.key_file else: - conn = httplib.HTTPSConnection(url[1], self.cloud.port) - elif url[0] == 'http': - self.logger.warn("Using a unsecure connection to docker API!") - conn = httplib.HTTPConnection(url[1], self.cloud.port) - - self.connection = conn - return conn + cert_file, key_file = self.get_user_cert_data(auth) + self.cert_file = cert_file + self.key_file = key_file + conn = httplib.HTTPSConnection(url[1], self.cloud.port, cert_file = cert_file, key_file = key_file) + else: + conn = httplib.HTTPSConnection(url[1], self.cloud.port) + elif url[0] == 'http': + self.logger.warn("Using a unsecure connection to docker API!") + conn = httplib.HTTPConnection(url[1], self.cloud.port) + + return conn + + def get_user_cert_data(self, auth): + """ + Get the Docker private_key and public_key files from the auth data + """ + certificate = auth[0]['cert'] + fd, cert_file = tempfile.mkstemp() + os.write(fd, certificate) + os.close(fd) + os.chmod(cert_file,0644) + private_key = auth[0]['key'] + fd, key_file = tempfile.mkstemp() + os.write(fd, private_key) + os.close(fd) + os.chmod(key_file,0600) + + return (cert_file, key_file) + + def concreteSystem(self, radl_system, auth_data): if radl_system.getValue("disk.0.image.url"): url = uriparse(radl_system.getValue("disk.0.image.url")) diff --git a/connectors/EC2.py b/connectors/EC2.py index 9a504a432..392d9e153 100644 --- a/connectors/EC2.py +++ b/connectors/EC2.py @@ -930,7 +930,7 @@ def finalize(self, vm, auth_data): self.cancel_spot_requests(conn, vm) # Delete the EBS volumes - self.delete_volumes(conn, volumes, instance.id) + self.delete_volumes(conn, volumes, instance_id) # Delete the SG if this is the last VM self.delete_security_group(conn, vm.inf) diff --git a/connectors/OCCI.py b/connectors/OCCI.py index 736cbd0fa..315212a00 100644 --- a/connectors/OCCI.py +++ b/connectors/OCCI.py @@ -42,13 +42,13 @@ class OCCICloudConnector(CloudConnector): 'waiting': VirtualMachine.PENDING, 'active': VirtualMachine.RUNNING, 'inactive': VirtualMachine.OFF, + 'error': VirtualMachine.FAILED, 'suspended': VirtualMachine.OFF } """Dictionary with a map with the OCCI VM states to the IM states.""" def __init__(self, cloud_info): self.proxy_filename = None - self.connection = None CloudConnector.__init__(self, cloud_info) def get_https_connection(self, auth, server, port): @@ -56,13 +56,16 @@ def get_https_connection(self, auth, server, port): Get a HTTPS connection with the specified server. It uses a proxy file if it has been specified in the auth credentials """ - if 'proxy' in auth[0]: - proxy = auth[0]['proxy'] - - (fproxy, proxy_filename) = tempfile.mkstemp() - os.write(fproxy, proxy) - os.close(fproxy) - self.proxy_filename = proxy_filename + if 'proxy' in auth: + if self.proxy_filename and os.path.isfile(self.proxy_filename): + proxy_filename = self.proxy_filename + else: + proxy = auth['proxy'] + + (fproxy, proxy_filename) = tempfile.mkstemp() + os.write(fproxy, proxy) + os.close(fproxy) + self.proxy_filename = proxy_filename return httplib.HTTPSConnection(server, port, cert_file = proxy_filename) else: @@ -72,19 +75,18 @@ def get_http_connection(self, auth_data): """ Get the HTTP connection to contact the OCCI server """ - # We check if the proxy file exists - if self.connection and (self.proxy_filename is None or os.path.isfile(self.proxy_filename)): - return self.connection + auths = auth_data.getAuthInfo(self.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to OCCI.") else: - auth = auth_data.getAuthInfo(OCCICloudConnector.type) - url = uriparse(self.cloud.server) - - if url[0] == 'https': - conn = self.get_https_connection(auth, url[1], self.cloud.port) - else: - conn = httplib.HTTPConnection(url[1], self.cloud.port) - - self.connection = conn + auth = auths[0] + + url = uriparse(self.cloud.server) + + if url[0] == 'https': + conn = self.get_https_connection(auth, url[1], self.cloud.port) + else: + conn = httplib.HTTPConnection(url[1], self.cloud.port) return conn @@ -93,8 +95,14 @@ def get_auth_header(self, auth_data): Generate the auth header needed to contact with the OCCI server. I supports Keystone tokens and basic auth. """ + auths = auth_data.getAuthInfo(self.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to OCCI.") + return None + else: + auth = auths[0] + auth_header = None - auth = auth_data.getAuthInfo(OCCICloudConnector.type) keystone_uri = KeyStoneAuth.get_keystone_uri(self, auth_data) if keystone_uri: @@ -102,9 +110,9 @@ def get_auth_header(self, auth_data): keystone_token = KeyStoneAuth.get_keystone_token(self, keystone_uri, auth) auth_header = {'X-Auth-Token' : keystone_token} else: - if auth and 'username' in auth[0] and 'password' in auth[0]: - passwd = auth[0]['password'] - user = auth[0]['username'] + if 'username' in auth and 'password' in auth: + passwd = auth['password'] + user = auth['username'] auth_header = { 'Authorization' : 'Basic ' + string.strip(base64.encodestring(user + ':' + passwd))} return auth_header diff --git a/connectors/OpenNebula.py b/connectors/OpenNebula.py index fe69e6b0a..b9d052538 100644 --- a/connectors/OpenNebula.py +++ b/connectors/OpenNebula.py @@ -149,9 +149,14 @@ def getSessionID(self, auth_data, hash_password = None): if self.session_id: return self.session_id else: - auth = auth_data.getAuthInfo(OpenNebulaCloudConnector.type) - if auth and 'username' in auth[0] and 'password' in auth[0]: - passwd = auth[0]['password'] + auths = auth_data.getAuthInfo(self.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to OpenNebula.") + else: + auth = auths[0] + + if 'username' in auth and 'password' in auth: + passwd = auth['password'] if hash_password is None: one_ver = self.getONEVersion(auth_data) if one_ver == "2.0.0" or one_ver == "3.0.0": @@ -159,7 +164,7 @@ def getSessionID(self, auth_data, hash_password = None): if hash_password: passwd = hashlib.sha1(passwd.strip()).hexdigest() - self.session_id = auth[0]['username'] + ":" + passwd + self.session_id = auth['username'] + ":" + passwd return self.session_id else: self.logger.error("No correct auth data has been specified to OpenNebula: username and password") diff --git a/connectors/OpenStack.py b/connectors/OpenStack.py index 80f09e3be..4d78359c2 100644 --- a/connectors/OpenStack.py +++ b/connectors/OpenStack.py @@ -41,9 +41,13 @@ def get_driver(self, auth_data): if self.driver: return self.driver else: - auth = auth_data.getAuthInfo(self.type) - - if auth and 'username' in auth[0] and 'password' in auth[0] and 'tenant' in auth[0]: + auths = auth_data.getAuthInfo(self.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to OpenStack.") + else: + auth = auths[0] + + if 'username' in auth and 'password' in auth and 'tenant' in auth: parameters = {"auth_version":'2.0_password', "auth_url":"http://" + self.cloud.server + ":" + str(self.cloud.port), "auth_token":None, @@ -53,15 +57,15 @@ def get_driver(self, auth_data): "base_url":None} for param in parameters: - if param in auth[0]: - parameters[param] = auth[0][param] + if param in auth: + parameters[param] = auth[param] else: self.logger.error("No correct auth data has been specified to OpenStack: username, password and tenant") return None cls = get_driver(Provider.OPENSTACK) - driver = cls(auth[0]['username'], auth[0]['password'], - ex_tenant_name=auth[0]['tenant'], + driver = cls(auth['username'], auth['password'], + ex_tenant_name=auth['tenant'], ex_force_auth_url=parameters["auth_url"], ex_force_auth_version=parameters["auth_version"], ex_force_service_region=parameters["service_region"], diff --git a/contextualization/ctxt_agent.py b/contextualization/ctxt_agent.py index 9039bdb16..28cfeab4b 100755 --- a/contextualization/ctxt_agent.py +++ b/contextualization/ctxt_agent.py @@ -23,6 +23,7 @@ import getpass import json import threading +from StringIO import StringIO from SSH import SSH, AuthenticationException from ansible_launcher import AnsibleThread @@ -108,21 +109,22 @@ def run_command(command, timeout = None, poll_delay = 5): except Exception, ex: return "ERROR: Exception msg: " + str(ex) -def wait_thread(thread): +def wait_thread(thread, output = None): """ Wait for a thread to finish """ thread.join() - (return_code, output, hosts_with_errors) = thread.results + (return_code, hosts_with_errors) = thread.results - if return_code==0: - logger.debug(output) - else: - logger.error(output) + if output: + if return_code==0: + logger.debug(output) + else: + logger.error(output) return (return_code==0, hosts_with_errors) -def LaunchAnsiblePlaybook(playbook_file, vm, threads, inventory_file, pk_file, retries, change_pass_ok): +def LaunchAnsiblePlaybook(output, playbook_file, vm, threads, inventory_file, pk_file, retries, change_pass_ok): logger.debug('Call Ansible') passwd = None @@ -143,7 +145,7 @@ def LaunchAnsiblePlaybook(playbook_file, vm, threads, inventory_file, pk_file, r if 'new_passwd' in vm and vm['new_passwd'] and change_pass_ok: passwd = vm['new_passwd'] - t = AnsibleThread(playbook_file, None, threads, gen_pk_file, passwd, retries, inventory_file, None, {'IM_HOST': vm['ip'] + ":" + str(vm['ssh_port'])}) + t = AnsibleThread(output, playbook_file, None, threads, gen_pk_file, passwd, retries, inventory_file, None, {'IM_HOST': vm['ip'] + ":" + str(vm['ssh_port'])}) t.start() return t @@ -249,7 +251,7 @@ def contextualize_vm(general_conf_data, vm_conf_data): pk_file = None if cred_used == "pk_file": pk_file = PK_FILE - ansible_thread = LaunchAnsiblePlaybook(playbook, ctxt_vm, 2, inventory_file, pk_file, INTERNAL_PLAYBOOK_RETRIES, change_creds) + ansible_thread = LaunchAnsiblePlaybook(logger, playbook, ctxt_vm, 2, inventory_file, pk_file, INTERNAL_PLAYBOOK_RETRIES, change_creds) else: # In some strange cases the pk_file disappears. So test it and remake basic recipe success = False @@ -262,11 +264,12 @@ def contextualize_vm(general_conf_data, vm_conf_data): if not success: logger.warn("Error connecting with SSH using the ansible key with: " + ctxt_vm['ip'] + ". Call the basic playbook again.") basic_playbook = general_conf_data['conf_dir'] + "/basic_task_all.yml" - ansible_thread = LaunchAnsiblePlaybook(basic_playbook, ctxt_vm, 2, inventory_file, None, INTERNAL_PLAYBOOK_RETRIES, True) + output_basic = StringIO() + ansible_thread = LaunchAnsiblePlaybook(output_basic, basic_playbook, ctxt_vm, 2, inventory_file, None, INTERNAL_PLAYBOOK_RETRIES, True) ansible_thread.join() # in the other tasks pk_file can be used - ansible_thread = LaunchAnsiblePlaybook(playbook, ctxt_vm, 2, inventory_file, PK_FILE, INTERNAL_PLAYBOOK_RETRIES, True) + ansible_thread = LaunchAnsiblePlaybook(logger, playbook, ctxt_vm, 2, inventory_file, PK_FILE, INTERNAL_PLAYBOOK_RETRIES, True) (task_ok, _) = wait_thread(ansible_thread) if not task_ok: @@ -298,7 +301,8 @@ def contextualize_vm(general_conf_data, vm_conf_data): # Root logger: is used by paramiko logging.basicConfig(filename=vm_conf_data['remote_dir'] +"/ctxt_agent.log", level=logging.WARNING, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + #format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + format='%(message)s', datefmt='%m-%d-%Y %H:%M:%S') # ctxt_agent logger logger = logging.getLogger('ctxt_agent') diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 9b2bac77c..f875ccfe0 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -287,20 +287,25 @@ Contextualization Number of retries of the Ansible playbooks in case of failure. The default value is 1. -.. confval:: CHECK_CTXT_PROCESS_INTERVAL = 5 +.. confval:: CHECK_CTXT_PROCESS_INTERVAL Interval to update the state of the contextualization process in the VMs (in secs). Reducing this time the load of the IM service will decrease in contextualization steps, but may introduce some overhead time. The default value is 5. -.. confval:: CONFMAMAGER_CHECK_STATE_INTERVAL = 5 +.. confval:: CONFMAMAGER_CHECK_STATE_INTERVAL Interval to update the state of the processes of the ConfManager (in secs). Reducing this time the load of the IM service will decrease in contextualization steps, but may introduce some overhead time. The default value is 5. +.. confval:: UPDATE_CTXT_LOG_INTERVAL + + Interval to update the log output of the contextualization process in the VMs (in secs). + The default value is 20. + .. _options-xmlrpc: XML-RPC API diff --git a/etc/im.cfg b/etc/im.cfg index 112c50410..70caa53aa 100644 --- a/etc/im.cfg +++ b/etc/im.cfg @@ -68,6 +68,8 @@ MAX_CONTEXTUALIZATION_TIME = 7200 REMOTE_CONF_DIR = /tmp/.im # Interval to update the state of the contextualization process in the VMs (in secs) CHECK_CTXT_PROCESS_INTERVAL = 5 +# Interval to update the log output of the contextualization process in the VMs (in secs) +UPDATE_CTXT_LOG_INTERVAL = 20 # Interval to update the state of the processes of the ConfManager (in secs) CONFMAMAGER_CHECK_STATE_INTERVAL = 5 diff --git a/test/TestRADL.py b/test/TestRADL.py index bdf6c54f0..37e5e23ed 100755 --- a/test/TestRADL.py +++ b/test/TestRADL.py @@ -143,6 +143,26 @@ def test_outports(self): with self.assertRaises(RADLParseException): self.radl_check(r) + def test_check_password(self): + + radl = """ +network publica () + +system main ( +disk.0.os.credentials.new.password = 'verysimple' +) """ + r = parse_radl(radl) + with self.assertRaises(RADLParseException): + r.check() + + radl = """ +network publica () + +system main ( +disk.0.os.credentials.new.password = 'NotS0simple+' +) """ + r = parse_radl(radl) + r.check() if __name__ == "__main__": unittest.main() diff --git a/test/test.radl b/test/test.radl index 2c2786b0f..5bfa6ea77 100644 --- a/test/test.radl +++ b/test/test.radl @@ -23,13 +23,13 @@ cpu.count>=1 and memory.size>=512m and net_interface.0.connection = 'privada' and disk.0.os.name='linux' and -disk.0.image.url = 'one://ramses.i3m.upv.es/73' and +disk.0.image.url = 'one://ramses.i3m.upv.es/95' and disk.0.os.credentials.username = 'ubuntu' and disk.0.os.credentials.password = 'yoyoyo' and -#disk.0.image.url = 'one://ramses.i3m.upv.es/28' and +#disk.0.image.url = 'one://ramses.i3m.upv.es/94' and #disk.0.os.credentials.username = 'root' and #disk.0.os.credentials.password = 'grycap01' and -disk.0.os.credentials.new.password = 'tututu' and +disk.0.os.credentials.new.password = 'Tututu+01' and disk.0.applications contains (name='ganglia') and disk.1.size=1GB and disk.1.device='hdb' @@ -71,4 +71,4 @@ contextualize ( system front configure hadoop step 1 system front configure hd step 1 system wn configure hd step 1 -) \ No newline at end of file +)