diff --git a/.gitignore b/.gitignore index 4c8e2f6d8..95b2812b7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ test/auth.dat .coverage dist cover +*.pem +*.cer +inf.dat +radl diff --git a/IM/CloudInfo.py b/IM/CloudInfo.py index 1338583b1..f1fe068ab 100644 --- a/IM/CloudInfo.py +++ b/IM/CloudInfo.py @@ -46,7 +46,7 @@ def getCloudConnector(self): try: module = __import__('IM.connectors.' + self.type, fromlist=[self.type + "CloudConnector"]) return getattr(module, self.type + "CloudConnector")(self) - except Exception, ex: + except Exception as ex: raise Exception("Cloud provider not supported: %s (error: %s)" % (self.type, str(ex))) def __str__(self): diff --git a/IM/ConfManager.py b/IM/ConfManager.py index de90064ec..2d97c67ef 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -23,14 +23,18 @@ import shutil import json import copy -from StringIO import StringIO + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO from multiprocessing import Queue from ansible.parsing.vault import VaultEditor from IM.ansible_utils.ansible_launcher import AnsibleThread -import InfrastructureManager +import IM.InfrastructureManager from IM.VirtualMachine import VirtualMachine from IM.SSH import AuthenticationException from IM.SSHRetry import SSHRetry @@ -68,7 +72,7 @@ def check_running_pids(self, vms_configuring): Update the status of the configuration processes """ res = {} - for step, vm_list in vms_configuring.iteritems(): + for step, vm_list in vms_configuring.items(): for vm in vm_list: if isinstance(vm, VirtualMachine): # Update the info of the VM to check it is in a correct @@ -113,7 +117,7 @@ def check_running_pids(self, vms_configuring): def stop(self): self._stop = True # put a task to assure to wake up the thread - self.inf.ctxt_tasks.put((-10, 0, None, None)) + self.inf.add_ctxt_tasks([(-10, 0, None, None)]) ConfManager.logger.debug( "Inf ID: " + str(self.inf.id) + ": Stop Configuration thread.") if self.ansible_process and self.ansible_process.is_alive(): @@ -803,7 +807,7 @@ def configure_master(self): success = configured_ok - except Exception, ex: + except Exception as ex: ConfManager.logger.exception( "Inf ID: " + str(self.inf.id) + ": Error in the ansible installation process") self.inf.add_cont_msg( @@ -990,7 +994,7 @@ def generate_playbooks_and_hosts(self): ssh.sftp_put_files(recipe_files) self.inf.set_configured(True) - except Exception, ex: + except Exception as ex: self.inf.set_configured(False) ConfManager.logger.exception( "Inf ID: " + str(self.inf.id) + ": Error generating playbooks.") @@ -1004,7 +1008,7 @@ def relaunch_vm(self, vm, failed_cloud=False): Remove and launch again the specified VM """ try: - removed = InfrastructureManager.InfrastructureManager.RemoveResource( + removed = IM.InfrastructureManager.InfrastructureManager.RemoveResource( self.inf.id, vm.im_id, self.auth) except: ConfManager.logger.exception( @@ -1025,7 +1029,7 @@ def relaunch_vm(self, vm, failed_cloud=False): failed_clouds = [] if failed_cloud: failed_clouds = [vm.cloud] - InfrastructureManager.InfrastructureManager.AddResource( + IM.InfrastructureManager.InfrastructureManager.AddResource( self.inf.id, new_radl, self.auth, False, failed_clouds) def wait_vm_running(self, vm, timeout, relaunch=False): @@ -1161,6 +1165,17 @@ def wait_vm_ssh_acccess(self, vm, timeout): # Timeout, return False return False, "Timeout waiting SSH access." + @staticmethod + def cmp_credentials(creds, other_creds): + if len(creds) != len(other_creds): + return 1 + + for i in range(len(creds)): + if creds[i] != other_creds[i]: + return 1 + + return 0 + def change_master_credentials(self, ssh): """ Chech the RADL of the VM master to see if we must change the user credentials @@ -1175,7 +1190,7 @@ def change_master_credentials(self, ssh): new_creds = self.inf.vm_master.getCredentialValues(new=True) if len(list(set(new_creds))) > 1 or list(set(new_creds))[0] is not None: change_creds = False - if cmp(new_creds, creds) != 0: + if self.cmp_credentials(new_creds, creds) != 0: (_, new_passwd, new_public_key, new_private_key) = new_creds # only change to the new password if there are a previous # passwd value @@ -1234,7 +1249,7 @@ def call_ansible(self, tmp_dir, inventory, playbook, ssh): pk_out = open(gen_pk_file, 'w') pk_out.write(ssh.private_key) pk_out.close() - os.chmod(gen_pk_file, 0400) + os.chmod(gen_pk_file, 0o400) else: gen_pk_file = None @@ -1414,7 +1429,7 @@ def configure_ansible(self, ssh, tmp_dir): ConfManager.logger.debug("Inf ID: " + str(self.inf.id) + ": Ansible successfully configured in the master VM:\n" + msg + "\n\n") self.inf.add_cont_msg("Ansible successfully configured in the master VM.") - except Exception, ex: + except Exception as ex: ConfManager.logger.exception( "Inf ID: " + str(self.inf.id) + ": Error configuring master node.") self.inf.add_cont_msg("Error configuring master node: " + str(ex)) @@ -1456,7 +1471,7 @@ def create_general_conf_file(self, conf_file, vm_list): _, vm_conf_data['private_key']) = creds # If there are new creds to set to the VM if len(list(set(new_creds))) > 1 or list(set(new_creds))[0] is not None: - if cmp(new_creds, creds) != 0: + if self.cmp_credentials(new_creds, creds) != 0: (_, vm_conf_data['new_passwd'], vm_conf_data[ 'new_public_key'], vm_conf_data['new_private_key']) = new_creds diff --git a/IM/InfrastructureInfo.py b/IM/InfrastructureInfo.py index e7710bcd6..0181efe4a 100644 --- a/IM/InfrastructureInfo.py +++ b/IM/InfrastructureInfo.py @@ -20,13 +20,16 @@ from uuid import uuid1 import json -from ganglia import ganglia_info -import ConfManager +from IM.ganglia import ganglia_info +import IM.ConfManager from datetime import datetime from radl.radl import RADL, Feature, deploy, system, contextualize_item from radl.radl_json import parse_radl as parse_radl_json, dump_radl as dump_radl_json -from config import Config -from Queue import PriorityQueue +from IM.config import Config +try: + from Queue import PriorityQueue +except ImportError: + from queue import PriorityQueue from IM.VirtualMachine import VirtualMachine from IM.auth import Authentication @@ -180,7 +183,11 @@ def add_cont_msg(self, msg): """ Add a line to the contextualization message """ - self.cont_out += str(datetime.now()) + ": " + str(msg.decode('utf8', 'ignore')) + "\n" + try: + str_msg = str(msg.decode('utf8', 'ignore')) + except: + str_msg = msg + self.cont_out += str(datetime.now()) + ": " + str_msg + "\n" def get_vm_list(self): """ @@ -359,7 +366,7 @@ def update_ganglia_info(self): if now - self.last_ganglia_update > Config.GANGLIA_INFO_UPDATE_FREQUENCY: try: (success, msg) = ganglia_info.update_ganglia_info(self) - except Exception, ex: + except Exception as ex: success = False msg = str(ex) else: @@ -505,7 +512,7 @@ def Contextualize(self, auth, vm_list=None): self.add_ctxt_tasks(ctxt_task) if self.cm is None or not self.cm.isAlive(): - self.cm = ConfManager.ConfManager(self, auth, max_ctxt_time) + self.cm = IM.ConfManager.ConfManager(self, auth, max_ctxt_time) self.cm.start() else: # update the ConfManager reference to the inf object diff --git a/IM/InfrastructureList.py b/IM/InfrastructureList.py index 12b68a7c7..62ec85c41 100644 --- a/IM/InfrastructureList.py +++ b/IM/InfrastructureList.py @@ -92,7 +92,7 @@ def load_data(): try: inf_list = InfrastructureList._get_data_from_db(Config.DATA_DB) InfrastructureList.infrastructure_list = inf_list - except Exception, ex: + except Exception as ex: InfrastructureList.logger.exception("ERROR loading data. Correct or delete it!!") sys.stderr.write("ERROR loading data: " + str(ex) + ".\nCorrect or delete it!! ") sys.exit(-1) @@ -114,7 +114,7 @@ def save_data(inf_id=None): if not res: InfrastructureList.logger.error("ERROR saving data.\nChanges not stored!!") sys.stderr.write("ERROR saving data.\nChanges not stored!!") - except Exception, ex: + except Exception as ex: InfrastructureList.logger.exception("ERROR saving data. Changes not stored!!") sys.stderr.write("ERROR saving data: " + str(ex) + ".\nChanges not stored!!") diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index 32e93f14d..a9a6b63ab 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -478,8 +478,7 @@ def AddResource(inf_id, radl_data, auth, context=True, failed_clouds=[]): # NOTE: consider fake deploys (vm_number == 0) deploys_group_cloud_list = {} for deploy_group in deploy_groups: - suggested_cloud_ids = list( - set([d.cloud_id for d in deploy_group if d.cloud_id])) + suggested_cloud_ids = list(set([d.cloud_id for d in deploy_group if d.cloud_id])) if len(suggested_cloud_ids) > 1: raise Exception("Two deployments that have to be launched in the same cloud provider " "are asked to be deployed in different cloud providers: %s" % deploy_group) @@ -493,11 +492,16 @@ def AddResource(inf_id, radl_data, auth, context=True, failed_clouds=[]): (suggested_cloud_ids[0], cloud_list[suggested_cloud_ids[0]])] else: cloud_list0 = cloud_list.items() - if d.vm_number: - scored_clouds = [(cloud_id, sum([d.vm_number * concrete_systems[cloud_id][d.id][1] - for d in deploy_group])) for cloud_id, _ in cloud_list0] - else: - scored_clouds = [(cloud_id, 1) for cloud_id, _ in cloud_list0] + + scored_clouds = [] + for cloud_id, _ in cloud_list0: + total = 0 + for d in deploy_group: + if d.vm_number: + total += d.vm_number * concrete_systems[cloud_id][d.id][1] + else: + total += 1 + scored_clouds.append((cloud_id, total)) ordered_cloud_list = [c.id for c in CloudInfo.get_cloud_list(auth)] # reverse the list to use the reverse order in the sort function @@ -559,7 +563,7 @@ def AddResource(inf_id, radl_data, auth, context=True, failed_clouds=[]): if passwd and not new_passwd: # The VM uses the VMI password, set to change it random_password = ''.join(random.choice( - string.letters + string.digits) for _ in range(8)) + string.ascii_letters + string.digits) for _ in range(8)) vm.info.systems[0].setCredentialValues( password=random_password, new=True) diff --git a/IM/REST.py b/IM/REST.py index f27703280..3d9a4f047 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -241,13 +241,13 @@ def RESTDestroyInfrastructure(id=None): InfrastructureManager.DestroyInfrastructure(id, auth) bottle.response.content_type = "text/plain" return "" - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error Destroying Inf: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error Destroying Inf: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error Destroying Inf: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Destroying Inf") return return_error(400, "Error Destroying Inf: " + str(ex)) @@ -271,13 +271,13 @@ def RESTGetInfrastructureInfo(id=None): 'HTTP_HOST'] + '/infrastructures/' + str(id) + '/vms/' + str(vm_id)) return format_output(res, "text/uri-list", "uri-list", "uri") - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error Getting Inf. info: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error Getting Inf. info: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error Getting Inf. info: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Getting Inf. info") return return_error(400, "Error Getting Inf. info: " + str(ex)) @@ -305,13 +305,13 @@ def RESTGetInfrastructureProperty(id=None, prop=None): return return_error(404, "Incorrect infrastructure property") return format_output(res, field_name=prop) - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error Getting Inf. prop: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error Getting Inf. prop: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error Getting Inf. prop: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Getting Inf. prop") return return_error(400, "Error Getting Inf. prop: " + str(ex)) @@ -335,9 +335,9 @@ def RESTGetInfrastructureList(): protocol + bottle.request.environ['HTTP_HOST'] + "/infrastructures/" + str(inf_id)) return format_output(res, "text/uri-list", "uri-list", "uri") - except InvaliddUserException, ex: + except InvaliddUserException as ex: return return_error(401, "Error Getting Inf. List: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Getting Inf. List") return return_error(400, "Error Getting Inf. List: " + str(ex)) @@ -374,9 +374,9 @@ def RESTCreateInfrastructure(): "/infrastructures/" + str(inf_id) return format_output(res, "text/uri-list", "uri") - except InvaliddUserException, ex: + except InvaliddUserException as ex: return return_error(401, "Error Getting Inf. info: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Creating Inf.") return return_error(400, "Error Creating Inf.: " + str(ex)) @@ -391,17 +391,17 @@ def RESTGetVMInfo(infid=None, vmid=None): try: radl = InfrastructureManager.GetVMInfo(infid, vmid, auth) return format_output(radl, field_name="radl") - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error Getting VM. info: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error Getting VM. info: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error Getting VM. info: " + str(ex)) - except DeletedVMException, ex: + except DeletedVMException as ex: return return_error(404, "Error Getting VM. info: " + str(ex)) - except IncorrectVMException, ex: + except IncorrectVMException as ex: return return_error(404, "Error Getting VM. info: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Getting VM info") return return_error(400, "Error Getting VM info: " + str(ex)) @@ -423,17 +423,17 @@ def RESTGetVMProperty(infid=None, vmid=None, prop=None): return return_error(404, "Incorrect property %s for VM ID %s" % (prop, vmid)) else: return format_output(info, field_name=prop) - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error Getting VM. property: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error Getting VM. property: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error Getting VM. property: " + str(ex)) - except DeletedVMException, ex: + except DeletedVMException as ex: return return_error(404, "Error Getting VM. property: " + str(ex)) - except IncorrectVMException, ex: + except IncorrectVMException as ex: return return_error(404, "Error Getting VM. property: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Getting VM property") return return_error(400, "Error Getting VM property: " + str(ex)) @@ -479,13 +479,13 @@ def RESTAddResource(id=None): 'HTTP_HOST'] + "/infrastructures/" + str(id) + "/vms/" + str(vm_id)) return format_output(res, "text/uri-list", "uri-list", "uri") - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error Adding resources: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error Adding resources: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error Adding resources: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Adding resources") return return_error(400, "Error Adding resources: " + str(ex)) @@ -511,17 +511,17 @@ def RESTRemoveResource(infid=None, vmid=None): InfrastructureManager.RemoveResource(infid, vmid, auth, context) bottle.response.content_type = "text/plain" return "" - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error Removing resources: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error Removing resources: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error Removing resources: " + str(ex)) - except DeletedVMException, ex: + except DeletedVMException as ex: return return_error(404, "Error Removing resources: " + str(ex)) - except IncorrectVMException, ex: + except IncorrectVMException as ex: return return_error(404, "Error Removing resources: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error Removing resources") return return_error(400, "Error Removing resources: " + str(ex)) @@ -548,17 +548,17 @@ def RESTAlterVM(infid=None, vmid=None): vm_info = InfrastructureManager.AlterVM(infid, vmid, radl_data, auth) return format_output(vm_info, field_name="radl") - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error modifying resources: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error modifying resources: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error modifying resources: " + str(ex)) - except DeletedVMException, ex: + except DeletedVMException as ex: return return_error(404, "Error modifying resources: " + str(ex)) - except IncorrectVMException, ex: + except IncorrectVMException as ex: return return_error(404, "Error modifying resources: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error modifying resources") return return_error(400, "Error modifying resources: " + str(ex)) @@ -594,13 +594,13 @@ def RESTReconfigureInfrastructure(id=None): radl_data = "" bottle.response.content_type = "text/plain" return InfrastructureManager.Reconfigure(id, radl_data, auth, vm_list) - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error reconfiguring infrastructure: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error reconfiguring infrastructure: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error reconfiguring infrastructure: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error reconfiguring infrastructure") return return_error(400, "Error reconfiguring infrastructure: " + str(ex)) @@ -615,13 +615,13 @@ def RESTStartInfrastructure(id=None): try: bottle.response.content_type = "text/plain" return InfrastructureManager.StartInfrastructure(id, auth) - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error starting infrastructure: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error starting infrastructure: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error starting infrastructure: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error starting infrastructure") return return_error(400, "Error starting infrastructure: " + str(ex)) @@ -636,13 +636,13 @@ def RESTStopInfrastructure(id=None): try: bottle.response.content_type = "text/plain" return InfrastructureManager.StopInfrastructure(id, auth) - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error stopping infrastructure: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error stopping infrastructure: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error stopping infrastructure: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error stopping infrastructure") return return_error(400, "Error stopping infrastructure: " + str(ex)) @@ -657,17 +657,17 @@ def RESTStartVM(infid=None, vmid=None, prop=None): try: bottle.response.content_type = "text/plain" return InfrastructureManager.StartVM(infid, vmid, auth) - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error starting VM: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error starting VM: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error starting VM: " + str(ex)) - except DeletedVMException, ex: + except DeletedVMException as ex: return return_error(404, "Error starting VM: " + str(ex)) - except IncorrectVMException, ex: + except IncorrectVMException as ex: return return_error(404, "Error starting VM: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error starting VM") return return_error(400, "Error starting VM: " + str(ex)) @@ -682,17 +682,17 @@ def RESTStopVM(infid=None, vmid=None, prop=None): try: bottle.response.content_type = "text/plain" return InfrastructureManager.StopVM(infid, vmid, auth) - except DeletedInfrastructureException, ex: + except DeletedInfrastructureException as ex: return return_error(404, "Error stopping VM: " + str(ex)) - except IncorrectInfrastructureException, ex: + except IncorrectInfrastructureException as ex: return return_error(404, "Error stopping VM: " + str(ex)) - except UnauthorizedUserException, ex: + except UnauthorizedUserException as ex: return return_error(403, "Error stopping VM: " + str(ex)) - except DeletedVMException, ex: + except DeletedVMException as ex: return return_error(404, "Error stopping VM: " + str(ex)) - except IncorrectVMException, ex: + except IncorrectVMException as ex: return return_error(404, "Error stopping VM: " + str(ex)) - except Exception, ex: + except Exception as ex: logger.exception("Error stopping VM") return return_error(400, "Error stopping VM: " + str(ex)) @@ -702,7 +702,7 @@ def RESTGeVersion(): try: from IM import __version__ as version return format_output(version, field_name="version") - except Exception, ex: + except Exception as ex: return return_error(400, "Error getting IM version: " + str(ex)) diff --git a/IM/SSH.py b/IM/SSH.py index a2590bdd4..cb2621794 100755 --- a/IM/SSH.py +++ b/IM/SSH.py @@ -19,7 +19,10 @@ import paramiko import scp import os -import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO from threading import Thread @@ -85,7 +88,7 @@ def __init__(self, host, user, passwd, private_key=None, port=22): self.private_key = private_key self.private_key_obj = None if (private_key is not None and private_key.strip() != ""): - private_key_obj = StringIO.StringIO() + private_key_obj = StringIO() if os.path.isfile(private_key): pkfile = open(private_key) for line in pkfile.readlines(): @@ -152,7 +155,7 @@ def test_connectivity(self, time_out=None): return True except paramiko.AuthenticationException: raise AuthenticationException("Authentication Error!!") - except paramiko.SSHException, e: + except paramiko.SSHException as e: if str(e) == "No authentication methods available": raise AuthenticationException("Authentication Error!!") return False @@ -314,9 +317,7 @@ def sftp_put_dir(self, src, dest): except: sftp.mkdir(dest_path) else: - out, err, code = self.execute( - "mkdir -p %s" % dest_path) - print out, err + out, err, code = self.execute("mkdir -p %s" % dest_path) for filename in filenames: src_file = os.path.join(dirname, filename) dest_file = os.path.join(dest, dirname[len(src) + 1:], diff --git a/IM/ServiceRequests.py b/IM/ServiceRequests.py index b7fd68c39..095145ecd 100644 --- a/IM/ServiceRequests.py +++ b/IM/ServiceRequests.py @@ -18,10 +18,10 @@ import logging -from request import Request, AsyncRequest -import InfrastructureManager +from IM.request import Request, AsyncRequest +import IM.InfrastructureManager import IM.InfrastructureList -from auth import Authentication +from IM.auth import Authentication from IM import __version__ as version logger = logging.getLogger('InfrastructureManager') @@ -119,7 +119,7 @@ def _execute(self): res = self._call_function() self.set(res) return True - except Exception, ex: + except Exception as ex: logger.exception(self._error_mesage) self.set(str(ex)) return False @@ -133,7 +133,7 @@ class Request_AddResource(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Adding resources." (inf_id, radl_data, auth_data, context) = self.arguments - return InfrastructureManager.InfrastructureManager.AddResource(inf_id, radl_data, + return IM.InfrastructureManager.InfrastructureManager.AddResource(inf_id, radl_data, Authentication(auth_data), context) @@ -146,7 +146,7 @@ class Request_RemoveResource(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Removing resources." (inf_id, vm_list, auth_data, context) = self.arguments - return InfrastructureManager.InfrastructureManager.RemoveResource(inf_id, vm_list, + return IM.InfrastructureManager.InfrastructureManager.RemoveResource(inf_id, vm_list, Authentication(auth_data), context) @@ -159,7 +159,7 @@ class Request_GetInfrastructureInfo(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Getting Inf. Info." (inf_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.GetInfrastructureInfo(inf_id, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.GetInfrastructureInfo(inf_id, Authentication(auth_data)) class Request_GetVMInfo(IMBaseRequest): @@ -170,7 +170,7 @@ class Request_GetVMInfo(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Getting VM Info." (inf_id, vm_id, auth_data) = self.arguments - return str(InfrastructureManager.InfrastructureManager.GetVMInfo(inf_id, vm_id, Authentication(auth_data))) + return str(IM.InfrastructureManager.InfrastructureManager.GetVMInfo(inf_id, vm_id, Authentication(auth_data))) class Request_GetVMProperty(IMBaseRequest): @@ -181,7 +181,7 @@ class Request_GetVMProperty(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Getting VM Property." (inf_id, vm_id, property_name, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.GetVMProperty(inf_id, vm_id, property_name, + return IM.InfrastructureManager.InfrastructureManager.GetVMProperty(inf_id, vm_id, property_name, Authentication(auth_data)) @@ -193,7 +193,7 @@ class Request_AlterVM(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Changing VM Info." (inf_id, vm_id, radl, auth_data) = self.arguments - return str(InfrastructureManager.InfrastructureManager.AlterVM(inf_id, vm_id, radl, Authentication(auth_data))) + return str(IM.InfrastructureManager.InfrastructureManager.AlterVM(inf_id, vm_id, radl, Authentication(auth_data))) class Request_DestroyInfrastructure(IMBaseRequest): @@ -204,7 +204,7 @@ class Request_DestroyInfrastructure(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Destroying Inf." (inf_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.DestroyInfrastructure(inf_id, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.DestroyInfrastructure(inf_id, Authentication(auth_data)) class Request_StopInfrastructure(IMBaseRequest): @@ -215,7 +215,7 @@ class Request_StopInfrastructure(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Stopping Inf." (inf_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.StopInfrastructure(inf_id, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.StopInfrastructure(inf_id, Authentication(auth_data)) class Request_StartInfrastructure(IMBaseRequest): @@ -226,7 +226,7 @@ class Request_StartInfrastructure(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Starting Inf." (inf_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.StartInfrastructure(inf_id, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.StartInfrastructure(inf_id, Authentication(auth_data)) class Request_CreateInfrastructure(IMBaseRequest): @@ -237,7 +237,7 @@ class Request_CreateInfrastructure(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Creating Inf." (radl_data, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.CreateInfrastructure(radl_data, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.CreateInfrastructure(radl_data, Authentication(auth_data)) class Request_GetInfrastructureList(IMBaseRequest): @@ -248,7 +248,7 @@ class Request_GetInfrastructureList(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Getting Inf. List." (auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.GetInfrastructureList(Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.GetInfrastructureList(Authentication(auth_data)) class Request_Reconfigure(IMBaseRequest): @@ -259,7 +259,7 @@ class Request_Reconfigure(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Reconfiguring Inf." (inf_id, radl_data, auth_data, vm_list) = self.arguments - return InfrastructureManager.InfrastructureManager.Reconfigure(inf_id, radl_data, + return IM.InfrastructureManager.InfrastructureManager.Reconfigure(inf_id, radl_data, Authentication(auth_data), vm_list) @@ -271,7 +271,7 @@ class Request_ImportInfrastructure(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Importing Inf." (str_inf, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.ImportInfrastructure(str_inf, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.ImportInfrastructure(str_inf, Authentication(auth_data)) class Request_ExportInfrastructure(IMBaseRequest): @@ -282,7 +282,7 @@ class Request_ExportInfrastructure(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Exporting Inf." (inf_id, delete, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.ExportInfrastructure(inf_id, delete, + return IM.InfrastructureManager.InfrastructureManager.ExportInfrastructure(inf_id, delete, Authentication(auth_data)) @@ -294,7 +294,7 @@ class Request_GetInfrastructureRADL(IMBaseRequest): def _call_function(self): self._error_mesage = "Error getting RADL of the Inf." (inf_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.GetInfrastructureRADL(inf_id, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.GetInfrastructureRADL(inf_id, Authentication(auth_data)) class Request_GetVMContMsg(IMBaseRequest): @@ -305,7 +305,7 @@ class Request_GetVMContMsg(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Getting VM cont msg." (inf_id, vm_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.GetVMContMsg(inf_id, vm_id, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.GetVMContMsg(inf_id, vm_id, Authentication(auth_data)) class Request_GetInfrastructureContMsg(IMBaseRequest): @@ -316,7 +316,7 @@ class Request_GetInfrastructureContMsg(IMBaseRequest): def _call_function(self): self._error_mesage = "Error gettinf the Inf. cont msg" (inf_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.GetInfrastructureContMsg(inf_id, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.GetInfrastructureContMsg(inf_id, Authentication(auth_data)) class Request_SaveData(IMBaseRequest): @@ -339,7 +339,7 @@ class Request_StartVM(IMBaseRequest): def _call_function(self): self._error_mesage = "Error starting VM" (inf_id, vm_id, auth_data) = self.arguments - InfrastructureManager.InfrastructureManager.StartVM( + IM.InfrastructureManager.InfrastructureManager.StartVM( inf_id, vm_id, Authentication(auth_data)) return "" @@ -352,7 +352,7 @@ class Request_StopVM(IMBaseRequest): def _call_function(self): self._error_mesage = "Error stopping VM" (inf_id, vm_id, auth_data) = self.arguments - InfrastructureManager.InfrastructureManager.StopVM( + IM.InfrastructureManager.InfrastructureManager.StopVM( inf_id, vm_id, Authentication(auth_data)) return "" @@ -365,7 +365,7 @@ class Request_GetInfrastructureState(IMBaseRequest): def _call_function(self): self._error_mesage = "Error getting the Inf. state" (inf_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.GetInfrastructureState(inf_id, Authentication(auth_data)) + return IM.InfrastructureManager.InfrastructureManager.GetInfrastructureState(inf_id, Authentication(auth_data)) class Request_GetVersion(IMBaseRequest): diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index a584c5775..012b52233 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -538,7 +538,7 @@ def setIps(self, public_ips, private_ips): # Search in previous used private ips private_net = None - for net_mask, net in private_net_map.iteritems(): + for net_mask, net in private_net_map.items(): if IPAddress(private_ip) in IPNetwork(net_mask): private_net = net @@ -579,6 +579,7 @@ def setIps(self, public_ips, private_ips): vm_system.setValue( 'net_interface.' + str(num_net) + '.connection', private_net.id) + def get_ssh(self, retry=False): """ Get SSH object to connect with this VM @@ -782,7 +783,7 @@ def get_ctxt_output(self, remote_dir, delete=False): # And process it self.process_ctxt_agent_out(ctxt_agent_out) msg = "Contextualization agent output processed successfully" - except IOError, ex: + except IOError as ex: msg = "Error getting contextualization agent output " + \ remote_dir + "/ctxt_agent.out: No such file." VirtualMachine.logger.error(msg) @@ -804,7 +805,7 @@ def get_ctxt_output(self, remote_dir, delete=False): VirtualMachine.logger.exception( "Error getting stdout and stderr to guess why the agent output is not there.") pass - except Exception, ex: + except Exception as ex: VirtualMachine.logger.exception( "Error getting contextualization agent output: " + remote_dir + '/ctxt_agent.out') self.configured = False @@ -845,3 +846,6 @@ def get_ssh_ansible_master(self): return SSHRetry(ansible_host.getHost(), user, passwd, private_key) else: return self.inf.vm_master.get_ssh(retry=True) + + def __lt__(self, other): + return True diff --git a/IM/ansible_utils/ansible_callbacks.py b/IM/ansible_utils/ansible_callbacks.py index 64d3f6500..1c618352b 100644 --- a/IM/ansible_utils/ansible_callbacks.py +++ b/IM/ansible_utils/ansible_callbacks.py @@ -57,7 +57,7 @@ def _increment(self, what, host): def compute(self, runner_results, setup=False, poll=False, ignore_errors=False): ''' walk through all results and increment stats ''' - for (host, value) in runner_results.get('contacted', {}).iteritems(): + for (host, value) in runner_results.get('contacted', {}).items(): if not ignore_errors and (('failed' in value and bool(value['failed'])) or ('rc' in value and value['rc'] != 0)): self._increment('failures', host) @@ -71,7 +71,7 @@ def compute(self, runner_results, setup=False, poll=False, ignore_errors=False): if not poll or ('finished' in value and bool(value['finished'])): self._increment('ok', host) - for (host, value) in runner_results.get('dark', {}).iteritems(): + for (host, value) in runner_results.get('dark', {}).items(): self._increment('dark', host) def summarize(self, host): diff --git a/IM/ansible_utils/ansible_launcher.py b/IM/ansible_utils/ansible_launcher.py index 921a040c6..e35fde835 100755 --- a/IM/ansible_utils/ansible_launcher.py +++ b/IM/ansible_utils/ansible_launcher.py @@ -34,14 +34,14 @@ import ansible.constants as C from ansible import utils - from ansible_callbacks import banner, AggregateStats, PlaybookCallbacks, PlaybookRunnerCallbacks + from .ansible_callbacks import banner, AggregateStats, PlaybookCallbacks, PlaybookRunnerCallbacks else: from ansible.cli import CLI from ansible.parsing.dataloader import DataLoader from ansible.vars import VariableManager import ansible.inventory - from ansible_executor_v2 import IMPlaybookExecutor + from .ansible_executor_v2 import IMPlaybookExecutor def display(msg, color=None, stderr=False, screen_only=False, log_only=False, output=None): @@ -127,7 +127,7 @@ def run(self): else: display("ERROR: Unknown Ansible version.", output=self.output) self.result.put((0, (1, []), output)) - except errors.AnsibleError, e: + except errors.AnsibleError as e: display("ERROR: %s" % e, output=self.output) self.result.put((0, (1, []), output)) @@ -219,7 +219,7 @@ def launch_playbook_v2(self): return_code = pbex.run() - except errors.AnsibleError, e: + except errors.AnsibleError as e: display("ERROR: %s" % e, output=self.output) return_code = 1 @@ -347,7 +347,7 @@ def launch_playbook_v1(self): if len(unreachable_hosts) > 0: return_code = 3 - except errors.AnsibleError, e: + except errors.AnsibleError as e: display("ERROR: %s" % e, color='red', output=self.output) return_code = 1 diff --git a/IM/config.py b/IM/config.py index 0d4dde0a2..a8670129f 100644 --- a/IM/config.py +++ b/IM/config.py @@ -14,7 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import configparser +try: + from ConfigParser import ConfigParser +except ImportError: + from configparser import ConfigParser import os import logging @@ -88,7 +91,7 @@ class Config: UPDATE_CTXT_LOG_INTERVAL = 20 ANSIBLE_INSTALL_TIMEOUT = 900 -config = configparser.ConfigParser() +config = ConfigParser() config.read([Config.IM_PATH + '/../im.cfg', Config.IM_PATH + '/../etc/im.cfg', '/etc/im/im.cfg']) diff --git a/IM/connectors/Azure.py b/IM/connectors/Azure.py index ffcc2d08a..32641dd77 100644 --- a/IM/connectors/Azure.py +++ b/IM/connectors/Azure.py @@ -17,7 +17,7 @@ import time from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import Feature try: @@ -26,9 +26,9 @@ from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.network import NetworkManagementClient from azure.common.credentials import UserPassCredentials -except Exception, ex: - print "WARN: Python Azure SDK not correctly installed. AzureCloudConnector will not work!." - print ex +except Exception as ex: + print("WARN: Python Azure SDK not correctly installed. AzureCloudConnector will not work!.") + print(ex) class AzureCloudConnector(CloudConnector): @@ -400,7 +400,7 @@ def create_storage_account(self, group_name, storage_account, credentials, subsc 'location': location} ) return storage_async_operation.result(), "" - except Exception, ex: + except Exception as ex: self.logger.exception("Error creating the storage account") return None, str(ex) @@ -515,7 +515,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): self.attach_data_disks(vm, storage_account_name, credentials, subscription_id, location) res.append((True, vm)) - except Exception, ex: + except Exception as ex: self.logger.exception("Error creating the VM") res.append((False, "Error creating the VM: " + str(ex))) @@ -561,7 +561,7 @@ def attach_data_disks(self, vm, storage_account_name, credentials, subscription_ } ) async_vm_update.wait() - except Exception, ex: + except Exception as ex: self.logger.exception("Error attaching disk %d to VM %s" % (cont, vm_name)) return False, "Error attaching disk %d to VM %s: %s" % (cont, vm_name, str(ex)) cont += 1 @@ -578,7 +578,7 @@ def updateVMInfo(self, vm, auth_data): compute_client = ComputeManagementClient(credentials, subscription_id) # Get one the virtual machine by name virtual_machine = compute_client.virtual_machines.get(group_name, vm_name) - except Exception, ex: + except Exception as ex: if "NotFound" in str(ex): vm.state = VirtualMachine.OFF return (True, vm) @@ -639,7 +639,7 @@ def finalize(self, vm, auth_data): self.logger.exception("Removing RG: %s" % "rg-%s" % vm.inf.id) resource_client.resource_groups.delete("rg-%s" % vm.inf.id) - except Exception, ex: + except Exception as ex: self.logger.exception("Error terminating the VM") return False, "Error terminating the VM: " + str(ex) @@ -652,7 +652,7 @@ def stop(self, vm, auth_data): credentials, subscription_id = self.get_credentials(auth_data) compute_client = ComputeManagementClient(credentials, subscription_id) compute_client.virtual_machines.power_off(group_name, vm_name) - except Exception, ex: + except Exception as ex: self.logger.exception("Error stopping the VM") return False, "Error stopping the VM: " + str(ex) @@ -665,7 +665,7 @@ def start(self, vm, auth_data): credentials, subscription_id = self.get_credentials(auth_data) compute_client = ComputeManagementClient(credentials, subscription_id) compute_client.virtual_machines.start(group_name, vm_name) - except Exception, ex: + except Exception as ex: self.logger.exception("Error starting the VM") return False, "Error starting the VM: " + str(ex) @@ -695,7 +695,7 @@ def alterVM(self, vm, radl, auth_data): async_vm_start.wait() return self.updateVMInfo(vm, auth_data) - except Exception, ex: + except Exception as ex: self.logger.exception("Error altering the VM") return False, "Error altering the VM: " + str(ex) diff --git a/IM/connectors/AzureClassic.py b/IM/connectors/AzureClassic.py index 5f824e636..6379e7b1f 100644 --- a/IM/connectors/AzureClassic.py +++ b/IM/connectors/AzureClassic.py @@ -22,7 +22,7 @@ from IM.xmlobject import XMLObject from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import UserPassCredential, Feature from IM.config import Config @@ -415,15 +415,15 @@ def get_user_cert_data(self, auth): if 'public_key' in auth and 'private_key' in auth: certificate = auth['public_key'] fd, cert_file = tempfile.mkstemp() - os.write(fd, certificate) + os.write(fd, certificate.encode()) os.close(fd) - os.chmod(cert_file, 0644) + os.chmod(cert_file, 0o644) private_key = auth['private_key'] fd, key_file = tempfile.mkstemp() - os.write(fd, private_key) + os.write(fd, private_key.encode()) os.close(fd) - os.chmod(key_file, 0600) + os.chmod(key_file, 0o600) return (cert_file, key_file) else: @@ -449,10 +449,10 @@ def create_service(self, auth_data, region): Service %s created by the IM %s - ''' % (service_name, base64.b64encode(service_name), service_name, region) + ''' % (service_name, base64.b64encode(service_name.encode()), service_name, region) headers = {'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'} resp = self.create_request('POST', uri, auth_data, headers, service_create_xml) - except Exception, ex: + except Exception as ex: self.logger.exception("Error creating the service") return None, "Error creating the service" + str(ex) @@ -471,7 +471,7 @@ def delete_service(self, service_name, auth_data): uri = "/services/hostedservices/%s?comp=media" % service_name headers = {'x-ms-version': '2013-08-01'} resp = self.create_request('DELETE', uri, auth_data, headers) - except Exception, ex: + except Exception as ex: self.logger.exception("Error deleting the service") return (False, "Error deleting the service: " + str(ex)) @@ -556,7 +556,7 @@ def create_storage_account(self, storage_account, auth_data, region, timeout=120 ''' % (storage_account, storage_account, base64.b64encode(storage_account), region) headers = {'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'} resp = self.create_request('POST', uri, auth_data, headers, storage_create_xml) - except Exception, ex: + except Exception as ex: self.logger.exception("Error creating the storage account") return None, "Error creating the storage account" + str(ex) @@ -710,7 +710,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): self.delete_service(service_name, auth_data) res.append((False, "Error waiting the VM creation")) - except Exception, ex: + except Exception as ex: self.logger.exception("Error creating the VM") if service_name: self.delete_service(service_name, auth_data) @@ -780,7 +780,7 @@ def updateVMInfo(self, vm, auth_data): uri = "/services/hostedservices/%s/deployments/%s" % (service_name, service_name) headers = {'x-ms-version': '2014-02-01'} resp = self.create_request('GET', uri, auth_data, headers) - except Exception, ex: + except Exception as ex: self.logger.exception("Error getting the VM info: " + vm.id) return (False, "Error getting the VM info: " + vm.id + ". " + str(ex)) @@ -870,7 +870,7 @@ def call_role_operation(self, op, vm, auth_data): headers = {'x-ms-version': '2013-06-01', 'Content-Type': 'application/xml'} resp = self.create_request('POST', uri, auth_data, headers) - except Exception, ex: + except Exception as ex: self.logger.exception("Error calling role operation") return (False, "Error calling role operation: " + str(ex)) @@ -972,7 +972,7 @@ def alterVM(self, vm, radl, auth_data): headers = {'x-ms-version': '2013-11-01', 'Content-Type': 'application/xml'} resp = self.create_request('PUT', uri, auth_data, headers, body) - except Exception, ex: + except Exception as ex: self.logger.exception("Error calling update operation") return (False, "Error calling update operation: " + str(ex)) diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index 4861b556c..417b14701 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -23,7 +23,7 @@ from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from IM.config import Config -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import Feature from IM import UnixHTTPAdapter @@ -75,15 +75,15 @@ def get_user_cert_data(self, auth): """ certificate = auth['cert'] fd, cert_file = tempfile.mkstemp() - os.write(fd, certificate) + os.write(fd, certificate.encode()) os.close(fd) - os.chmod(cert_file, 0644) + os.chmod(cert_file, 0o644) private_key = auth['key'] fd, key_file = tempfile.mkstemp() - os.write(fd, private_key) + os.write(fd, private_key.encode()) os.close(fd) - os.chmod(key_file, 0600) + os.chmod(key_file, 0o600) return (cert_file, key_file) @@ -330,7 +330,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res.append((True, vm)) - except Exception, ex: + except Exception as ex: self.logger.exception("Error connecting with Docker server") res.append((False, "ERROR: " + str(ex))) @@ -357,7 +357,7 @@ def updateVMInfo(self, vm, auth_data): self.setIPs(vm, output) return (True, vm) - except Exception, ex: + except Exception as ex: self.logger.exception("Error connecting with Docker server") self.logger.error(ex) return (False, "Error connecting with Docker server") diff --git a/IM/connectors/Dummy.py b/IM/connectors/Dummy.py index 6e6a5b6f7..f3674fd18 100644 --- a/IM/connectors/Dummy.py +++ b/IM/connectors/Dummy.py @@ -16,7 +16,7 @@ import time from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import Feature diff --git a/IM/connectors/EC2.py b/IM/connectors/EC2.py index a2832909d..23b3c5635 100644 --- a/IM/connectors/EC2.py +++ b/IM/connectors/EC2.py @@ -21,13 +21,13 @@ try: import boto.ec2 import boto.vpc -except Exception, ex: - print "WARN: Boto library not correctly installed. EC2CloudConnector will not work!." - print ex +except Exception as ex: + print("WARN: Boto library not correctly installed. EC2CloudConnector will not work!.") + print(ex) from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import Feature @@ -184,7 +184,7 @@ def get_connection(self, region_name, auth_data): raise Exception("No correct auth data has been specified to EC2: " "username (Access Key) and password (Secret Key)") - except Exception, ex: + except Exception as ex: self.logger.exception( "Error getting the region " + region_name) raise Exception("Error getting the region " + @@ -338,7 +338,7 @@ def create_security_group(self, conn, inf, radl, vpc=None): try: sg = conn.create_security_group( sg_name, "Security group created by the IM", vpc_id=vpc) - except Exception, crex: + except Exception as crex: # First check if the SG does exist sg = self._get_security_group(conn, sg_name) if not sg: @@ -378,12 +378,12 @@ def create_security_group(self, conn, inf, radl, vpc=None): sg.authorize('tcp', 0, 65535, src_group=sg) sg.authorize('udp', 0, 65535, src_group=sg) # sg.authorize('icmp', 0, 65535, src_group=sg) - except Exception, addex: + except Exception as addex: self.logger.warn( "Exception adding SG rules. Probably the rules exists:" + str(addex)) pass - except Exception, ex: + except Exception as ex: self.logger.exception("Error Creating the Security group") if vpc: raise Exception( @@ -418,7 +418,7 @@ def create_keypair(self, system, conn): keypair = conn.create_key_pair(keypair_name) created = True keypair.save(self.KEYPAIR_DIR) - os.chmod(keypair_file, 0400) + os.chmod(keypair_file, 0o400) with open(keypair_file, "r") as fkeypair: system.setUserKeyCredentials(system.getCredentials().username, None, fkeypair.read()) @@ -500,7 +500,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): return res else: block_device_name = None - for name, device in image.block_device_mapping.iteritems(): + for name, device in image.block_device_mapping.items(): if device.snapshot_id or device.volume_id: block_device_name = name @@ -676,7 +676,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res.append( (False, "Error launching the image")) - except Exception, ex: + except Exception as ex: self.logger.exception("Error launching instance.") res.append( (False, "Error launching the instance: " + str(ex))) @@ -791,7 +791,7 @@ def delete_volumes(self, conn, volumes, instance_id, timeout=240): else: self.logger.debug( "State: " + str(curr_vol.attachment_state())) - except Exception, ex: + except Exception as ex: self.logger.warn("Error removing the volume: " + str(ex)) if not deleted: @@ -1039,7 +1039,7 @@ def updateVMInfo(self, vm, auth_data): im_username = auth_data.getAuthInfo( 'InfrastructureManager')[0]['username'] instance.add_tag("IM-USER", im_username) - except Exception, ex: + except Exception as ex: self.logger.exception( "Error updating the instance " + instance_id) return (False, "Error updating the instance " + instance_id + ": " + str(ex)) @@ -1063,7 +1063,7 @@ def updateVMInfo(self, vm, auth_data): try: vm.info.systems[0].setValue('launch_time', int(time.mktime( time.strptime(instance.launch_time[:19], '%Y-%m-%dT%H:%M:%S')))) - except Exception, ex: + except Exception as ex: self.logger.warn( "Error setting the launch_time of the instance. Probably the instance is not running:" + str(ex)) @@ -1177,7 +1177,7 @@ def delete_security_group(self, conn, inf, timeout=90): sg.revoke('tcp', 0, 65535, src_group=sg) sg.revoke('udp', 0, 65535, src_group=sg) time.sleep(2) - except Exception, ex: + except Exception as ex: self.logger.warn( "Error revoking self rules: " + str(ex)) @@ -1188,7 +1188,7 @@ def delete_security_group(self, conn, inf, timeout=90): try: sg.delete() deleted = True - except Exception, ex: + except Exception as ex: # Check if it has been deleted yet sg = self._get_security_group(conn, sg_name) if not sg: diff --git a/IM/connectors/FogBow.py b/IM/connectors/FogBow.py index 5423a4501..0ff597506 100644 --- a/IM/connectors/FogBow.py +++ b/IM/connectors/FogBow.py @@ -17,10 +17,15 @@ import json import os import sys -import httplib + +try: + from httplib import HTTPSConnection, HTTPConnection +except ImportError: + from http.client import HTTPSConnection, HTTPConnection + from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import Feature @@ -59,9 +64,9 @@ def get_http_connection(self): """ if self.cloud.protocol == 'https': - conn = httplib.HTTPSConnection(self.cloud.server, self.cloud.port) + conn = HTTPSConnection(self.cloud.server, self.cloud.port) else: - conn = httplib.HTTPConnection(self.cloud.server, self.cloud.port) + conn = HTTPConnection(self.cloud.server, self.cloud.port) return conn @@ -264,7 +269,7 @@ def updateVMInfo(self, vm, auth_data): return (True, vm) - except Exception, ex: + except Exception as ex: self.logger.exception("Error connecting with FogBow Manager") return (False, "Error connecting with FogBow Manager: " + str(ex)) @@ -301,7 +306,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): conn.putheader('Content-Type', 'text/occi') # conn.putheader('Accept', 'text/occi') if auth_headers: - for k, v in auth_headers.iteritems(): + for k, v in auth_headers.items(): conn.putheader(k, v) conn.putheader( @@ -363,7 +368,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm.info.systems[0].setValue('instance_id', str(vm.id)) res.append((True, vm)) - except Exception, ex: + except Exception as ex: self.logger.exception("Error connecting with FogBow manager") res.append((False, "ERROR: " + str(ex))) @@ -446,7 +451,7 @@ def getIdentityPlugin(identity_type): raise Exception("Not valid Identity Plugin.") try: return getattr(sys.modules[__name__], identity_type + "IdentityPlugin")() - except Exception, ex: + except Exception as ex: raise Exception("IdentityPlugin not supported: %s (error: %s)" % ( identity_type, str(ex))) @@ -499,7 +504,7 @@ def create_token(params): server = uri[1].split(":")[0] port = int(uri[1].split(":")[1]) - conn = httplib.HTTPSConnection(server, port) + conn = HTTPSConnection(server, port) conn.putrequest('POST', "/v2.0/tokens") conn.putheader('Accept', 'application/json') conn.putheader('Content-Type', 'application/json') diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 5ab34e31f..56ec3de94 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -22,11 +22,11 @@ from libcloud.compute.types import NodeState, Provider from libcloud.compute.providers import get_driver from libcloud.common.google import ResourceNotFoundError -except Exception, ex: - print "WARN: libcloud library not correctly installed. GCECloudConnector will not work!." - print ex +except Exception as ex: + print("WARN: libcloud library not correctly installed. GCECloudConnector will not work!.") + print(ex) -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from radl.radl import Feature @@ -338,7 +338,7 @@ def create_firewall(self, inf, net_name, radl, driver): try: driver.ex_create_firewall(firewall_name, allowed, network=net_name) - except Exception, addex: + except Exception as addex: self.logger.warn("Exception creating FW: " + str(addex)) pass @@ -443,7 +443,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): def finalize(self, vm, auth_data): try: node = self.get_node_with_id(vm.id, auth_data) - except Exception, ex: + except Exception as ex: self.logger.exception("Error getting VM: %s. Err: %s." % (vm.id, str(ex))) return (False, "Error getting VM: %s. Err: %s." % (vm.id, str(ex))) @@ -617,7 +617,7 @@ def updateVMInfo(self, vm, auth_data): node = driver.ex_get_node(vm.id) except ResourceNotFoundError: self.logger.warn("VM " + str(vm.id) + " does not exist.") - except Exception, ex: + except Exception as ex: self.logger.exception("Error getting VM info: %s" % vm.id) return (False, "Error getting VM info: %s. %s" % (vm.id, str(ex))) @@ -656,13 +656,13 @@ def start(self, vm, auth_data): node = driver.ex_get_node(vm.id) except ResourceNotFoundError: return (False, "VM " + str(vm.id) + " does not exist.") - except Exception, ex: + except Exception as ex: self.logger.exception("Error getting VM %s" % vm.id) return (False, "Error getting VM %s: %s" % (vm.id, str(ex))) try: driver.ex_start_node(node) - except Exception, ex: + except Exception as ex: self.logger.exception("Error starting VM %s" % vm.id) return (False, "Error starting VM %s: %s" % (vm.id, str(ex))) @@ -675,13 +675,13 @@ def stop(self, vm, auth_data): node = driver.ex_get_node(vm.id) except ResourceNotFoundError: return (False, "VM " + str(vm.id) + " does not exist.") - except Exception, ex: + except Exception as ex: self.logger.exception("Error getting VM %s" % vm.id) return (False, "Error getting VM %s: %s" % (vm.id, str(ex))) try: driver.ex_stop_node(node) - except Exception, ex: + except Exception as ex: self.logger.exception("Error stopping VM %s" % vm.id) return (False, "Error stopping VM %s: %s" % (vm.id, str(ex))) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index d3d43e646..c98b7bd87 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -21,7 +21,7 @@ import requests from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import Feature from IM.config import Config @@ -366,7 +366,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res.append((True, vm)) - except Exception, ex: + except Exception as ex: self.logger.exception( "Error connecting with Kubernetes API server") res.append((False, "ERROR: " + str(ex))) @@ -388,7 +388,7 @@ def _get_pod(self, vm_id, auth_data): else: return (False, resp.status, resp.text) - except Exception, ex: + except Exception as ex: self.logger.exception( "Error connecting with Kubernetes API server") return (False, None, "Error connecting with Kubernetes API server: " + str(ex)) @@ -521,7 +521,7 @@ def alterVM(self, vm, radl, auth_data): Feature("memory.size", "=", new_memory, 'B'), conflict="other", missing="other") return (True, self.updateVMInfo(vm, auth_data)) - except Exception, ex: + except Exception as ex: self.logger.exception( "Error connecting with Kubernetes API server") return (False, "ERROR: " + str(ex)) diff --git a/IM/connectors/LibCloud.py b/IM/connectors/LibCloud.py index 54d78b241..416d36564 100644 --- a/IM/connectors/LibCloud.py +++ b/IM/connectors/LibCloud.py @@ -20,13 +20,13 @@ from libcloud.compute.base import NodeImage, NodeAuthSSHKey from libcloud.compute.types import Provider, NodeState from libcloud.compute.providers import get_driver -except Exception, ex: - print "WARN: libcloud library not correctly installed. LibCloudCloudConnector will not work!." - print ex +except Exception as ex: + print("WARN: libcloud library not correctly installed. LibCloudCloudConnector will not work!.") + print(ex) from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import Feature @@ -61,7 +61,7 @@ def get_driver(self, auth_data): MAP = {"username": "key", "password": "secret"} params = {} - for key, value in auth[0].iteritems(): + for key, value in auth[0].items(): if key not in ["type", "driver", "id"]: params[MAP[key]] = value @@ -530,7 +530,7 @@ def alterVM(self, vm, radl, auth_data): try: success = resize_func(node, instance_type) - except Exception, ex: + except Exception as ex: self.logger.exception("Error resizing VM.") return (False, "Error resizing VM: " + str(ex)) diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index b3045f76f..c0181eea3 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -24,7 +24,7 @@ import tempfile from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import Feature from netaddr import IPNetwork, IPAddress from IM.config import Config @@ -57,7 +57,7 @@ def create_request_static(method, url, auth, headers, body=None): proxy = auth['proxy'] (fproxy, proxy_filename) = tempfile.mkstemp() - os.write(fproxy, proxy) + os.write(fproxy, proxy.encode()) os.close(fproxy) cert = proxy_filename else: @@ -344,7 +344,7 @@ def updateVMInfo(self, vm, auth_data): self.set_disk_info(vm, resp.text, auth_data) return (True, vm) - except Exception, ex: + except Exception as ex: self.logger.exception("Error connecting with OCCI server") return (False, "Error connecting with OCCI server: " + str(ex)) @@ -526,7 +526,7 @@ def get_volume_info(self, storage_id, auth_data): return (False, resp.reason + "\n" + resp.text) else: return (True, resp.text) - except Exception, ex: + except Exception as ex: self.logger.exception("Error getting volume info") return False, str(ex) @@ -553,7 +553,7 @@ def create_volume(self, size, name, auth_data): else: occi_id = os.path.basename(resp.text) return True, occi_id - except Exception, ex: + except Exception as ex: self.logger.exception("Error creating volume") return False, str(ex) @@ -643,9 +643,9 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): if public_key: # Add user cloud init data cloud_config_str = self.get_cloud_init_data(radl) - cloud_config = self.gen_cloud_config(public_key, user, cloud_config_str) - user_data = base64.b64encode(cloud_config).replace("\n", "") - self.logger.debug("Cloud init: " + cloud_config) + cloud_config = self.gen_cloud_config(public_key, user, cloud_config_str).encode() + user_data = str(base64.b64encode(cloud_config)).replace("\n", "") + self.logger.debug("Cloud init: %s" % cloud_config) # Get the info about the OCCI server (GET /-/) occi_info = self.query_occi(auth_data) @@ -751,7 +751,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): else: res.append((False, 'Unknown Error launching the VM.')) - except Exception, ex: + except Exception as ex: self.logger.exception("Error connecting with OCCI server") res.append((False, "ERROR: " + str(ex))) for _, volume_id in volumes: @@ -791,7 +791,7 @@ def get_attached_volumes(self, vm, auth_data): if not device.endswith("vda") and not device.endswith("hda"): deleted_vols.append((link, num_storage, device)) return (True, deleted_vols) - except Exception, ex: + except Exception as ex: self.logger.exception("Error deleting volumes") return (False, "Error deleting volumes " + str(ex)) @@ -922,7 +922,7 @@ def alterVM(self, vm, radl, auth_data): else: return (False, "Error creating the new volume: " + volume_id) cont += 1 - except Exception, ex: + except Exception as ex: self.logger.exception("Error connecting with OCCI server") return (False, "Error connecting with OCCI server: " + str(ex)) @@ -989,7 +989,7 @@ def get_keystone_uri(occi, auth_data): return www_auth_head.split('=')[1].replace("'", "") else: return None - except SSLError, ex: + except SSLError as ex: occi.logger.exception( "Error with the credentials when contacting with the OCCI server.") raise Exception( @@ -1064,6 +1064,6 @@ def get_keystone_token(occi, keystone_uri, auth): break return tenant_token_id - except Exception, ex: + except Exception as ex: occi.logger.exception("Error obtaining Keystone Token.") raise Exception("Error obtaining Keystone Token: %s" % str(ex)) diff --git a/IM/connectors/OpenNebula.py b/IM/connectors/OpenNebula.py index b47ab3ad8..3530ca79a 100644 --- a/IM/connectors/OpenNebula.py +++ b/IM/connectors/OpenNebula.py @@ -15,13 +15,17 @@ # along with this program. If not, see . import hashlib -import xmlrpclib +try: + from xmlrpclib import ServerProxy +except ImportError: + from xmlrpc.client import ServerProxy + import time from IM.xmlobject import XMLObject from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine -from CloudConnector import CloudConnector +from .CloudConnector import CloudConnector from radl.radl import network, Feature from IM.config import ConfigOpenNebula from netaddr import IPNetwork, IPAddress @@ -203,7 +207,7 @@ def getSessionID(self, auth_data, hash_password=None): auth = auths[0] if 'username' in auth and 'password' in auth: - passwd = auth['password'] + passwd = auth['password'].encode('utf-8') if hash_password is None: one_ver = self.getONEVersion(auth_data) if one_ver == "2.0.0" or one_ver == "3.0.0": @@ -211,7 +215,7 @@ def getSessionID(self, auth_data, hash_password=None): if hash_password: passwd = hashlib.sha1(passwd.strip()).hexdigest() - return auth['username'] + ":" + passwd + return auth['username'] + ":" + str(passwd) else: raise Exception("No correct auth data has been specified to OpenNebula: username and password") @@ -254,7 +258,7 @@ def setIPsFromTemplate(self, vm, template): i += 1 def updateVMInfo(self, vm, auth_data): - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) session_id = self.getSessionID(auth_data) if session_id is None: @@ -318,7 +322,7 @@ def updateVMInfo(self, vm, auth_data): return (success, res_info) def launch(self, inf, radl, requested_radl, num_vm, auth_data): - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) session_id = self.getSessionID(auth_data) if session_id is None: return [(False, "Incorrect auth data, username and password must be specified for OpenNebula provider.")] @@ -352,7 +356,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): return res def finalize(self, vm, auth_data): - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) session_id = self.getSessionID(auth_data) if session_id is None: return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") @@ -371,7 +375,7 @@ def finalize(self, vm, auth_data): return (success, err) def stop(self, vm, auth_data): - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) session_id = self.getSessionID(auth_data) if session_id is None: return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") @@ -390,7 +394,7 @@ def stop(self, vm, auth_data): return (success, err) def start(self, vm, auth_data): - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) session_id = self.getSessionID(auth_data) if session_id is None: return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") @@ -511,7 +515,7 @@ def getONEVersion(self, auth_data): Returns: str with the ONE version (format: X.X.X) """ - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) version = "2.0.0" methods = server.system.listMethods() @@ -591,7 +595,7 @@ def getONENetworks(self, auth_data): Returns: a list of tuples (net_name, net_id, is_public) with the name, ID, and boolean specifying if it is a public network of the found network None if not found """ - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) session_id = self.getSessionID(auth_data) if session_id is None: return None @@ -795,7 +799,7 @@ def checkResize(self): Returns: bool, True if the one.vm.resize function appears in the ONE server or false otherwise """ - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) methods = server.system.listMethods() if "one.vm.resize" in methods: @@ -807,7 +811,7 @@ def poweroff(self, vm, auth_data, timeout=60): """ Poweroff the VM and waits for it to be in poweredoff state """ - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) session_id = self.getSessionID(auth_data) if session_id is None: return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") @@ -874,7 +878,7 @@ def alterVM(self, vm, radl, auth_data): return (True, "") def attach_volume(self, vm, disk_size, disk_device, disk_fstype, session_id): - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) disk_temp = ''' DISK = [ @@ -932,7 +936,7 @@ def attach_new_disks(self, vm, system, session_id): return (True, "") def alter_mem_cpu(self, vm, system, session_id, auth_data): - server = xmlrpclib.ServerProxy(self.server_url, allow_none=True) + server = ServerProxy(self.server_url, allow_none=True) cpu = vm.info.systems[0].getValue('cpu.count') memory = vm.info.systems[0].getFeature('memory.size').getValue('M') diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index e503b0d0a..cca8d0481 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -21,9 +21,9 @@ from libcloud.compute.types import Provider, NodeState from libcloud.compute.providers import get_driver from libcloud.compute.base import NodeImage, NodeAuthSSHKey -except Exception, ex: - print "WARN: libcloud library not correctly installed. OpenStackCloudConnector will not work!." - print ex +except Exception as ex: + print("WARN: libcloud library not correctly installed. OpenStackCloudConnector will not work!.") + print(ex) from IM.connectors.LibCloud import LibCloudCloudConnector from IM.config import Config @@ -430,7 +430,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): msg = "Error creating the node. " try: node = driver.create_node(**args) - except Exception, ex: + except Exception as ex: msg += str(ex) if node: @@ -628,7 +628,7 @@ def create_security_group(self, driver, inf, radl): try: driver.ex_create_security_group_rule( sg, protocol, remote_port, remote_port, '0.0.0.0/0') - except Exception, ex: + except Exception as ex: self.logger.warn( "Exception adding SG rules: " + str(ex)) @@ -640,7 +640,7 @@ def create_security_group(self, driver, inf, radl): sg, 'tcp', 1, 65535, source_security_group=sg) driver.ex_create_security_group_rule( sg, 'udp', 1, 65535, source_security_group=sg) - except Exception, addex: + except Exception as addex: self.logger.warn( "Exception adding SG rules. Probably the rules exists:" + str(addex)) pass @@ -713,7 +713,7 @@ def delete_security_group(self, node, sgs, inf, vm_id, timeout=60): try: node.driver.ex_delete_security_group(sg) deleted = True - except Exception, ex: + except Exception as ex: # Check if it has been deleted yet sg = self._get_security_group(node.driver, sg.name) if not sg: diff --git a/IM/db.py b/IM/db.py index abcb01b15..e5d10c155 100644 --- a/IM/db.py +++ b/IM/db.py @@ -148,7 +148,7 @@ def _execute_retry(self, sql, args, fetch=False): res = True return res # If the operational error is db lock, retry - except sqlite.OperationalError, ex: + except sqlite.OperationalError as ex: if str(ex).lower() == 'database is locked': retries_cont += 1 # release the connection @@ -158,7 +158,7 @@ def _execute_retry(self, sql, args, fetch=False): self.connect() else: raise ex - except sqlite.IntegrityError, ex: + except sqlite.IntegrityError as ex: raise IntegrityError() def execute(self, sql, args=None): diff --git a/IM/ganglia.py b/IM/ganglia.py index 526ee3cdf..469643f49 100644 --- a/IM/ganglia.py +++ b/IM/ganglia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import socket -from xmlobject import XMLObject +from IM.xmlobject import XMLObject class EXTRA_ELEMENT(XMLObject): @@ -70,7 +70,7 @@ def update_ganglia_info(inf): s.settimeout(2) if(s.connect_ex((master_ip, ganglia_info.ganglia_port)) == 0): port_open = True - except Exception, ex: + except Exception as ex: return (False, "Error connecting to ganglia: " + str(ex)) else: return (False, "VM master without public IP") @@ -137,7 +137,7 @@ def update_ganglia_info(inf): vm.info.systems[0].setValue( "swap", float_val, metric.UNITS) - except Exception, ex: + except Exception as ex: return (False, "Error getting ganglia information: " + str(ex)) return (True, "") diff --git a/IM/recipe.py b/IM/recipe.py index 02818869d..cff5daee7 100644 --- a/IM/recipe.py +++ b/IM/recipe.py @@ -14,9 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from db import DataBase +from IM.db import DataBase -from config import Config +from IM.config import Config from radl.radl import FeaturesApp diff --git a/IM/request.py b/IM/request.py index c65d3ee56..8807445b5 100644 --- a/IM/request.py +++ b/IM/request.py @@ -15,11 +15,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . import sys -from queue import Queue, Empty import threading -from xmlrpc.server import SimpleXMLRPCServer -import socketserver import time + +try: + from Queue import Queue, Empty +except ImportError: + from queue import Queue, Empty +try: + from SimpleXMLRPCServer import SimpleXMLRPCServer +except ImportError: + from xmlrpc.server import SimpleXMLRPCServer +try: + from SocketServer import ThreadingMixIn +except ImportError: + from socketserver import ThreadingMixIn + from IM.timedcall import TimedCall from IM.config import Config @@ -248,7 +259,7 @@ def process(self): self.__thread.start() -class AsyncXMLRPCServer(socketserver.ThreadingMixIn, SimpleXMLRPCServer): +class AsyncXMLRPCServer(ThreadingMixIn, SimpleXMLRPCServer): def serve_forever_in_thread(self): """ @@ -266,7 +277,7 @@ def serve_forever_in_thread(self): if Config.XMLRCP_SSL: from springpython.remoting.xmlrpc import SSLServer - class AsyncSSLXMLRPCServer(socketserver.ThreadingMixIn, SSLServer): + class AsyncSSLXMLRPCServer(ThreadingMixIn, SSLServer): def __init__(self, *args, **kwargs): super(AsyncSSLXMLRPCServer, self).__init__(*args, **kwargs) diff --git a/IM/retry.py b/IM/retry.py index 87a655ad4..75c0da672 100644 --- a/IM/retry.py +++ b/IM/retry.py @@ -31,14 +31,14 @@ def f_retry(*args, **kwargs): while mtries > 1: try: return f(*args, **kwargs) - except ExceptionToCheck, e: + except ExceptionToCheck as e: if not quiet: msg = "%s, Retrying in %d seconds..." % ( str(e), mdelay) if logger: logger.warning(msg) else: - print msg + print(msg) time.sleep(mdelay) mtries -= 1 mdelay *= backoff diff --git a/setup.py b/setup.py index 16b786782..5a81f2535 100644 --- a/setup.py +++ b/setup.py @@ -58,5 +58,6 @@ install_requires=["ansible >= 1.8", "paramiko >= 1.14", "PyYAML", "suds-py3", "boto >= 2.29", "apache-libcloud >= 0.17", "RADL", "bottle", "netaddr", "requests", "scp", "cherrypy", "mysqlclient", - "azure-common", "azure-mgmt-storage", "azure-mgmt-compute", "azure-mgmt-network", "azure-mgmt-resource"] + "azure-common", "msrest", "msrestazure", "azure-mgmt-storage", + "azure-mgmt-compute", "azure-mgmt-network", "azure-mgmt-resource"] ) diff --git a/test/functional/test_im.py b/test/functional/test_im.py index a46c7935b..97198b32a 100755 --- a/test/functional/test_im.py +++ b/test/functional/test_im.py @@ -54,9 +54,9 @@ def __init__(self, *args): def setUp(self): + Config.DATA_DB = "/tmp/inf.dat" + InfrastructureList.load_data() IM._reinit() - # Patch save_data - IM.save_data = staticmethod(lambda *args: None) ch = logging.StreamHandler(sys.stdout) log = logging.getLogger('InfrastructureManager') diff --git a/test/integration/QuickTestIM.py b/test/integration/QuickTestIM.py index 2ad3b0c34..9c243762c 100755 --- a/test/integration/QuickTestIM.py +++ b/test/integration/QuickTestIM.py @@ -148,7 +148,7 @@ def test_12_getradl(self): success, msg="ERROR calling GetInfrastructureRADL: " + str(res)) try: radl_parse.parse_radl(res) - except Exception, ex: + except Exception as ex: self.assertTrue( False, msg="ERROR parsing the RADL returned by GetInfrastructureRADL: " + str(ex)) @@ -186,7 +186,7 @@ def test_15_get_vm_info(self): self.assertTrue(success, msg="ERROR calling GetVMInfo: " + str(info)) try: radl_parse.parse_radl(info) - except Exception, ex: + except Exception as ex: self.assertTrue( False, msg="ERROR parsing the RADL returned by GetVMInfo: " + str(ex)) @@ -254,7 +254,7 @@ def test_20_getstate(self): vm_states = res['vm_states'] self.assertEqual(len(vm_states), 3, msg="ERROR getting infrastructure state: Incorrect number of VMs(" + str(len(vm_states)) + "). It must be 3") - for vm_id, vm_state in vm_states.iteritems(): + for vm_id, vm_state in vm_states.items(): self.assertEqual(vm_state, "configured", msg="Unexpected vm state: " + vm_state + " in VM ID " + str(vm_id) + ". It must be 'configured'.") diff --git a/test/integration/TestIM.py b/test/integration/TestIM.py index 95430f9b3..5cd5df2a8 100755 --- a/test/integration/TestIM.py +++ b/test/integration/TestIM.py @@ -156,7 +156,7 @@ def test_12_getradl(self): success, msg="ERROR calling GetInfrastructureRADL: " + str(res)) try: radl_parse.parse_radl(res) - except Exception, ex: + except Exception as ex: self.assertTrue( False, msg="ERROR parsing the RADL returned by GetInfrastructureRADL: " + str(ex)) @@ -194,7 +194,7 @@ def test_15_get_vm_info(self): self.assertTrue(success, msg="ERROR calling GetVMInfo: " + str(info)) try: radl_parse.parse_radl(info) - except Exception, ex: + except Exception as ex: self.assertTrue( False, msg="ERROR parsing the RADL returned by GetVMInfo: " + str(ex)) @@ -261,7 +261,7 @@ def test_20_getstate(self): vm_states = res['vm_states'] self.assertEqual(len(vm_states), 4, msg="ERROR getting infrastructure state: Incorrect number of VMs(" + str(len(vm_states)) + "). It must be 4") - for vm_id, vm_state in vm_states.iteritems(): + for vm_id, vm_state in vm_states.items(): self.assertEqual(vm_state, "configured", msg="Unexpected vm state: " + vm_state + " in VM ID " + str(vm_id) + ". It must be 'configured'.") diff --git a/test/integration/TestREST.py b/test/integration/TestREST.py index 28c11d794..16a0af9a6 100755 --- a/test/integration/TestREST.py +++ b/test/integration/TestREST.py @@ -276,7 +276,7 @@ def test_34_get_radl(self): msg="ERROR getting the infrastructure RADL:" + output) try: radl_parse.parse_radl(output) - except Exception, ex: + except Exception as ex: self.assertTrue( False, msg="ERROR parsing the RADL returned by GetInfrastructureRADL: " + str(ex)) @@ -340,7 +340,7 @@ def test_45_getstate(self): vm_states = res['state']['vm_states'] self.assertEqual(state, "configured", msg="Unexpected inf state: " + state + ". It must be 'configured'.") - for vm_id, vm_state in vm_states.iteritems(): + for vm_id, vm_state in vm_states.items(): self.assertEqual(vm_state, "configured", msg="Unexpected vm state: " + vm_state + " in VM ID " + str(vm_id) + ". It must be 'configured'.") diff --git a/test/loadtest/LoadTestREST.py b/test/loadtest/LoadTestREST.py index b9874f65e..020dcae83 100755 --- a/test/loadtest/LoadTestREST.py +++ b/test/loadtest/LoadTestREST.py @@ -403,7 +403,7 @@ def test_45_getstate(self): vm_states = res['state']['vm_states'] self.assertEqual(state, "configured", msg="Unexpected inf state: " + state + ". It must be 'configured'.") - for vm_id, vm_state in vm_states.iteritems(): + for vm_id, vm_state in vm_states.items(): self.assertEqual(vm_state, "configured", msg="Unexpected vm state: " + vm_state + " in VM ID " + str(vm_id) + ". It must be 'configured'.") diff --git a/test/unit/SSH.py b/test/unit/SSH.py index e9ec4c560..d3f69ae90 100755 --- a/test/unit/SSH.py +++ b/test/unit/SSH.py @@ -176,7 +176,7 @@ def test_sftp_chmod(self, from_transport, ssh_client): client = MagicMock() from_transport.return_value = client - res = ssh.sftp_chmod("some_file", 0644) + res = ssh.sftp_chmod("some_file", 0o644) self.assertTrue(res) diff --git a/test/unit/connectors/Azure.py b/test/unit/connectors/Azure.py index 0422fb78f..5a7c077d5 100755 --- a/test/unit/connectors/Azure.py +++ b/test/unit/connectors/Azure.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/connectors/AzureClassic.py b/test/unit/connectors/AzureClassic.py index b43bd59a5..498d6036f 100644 --- a/test/unit/connectors/AzureClassic.py +++ b/test/unit/connectors/AzureClassic.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/connectors/Docker.py b/test/unit/connectors/Docker.py index ccf1f7297..83b316d55 100755 --- a/test/unit/connectors/Docker.py +++ b/test/unit/connectors/Docker.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/connectors/EC2.py b/test/unit/connectors/EC2.py index 12cddc764..f7f7f792a 100755 --- a/test/unit/connectors/EC2.py +++ b/test/unit/connectors/EC2.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/connectors/Fogbow.py b/test/unit/connectors/Fogbow.py index 1bec7c10a..c19e56334 100755 --- a/test/unit/connectors/Fogbow.py +++ b/test/unit/connectors/Fogbow.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") @@ -126,7 +129,7 @@ def get_response(self): def request(self, method, url, body=None, headers={}): self.__class__.last_op = method, url - @patch('httplib.HTTPConnection') + @patch('IM.connectors.FogBow.HTTPConnection') def test_20_launch(self, connection): radl_data = """ network net1 (outbound = 'yes' and outports = '8080') @@ -164,7 +167,7 @@ def test_20_launch(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') + @patch('IM.connectors.FogBow.HTTPConnection') def test_30_updateVMInfo(self, connection): radl_data = """ network net (outbound = 'yes') @@ -201,7 +204,7 @@ def test_30_updateVMInfo(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') + @patch('IM.connectors.FogBow.HTTPConnection') def test_60_finalize(self, connection): auth = Authentication([{'id': 'fogbow', 'type': 'FogBow', 'proxy': 'user', 'host': 'server.com:8182'}]) fogbow_cloud = self.get_fogbow_cloud() diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index 2d2b35ed2..9a2d451b7 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 6941eece1..0a4002cb5 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/connectors/LibCloud.py b/test/unit/connectors/LibCloud.py index d30da9ced..3994bda21 100755 --- a/test/unit/connectors/LibCloud.py +++ b/test/unit/connectors/LibCloud.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/connectors/OCCI.py b/test/unit/connectors/OCCI.py index 0398e524e..34fbd27d0 100755 --- a/test/unit/connectors/OCCI.py +++ b/test/unit/connectors/OCCI.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/connectors/OpenNebula.py b/test/unit/connectors/OpenNebula.py index dc5eae7d6..cad112df4 100755 --- a/test/unit/connectors/OpenNebula.py +++ b/test/unit/connectors/OpenNebula.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") @@ -99,7 +102,7 @@ def test_10_concrete(self): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('xmlrpclib.ServerProxy') + @patch('IM.connectors.OpenNebula.ServerProxy') @patch('IM.connectors.OpenNebula.OpenNebulaCloudConnector.getONEVersion') def test_20_launch(self, getONEVersion, server_proxy): radl_data = """ @@ -139,7 +142,7 @@ def test_20_launch(self, getONEVersion, server_proxy): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('xmlrpclib.ServerProxy') + @patch('IM.connectors.OpenNebula.ServerProxy') def test_30_updateVMInfo(self, server_proxy): radl_data = """ network net () @@ -175,7 +178,7 @@ def test_30_updateVMInfo(self, server_proxy): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('xmlrpclib.ServerProxy') + @patch('IM.connectors.OpenNebula.ServerProxy') def test_40_stop(self, server_proxy): auth = Authentication([{'id': 'one', 'type': 'OpenNebula', 'username': 'user', 'password': 'pass', 'host': 'server.com:2633'}]) @@ -195,7 +198,7 @@ def test_40_stop(self, server_proxy): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('xmlrpclib.ServerProxy') + @patch('IM.connectors.OpenNebula.ServerProxy') def test_50_start(self, server_proxy): auth = Authentication([{'id': 'one', 'type': 'OpenNebula', 'username': 'user', 'password': 'pass', 'host': 'server.com:2633'}]) @@ -215,7 +218,7 @@ def test_50_start(self, server_proxy): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('xmlrpclib.ServerProxy') + @patch('IM.connectors.OpenNebula.ServerProxy') @patch('IM.connectors.OpenNebula.OpenNebulaCloudConnector.checkResize') def test_55_alter(self, checkResize, server_proxy): radl_data = """ @@ -266,7 +269,7 @@ def test_55_alter(self, checkResize, server_proxy): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('xmlrpclib.ServerProxy') + @patch('IM.connectors.OpenNebula.ServerProxy') def test_60_finalize(self, server_proxy): auth = Authentication([{'id': 'one', 'type': 'OpenNebula', 'username': 'user', 'password': 'pass', 'host': 'server.com:2633'}]) diff --git a/test/unit/connectors/OpenStack.py b/test/unit/connectors/OpenStack.py index 6d7392e59..9514fb3c5 100755 --- a/test/unit/connectors/OpenStack.py +++ b/test/unit/connectors/OpenStack.py @@ -21,7 +21,10 @@ import os import logging import logging.config -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO sys.path.append(".") sys.path.append("..") diff --git a/test/unit/test_ansible.py b/test/unit/test_ansible.py index 04695ee3c..c16125931 100755 --- a/test/unit/test_ansible.py +++ b/test/unit/test_ansible.py @@ -19,7 +19,10 @@ import unittest import os from multiprocessing import Queue -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO import time from IM.ansible_utils.ansible_launcher import AnsibleThread @@ -44,7 +47,7 @@ def test_ansible_thread(self): self.assertEqual(return_code, 0) self.assertIn("failed=0", output.getvalue()) self.assertIn("changed=2", output.getvalue()) - print output.getvalue() + print(output.getvalue()) if __name__ == '__main__': unittest.main() diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py index 2d9f65361..f82b9a798 100755 --- a/test/unit/test_im_logic.py +++ b/test/unit/test_im_logic.py @@ -120,7 +120,7 @@ def test_inf_creation0(self): IM.DestroyInfrastructure(infId, auth0) def test_inf_creation1(self): - """Create infrastructure with empty RADL.""" + """Create infrastructure with an incorrect RADL with 2 clouds.""" radl = """" network publica (outbound = 'yes') @@ -160,6 +160,55 @@ def test_inf_creation1(self): " are asked to be deployed in different cloud providers", str(ex.exception)) + def test_inf_creation_addition_clouds(self): + """Add resources infrastructure with an incorrect RADL with 2 clouds.""" + + radl = """" + network publica (outbound = 'yes') + network privada () + + system front ( + cpu.arch='x86_64' and + cpu.count>=1 and + memory.size>=512m and + net_interface.0.connection = 'publica' and + net_interface.1.connection = 'privada' and + disk.0.image.url = 'mock0://linux.for.ev.er' and + disk.0.os.credentials.username = 'ubuntu' and + disk.0.os.credentials.password = 'yoyoyo' and + disk.0.os.name = 'linux' + ) + + system wn ( + cpu.arch='x86_64' and + cpu.count>=1 and + memory.size>=512m and + net_interface.0.connection = 'privada' and + disk.0.image.url = 'mock0://linux.for.ev.er' and + disk.0.os.credentials.username = 'ubuntu' and + disk.0.os.credentials.password = 'yoyoyo' and + disk.0.os.name = 'linux' + ) + + deploy front 1 cloud0 + deploy wn 1 + """ + + auth0 = self.getAuth([0], [], [("Dummy", 0), ("Dummy", 1)]) + infId = IM.CreateInfrastructure(radl, auth0) + + radl = """ + network privada + system wn + deploy wn 1 cloud1 + """ + + with self.assertRaises(Exception) as ex: + _ = IM.AddResource(infId, radl, auth0) + self.assertIn("Two deployments that have to be launched in the same cloud provider" + " are asked to be deployed in different cloud providers", + str(ex.exception)) + def test_inf_auth(self): """Try to access not owned Infs.""" @@ -191,7 +240,7 @@ def test_inf_addresources_without_credentials(self): with self.assertRaises(Exception) as ex: IM.AddResource(infId, str(radl), auth0) - self.assertIn("No username", ex.exception.message) + self.assertIn("No username", str(ex.exception)) IM.DestroyInfrastructure(infId, auth0) @@ -238,7 +287,7 @@ def test_inf_addresources1(self): """Deploy n independent virtual machines.""" n = 20 # Machines to deploy - Config.MAX_SIMULTANEOUS_LAUNCHES = n / 2 # Test the pool + Config.MAX_SIMULTANEOUS_LAUNCHES = int(n / 2) # Test the pool radl = RADL() radl.add(system("s0", [Feature("disk.0.image.url", "=", "mock0://linux.for.ev.er"), Feature("disk.0.os.credentials.username", "=", "user"),