From 39bb7edd064e455241986eeebe994797c88c53f6 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 08:48:20 +0100 Subject: [PATCH 01/28] Move Docker conn to requests --- IM/UnixHTTPConnection.py | 30 --------- IM/connectors/Docker.py | 141 +++++++++++++-------------------------- 2 files changed, 48 insertions(+), 123 deletions(-) delete mode 100644 IM/UnixHTTPConnection.py diff --git a/IM/UnixHTTPConnection.py b/IM/UnixHTTPConnection.py deleted file mode 100644 index 5845481d0..000000000 --- a/IM/UnixHTTPConnection.py +++ /dev/null @@ -1,30 +0,0 @@ -# IM - Infrastructure Manager -# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import httplib -import socket - - -class UnixHTTPConnection(httplib.HTTPConnection): - - def __init__(self, path, host='localhost', port=None, strict=None, timeout=None): - httplib.HTTPConnection.__init__(self, host, port=port, strict=strict, timeout=timeout) - self.path = path - - def connect(self): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(self.path) - self.sock = sock diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index d449b9271..3af098563 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -18,13 +18,13 @@ import tempfile import json import socket -import httplib +import requests from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from IM.config import Config from CloudConnector import CloudConnector from radl.radl import Feature -from IM import UnixHTTPConnection +from IM import UnixHTTPAdapter class DockerCloudConnector(CloudConnector): @@ -41,53 +41,30 @@ class DockerCloudConnector(CloudConnector): _root_password = "Aspecial+0ne" """ Default password to set to the root in the container""" - def __init__(self, cloud_info): - self.cert_file = '' - self.key_file = '' - CloudConnector.__init__(self, cloud_info) + def create_request(self, method, url, auth_data, headers=None, body=None): - def get_http_connection(self, auth_data): - """ - Get the HTTPConnection object to contact the Docker API - - Arguments: - - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider. - Returns(HTTPConnection or HTTPSConnection): HTTPConnection connection object - """ - - self.cert_file or os.path.isfile(self.cert_file) - - auths = auth_data.getAuthInfo( - DockerCloudConnector.type, self.cloud.server) + auths = auth_data.getAuthInfo(DockerCloudConnector.type, self.cloud.server) if not auths: - self.logger.error( - "No correct auth data has been specified to Docker.") + self.logger.error("No correct auth data has been specified to Docker.") return None else: auth = auths[0] if self.cloud.protocol == 'unix': - socket_path = "/" + self.cloud.server - conn = UnixHTTPConnection.UnixHTTPConnection(socket_path) - elif self.cloud.protocol == 'https': + url = "http+unix://" + self.cloud.server + url + session = requests.Session() + session.mount('http+unix://', UnixHTTPAdapter.UnixHTTPAdapter()) + resp = session.request(method, url, verify=False, headers=headers, data=body) + else: + url = "%s://%s:%d%s" % (self.cloud.protocol, self.cloud.server, self.cloud.port, url) if 'cert' in auth and 'key' in auth: - if os.path.isfile(self.cert_file) and os.path.isfile(self.key_file): - cert_file = self.cert_file - key_file = self.key_file - else: - cert_file, key_file = self.get_user_cert_data(auth) - self.cert_file = cert_file - self.key_file = key_file - conn = httplib.HTTPSConnection( - self.cloud.server, self.cloud.port, cert_file=cert_file, key_file=key_file) + cert = self.get_user_cert_data(auth) else: - conn = httplib.HTTPSConnection( - self.cloud.server, self.cloud.port) - elif self.cloud.protocol == 'http' or not self.cloud.protocol: - self.logger.warn("Using a unsecure connection to docker API!") - conn = httplib.HTTPConnection(self.cloud.server, self.cloud.port) + cert = None + + resp = requests.request(method, url, verify=False, cert=cert, headers=headers, data=body) - return conn + return resp def get_user_cert_data(self, auth): """ @@ -273,7 +250,6 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): if public_net: outports = public_net.getOutPorts() - conn = self.get_http_connection(auth_data) res = [] i = 0 while i < num_vm: @@ -287,28 +263,20 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): DockerCloudConnector._port_counter += 1 # Create the VM to get the nodename - vm = VirtualMachine(inf, None, self.cloud, - radl, requested_radl, self) + vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self) # Create the container - conn.putrequest('POST', "/containers/create") - conn.putheader('Content-Type', 'application/json') - - cont_data = self._generate_create_request_data( - outports, system, vm, ssh_port) + cont_data = self._generate_create_request_data(outports, system, vm, ssh_port) body = json.dumps(cont_data) + + headers = {'Accept': 'application/json'} + resp = self.create_request('POST', self.cloud.path + "/containers/create", auth_data, headers, body) - conn.putheader('Content-Length', len(body)) - conn.endheaders(body) - - resp = conn.getresponse() - output = resp.read() - if resp.status != 201: - res.append( - (False, "Error creating the Container: " + output)) + if resp.status_code != 201: + res.append((False, "Error creating the Container: " + resp.text)) continue - output = json.loads(output) + output = json.loads(resp.text) # Set the cloud id to the VM vm.id = output["Id"] vm.info.systems[0].setValue('instance_id', str(vm.id)) @@ -316,19 +284,14 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): # Now start it success, _ = self.start(vm, auth_data) if not success: - res.append( - (False, "Error starting the Container: " + str(output))) + res.append((False, "Error starting the Container: " + str(output))) # Delete the container - conn.request('DELETE', "/containers/" + vm.id) - resp = conn.getresponse() - resp.read() + resp = self.create_request('DELETE', self.cloud.path + "/containers/" + vm.id, auth_data) continue # Set the default user and password to access the container - vm.info.systems[0].setValue( - 'disk.0.os.credentials.username', 'root') - vm.info.systems[0].setValue( - 'disk.0.os.credentials.password', self._root_password) + vm.info.systems[0].setValue('disk.0.os.credentials.username', 'root') + vm.info.systems[0].setValue('disk.0.os.credentials.password', self._root_password) # Set ssh port in the RADL info of the VM vm.setSSHPort(ssh_port) @@ -343,18 +306,16 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): def updateVMInfo(self, vm, auth_data): try: - conn = self.get_http_connection(auth_data) - conn.request('GET', "/containers/" + vm.id + "/json") - resp = conn.getresponse() - output = resp.read() - if resp.status == 404: + resp = self.create_request('GET', "/containers/" + vm.id + "/json", auth_data) + + if resp.status_code == 404: # If the container does not exist, set state to OFF vm.state = VirtualMachine.OFF return (True, vm) - elif resp.status != 200: - return (False, "Error getting info about the Container: " + output) + elif resp.status_code != 200: + return (False, "Error getting info about the Container: " + resp.text) - output = json.loads(output) + output = json.loads(resp.text) if output["State"]["Running"]: vm.state = VirtualMachine.RUNNING else: @@ -374,17 +335,15 @@ def finalize(self, vm, auth_data): # First Stop it self.stop(vm, auth_data) - # Now delete it - conn = self.get_http_connection(auth_data) - conn.request('DELETE', "/containers/" + vm.id) - resp = conn.getresponse() - output = str(resp.read()) - if resp.status == 404: + # Now delete it + resp = self.create_request('DELETE', "/containers/" + vm.id, auth_data) + + if resp.status_code == 404: self.logger.warn( "Trying to remove a non existing container id: " + vm.id) return (True, vm.id) elif resp.status != 204: - return (False, "Error deleting the Container: " + output) + return (False, "Error deleting the Container: " + resp.text) else: return (True, vm.id) except Exception: @@ -392,13 +351,11 @@ def finalize(self, vm, auth_data): return (False, "Error connecting with Docker server") def stop(self, vm, auth_data): - try: - conn = self.get_http_connection(auth_data) - conn.request('POST', "/containers/" + vm.id + "/stop") - resp = conn.getresponse() - output = str(resp.read()) - if resp.status != 204: - return (False, "Error stopping the Container: " + output) + try: + resp = self.create_request('POST', "/containers/" + vm.id + "/stop", auth_data) + + if resp.status_code != 204: + return (False, "Error stopping the Container: " + resp.text) else: return (True, vm.id) except Exception: @@ -407,12 +364,10 @@ def stop(self, vm, auth_data): def start(self, vm, auth_data): try: - conn = self.get_http_connection(auth_data) - conn.request('POST', "/containers/" + vm.id + "/start") - resp = conn.getresponse() - output = str(resp.read()) - if resp.status != 204: - return (False, "Error starting the Container: " + output) + resp = self.create_request('POST', "/containers/" + vm.id + "/start", auth_data) + + if resp.status_code != 204: + return (False, "Error starting the Container: " + resp.text) else: return (True, vm.id) except Exception: From 27b664073efaacb1dbbba9dcb7983df054d7954b Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 08:50:17 +0100 Subject: [PATCH 02/28] Move Docker conn to requests --- IM/UnixHTTPAdapter.py | 79 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 IM/UnixHTTPAdapter.py diff --git a/IM/UnixHTTPAdapter.py b/IM/UnixHTTPAdapter.py new file mode 100644 index 000000000..8d2a850b5 --- /dev/null +++ b/IM/UnixHTTPAdapter.py @@ -0,0 +1,79 @@ +# IM - Infrastructure Manager +# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# The following file has been taken from requests-unixsocket +#https://github.com/msabramo/requests-unixsocket/blob/master/requests_unixsocket/adapters.py + +import socket + +from requests.adapters import HTTPAdapter +from requests.compat import urlparse, unquote +try: + from requests.packages.urllib3.connection import HTTPConnection + from requests.packages.urllib3.connectionpool import HTTPConnectionPool +except ImportError: + from urllib3.connection import HTTPConnection + from urllib3.connectionpool import HTTPConnectionPool + + +class UnixHTTPConnection(HTTPConnection): + + def __init__(self, unix_socket_url, timeout=60): + """Create an HTTP connection to a unix domain socket + :param unix_socket_url: A URL with a scheme of 'http+unix' and the + netloc is a percent-encoded path to a unix domain socket. E.g.: + 'http+unix://%2Ftmp%2Fprofilesvc.sock/status/pid' + """ + HTTPConnection.__init__(self, 'localhost', timeout=timeout) + self.unix_socket_url = unix_socket_url + self.timeout = timeout + + def connect(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(self.timeout) + socket_path = unquote(urlparse(self.unix_socket_url).netloc) + sock.connect(socket_path) + self.sock = sock + + +class UnixHTTPConnectionPool(HTTPConnectionPool): + + def __init__(self, socket_path, timeout=60): + HTTPConnectionPool.__init__(self, 'localhost', timeout=timeout) + self.socket_path = socket_path + self.timeout = timeout + + def _new_conn(self): + return UnixHTTPConnection(self.socket_path, self.timeout) + + +class UnixHTTPAdapter(HTTPAdapter): + + def __init__(self, timeout=60): + super(UnixHTTPAdapter, self).__init__() + self.timeout = timeout + + def get_connection(self, socket_path, proxies=None): + proxies = proxies or {} + proxy = proxies.get(urlparse(socket_path.lower()).scheme) + + if proxy: + raise ValueError('%s does not support specifying proxies' + % self.__class__.__name__) + return UnixHTTPConnectionPool(socket_path, self.timeout) + + def request_url(self, request, proxies): + return request.path_url \ No newline at end of file From cf8edbdce6b53c6a041580388d8330372695250c Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 09:08:58 +0100 Subject: [PATCH 03/28] Bugfixes --- IM/connectors/Docker.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index 3af098563..fecdab684 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -51,7 +51,9 @@ def create_request(self, method, url, auth_data, headers=None, body=None): auth = auths[0] if self.cloud.protocol == 'unix': - url = "http+unix://" + self.cloud.server + url + url = "http+unix://%%2F%s%s%s" % (self.cloud.server.replace("/", "%2F"), + self.cloud.path.replace("/", "%2F"), + url) session = requests.Session() session.mount('http+unix://', UnixHTTPAdapter.UnixHTTPAdapter()) resp = session.request(method, url, verify=False, headers=headers, data=body) @@ -269,8 +271,8 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): cont_data = self._generate_create_request_data(outports, system, vm, ssh_port) body = json.dumps(cont_data) - headers = {'Accept': 'application/json'} - resp = self.create_request('POST', self.cloud.path + "/containers/create", auth_data, headers, body) + headers = {'Content-Type': 'application/json'} + resp = self.create_request('POST', "/containers/create", auth_data, headers, body) if resp.status_code != 201: res.append((False, "Error creating the Container: " + resp.text)) @@ -286,7 +288,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): if not success: res.append((False, "Error starting the Container: " + str(output))) # Delete the container - resp = self.create_request('DELETE', self.cloud.path + "/containers/" + vm.id, auth_data) + resp = self.create_request('DELETE', "/containers/" + vm.id, auth_data) continue # Set the default user and password to access the container @@ -342,7 +344,7 @@ def finalize(self, vm, auth_data): self.logger.warn( "Trying to remove a non existing container id: " + vm.id) return (True, vm.id) - elif resp.status != 204: + elif resp.status_code != 204: return (False, "Error deleting the Container: " + resp.text) else: return (True, vm.id) From 6dc235a8151cf84dcf1423a94e9cac78234148fd Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 09:32:51 +0100 Subject: [PATCH 04/28] Bugfix --- IM/connectors/Docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index fecdab684..c1305527e 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -58,7 +58,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): session.mount('http+unix://', UnixHTTPAdapter.UnixHTTPAdapter()) resp = session.request(method, url, verify=False, headers=headers, data=body) else: - url = "%s://%s:%d%s" % (self.cloud.protocol, self.cloud.server, self.cloud.port, url) + url = "%s://%s:%d%s%s" % (self.cloud.protocol, self.cloud.server, self.cloud.port, self.cloud.path, url) if 'cert' in auth and 'key' in auth: cert = self.get_user_cert_data(auth) else: From 6b51cf077507cb936698b28e89c2c80d2c12201a Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 09:36:35 +0100 Subject: [PATCH 05/28] Bugfix --- IM/connectors/Docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index c1305527e..d339e4118 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -173,7 +173,7 @@ def _generate_create_request_data(self, outports, system, vm, ssh_port): cont_data['Volumes'] = volumes HostConfig = {} - # HostConfig['CpuShares'] = "%d" % cpu + HostConfig['CpuShares'] = "%d" % cpu HostConfig['Memory'] = memory HostConfig['PortBindings'] = self._generate_port_bindings( outports, ssh_port) From 8805939518a59f49814b6829394ebe563f11cb56 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 10:44:22 +0100 Subject: [PATCH 06/28] Bugfixes --- IM/connectors/Docker.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index d339e4118..bb7a2eaea 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -146,14 +146,12 @@ def setIPs(self, vm, cont_info): vm.setIps(public_ips, private_ips) - def _generate_create_request_data(self, outports, system, vm, ssh_port): + def _generate_create_request_data(self, image_name, outports, system, vm, ssh_port): cont_data = {} cpu = int(system.getValue('cpu.count')) - 1 memory = system.getFeature('memory.size').getValue('B') # name = system.getValue("disk.0.image.name") - # The URI has this format: docker://image_name - image_name = system.getValue("disk.0.image.url")[9:] (nodename, nodedom) = vm.getRequestedName( default_hostname=Config.DEFAULT_VM_NAME, default_domain=Config.DEFAULT_DOMAIN) @@ -173,7 +171,7 @@ def _generate_create_request_data(self, outports, system, vm, ssh_port): cont_data['Volumes'] = volumes HostConfig = {} - HostConfig['CpuShares'] = "%d" % cpu + HostConfig['CpuShares'] = cpu HostConfig['Memory'] = memory HostConfig['PortBindings'] = self._generate_port_bindings( outports, ssh_port) @@ -267,11 +265,28 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): # Create the VM to get the nodename vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self) - # Create the container - cont_data = self._generate_create_request_data(outports, system, vm, ssh_port) - body = json.dumps(cont_data) + # The URI has this format: docker://image_name + full_image_name = system.getValue("disk.0.image.url")[9:] + # First we have to pull the image headers = {'Content-Type': 'application/json'} + image_parts = full_image_name.split(":") + image_name = image_parts[0] + if len(image_parts) < 2: + tag = "latest" + else: + tag = image_parts[1] + resp = self.create_request('POST', "/images/create?fromImage=%s&tag=%s" % (image_name, tag), + auth_data, headers) + + if resp.status_code not in [201, 200]: + res.append((False, "Error pulling the image: " + resp.text)) + continue + + # Create the container + cont_data = self._generate_create_request_data(full_image_name, outports, system, vm, ssh_port) + body = json.dumps(cont_data) + resp = self.create_request('POST', "/containers/create", auth_data, headers, body) if resp.status_code != 201: From fe26ad23059d06f9ee84fe30b08d393f0b415c2b Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 10:48:24 +0100 Subject: [PATCH 07/28] Style changes --- IM/UnixHTTPAdapter.py | 4 ++-- IM/connectors/Docker.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/IM/UnixHTTPAdapter.py b/IM/UnixHTTPAdapter.py index 8d2a850b5..96b3acb51 100644 --- a/IM/UnixHTTPAdapter.py +++ b/IM/UnixHTTPAdapter.py @@ -15,7 +15,7 @@ # along with this program. If not, see . # The following file has been taken from requests-unixsocket -#https://github.com/msabramo/requests-unixsocket/blob/master/requests_unixsocket/adapters.py +# https://github.com/msabramo/requests-unixsocket/blob/master/requests_unixsocket/adapters.py import socket @@ -76,4 +76,4 @@ def get_connection(self, socket_path, proxies=None): return UnixHTTPConnectionPool(socket_path, self.timeout) def request_url(self, request, proxies): - return request.path_url \ No newline at end of file + return request.path_url diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index bb7a2eaea..95ffe6bdf 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -52,8 +52,8 @@ def create_request(self, method, url, auth_data, headers=None, body=None): if self.cloud.protocol == 'unix': url = "http+unix://%%2F%s%s%s" % (self.cloud.server.replace("/", "%2F"), - self.cloud.path.replace("/", "%2F"), - url) + self.cloud.path.replace("/", "%2F"), + url) session = requests.Session() session.mount('http+unix://', UnixHTTPAdapter.UnixHTTPAdapter()) resp = session.request(method, url, verify=False, headers=headers, data=body) @@ -63,7 +63,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): cert = self.get_user_cert_data(auth) else: cert = None - + resp = requests.request(method, url, verify=False, cert=cert, headers=headers, data=body) return resp @@ -267,7 +267,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): # The URI has this format: docker://image_name full_image_name = system.getValue("disk.0.image.url")[9:] - + # First we have to pull the image headers = {'Content-Type': 'application/json'} image_parts = full_image_name.split(":") @@ -324,7 +324,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): def updateVMInfo(self, vm, auth_data): try: resp = self.create_request('GET', "/containers/" + vm.id + "/json", auth_data) - + if resp.status_code == 404: # If the container does not exist, set state to OFF vm.state = VirtualMachine.OFF @@ -352,9 +352,9 @@ def finalize(self, vm, auth_data): # First Stop it self.stop(vm, auth_data) - # Now delete it + # Now delete it resp = self.create_request('DELETE', "/containers/" + vm.id, auth_data) - + if resp.status_code == 404: self.logger.warn( "Trying to remove a non existing container id: " + vm.id) @@ -368,9 +368,9 @@ def finalize(self, vm, auth_data): return (False, "Error connecting with Docker server") def stop(self, vm, auth_data): - try: + try: resp = self.create_request('POST', "/containers/" + vm.id + "/stop", auth_data) - + if resp.status_code != 204: return (False, "Error stopping the Container: " + resp.text) else: From d850dacc2e6474703232473cf16f5612d544ae9d Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 12:48:38 +0100 Subject: [PATCH 08/28] Bugfixes --- IM/connectors/Docker.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index 95ffe6bdf..0797efaf3 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -160,11 +160,25 @@ def _generate_create_request_data(self, image_name, outports, system, vm, ssh_po cont_data['Hostname'] = nodename cont_data['Domainname'] = nodedom - cont_data['Cmd'] = ["/bin/bash", "-c", ("yum install -y openssh-server ; apt-get update && apt-get install" - " -y openssh-server && sed -i 's/PermitRootLogin without-password/" - "PermitRootLogin yes/g' /etc/ssh/sshd_config && service ssh start " - "&& service ssh stop ; echo 'root:" + self._root_password + - "' | chpasswd ; /usr/sbin/sshd -D")] + command = "yum install -y openssh-server python" + command += " ; " + command += "apt-get update && apt-get install -y openssh-server python" + command += " ; " + command += "mkdir /var/run/sshd" + command += " ; " + command += "sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/g' /etc/ssh/sshd_config" + command += " ; " + command += "sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config" + command += " ; " + command += "ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N ''" + command += " ; " + command += "echo 'root:" + self._root_password + "' | chpasswd" + command += " ; " + command += "sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd" + command += " ; " + command += " /usr/sbin/sshd -D" + + cont_data['Cmd'] = ["/bin/bash", "-c", command] cont_data['Image'] = image_name cont_data['ExposedPorts'] = self._generate_exposed_ports(outports) if volumes: From 96a6805930c85b9a5f3ec9ebf6c86d73b5dfd7f1 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 13:05:21 +0100 Subject: [PATCH 09/28] Change docker conn test to requests --- test/unit/connectors/Docker.py | 83 +++++++++++++--------------------- 1 file changed, 31 insertions(+), 52 deletions(-) diff --git a/test/unit/connectors/Docker.py b/test/unit/connectors/Docker.py index a1cf71ba7..ccf1f7297 100755 --- a/test/unit/connectors/Docker.py +++ b/test/unit/connectors/Docker.py @@ -31,6 +31,7 @@ from IM.VirtualMachine import VirtualMachine from IM.InfrastructureInfo import InfrastructureInfo from IM.connectors.Docker import DockerCloudConnector +from IM.uriparse import uriparse from mock import patch, MagicMock @@ -47,7 +48,6 @@ class TestDockerConnector(unittest.TestCase): @classmethod def setUpClass(cls): - cls.last_op = None, None cls.log = StringIO() ch = logging.StreamHandler(cls.log) formatter = logging.Formatter( @@ -100,37 +100,37 @@ def test_10_concrete(self): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - def get_response(self): - method, url = self.__class__.last_op - + def get_response(self, method, url, verify, cert, headers, data): resp = MagicMock() + parts = uriparse(url) + url = parts[2] + params = parts[4] if method == "GET": if url == "/api/": - resp.status = 200 - resp.read.return_value = '{"versions": "v1"}' + resp.status_code = 200 + resp.text = '{"versions": "v1"}' if url == "/containers/1/json": - resp.status = 200 - resp.read.return_value = '{"State": {"Running": 1}, "NetworkSettings": {"IPAddress": "10.0.0.1"}}' + resp.status_code = 200 + resp.text = '{"State": {"Running": 1}, "NetworkSettings": {"IPAddress": "10.0.0.1"}}' elif method == "POST": if url == "/containers/create": - resp.status = 201 - resp.read.return_value = '{"Id": "id"}' + resp.status_code = 201 + resp.text = '{"Id": "id"}' + elif url == "/images/create": + resp.status_code = 200 elif url.endswith("/start"): - resp.status = 204 + resp.status_code = 204 elif url.endswith("/stop"): - resp.status = 204 + resp.status_code = 204 elif method == "DELETE": if url.endswith("/containers/1"): - resp.status = 204 + resp.status_code = 204 return resp - def request(self, method, url, body=None, headers={}): - self.__class__.last_op = method, url - - @patch('httplib.HTTPConnection') - def test_20_launch(self, connection): + @patch('requests.request') + def test_20_launch(self, requests): radl_data = """ network net1 (outbound = 'yes' and outports = '8080') network net2 () @@ -154,12 +154,7 @@ def test_20_launch(self, connection): auth = Authentication([{'id': 'docker', 'type': 'Docker', 'host': 'http://server.com:2375'}]) docker_cloud = self.get_docker_cloud() - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.putrequest.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response res = docker_cloud.launch(InfrastructureInfo(), radl, radl, 1, auth) success, _ = res[0] @@ -167,8 +162,8 @@ def test_20_launch(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') - def test_30_updateVMInfo(self, connection): + @patch('requests.request') + def test_30_updateVMInfo(self, requests): radl_data = """ network net (outbound = 'yes') system test ( @@ -192,11 +187,7 @@ def test_30_updateVMInfo(self, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "1", docker_cloud.cloud, radl, radl, docker_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, vm = docker_cloud.updateVMInfo(vm, auth) @@ -204,8 +195,8 @@ def test_30_updateVMInfo(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') - def test_40_stop(self, connection): + @patch('requests.request') + def test_40_stop(self, requests): auth = Authentication([{'id': 'docker', 'type': 'Docker', 'host': 'http://server.com:2375'}]) docker_cloud = self.get_docker_cloud() @@ -213,11 +204,7 @@ def test_40_stop(self, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "1", docker_cloud.cloud, "", "", docker_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = docker_cloud.stop(vm, auth) @@ -225,8 +212,8 @@ def test_40_stop(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') - def test_50_start(self, connection): + @patch('requests.request') + def test_50_start(self, requests): auth = Authentication([{'id': 'docker', 'type': 'Docker', 'host': 'http://server.com:2375'}]) docker_cloud = self.get_docker_cloud() @@ -234,11 +221,7 @@ def test_50_start(self, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "1", docker_cloud.cloud, "", "", docker_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = docker_cloud.start(vm, auth) @@ -246,8 +229,8 @@ def test_50_start(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') - def test_60_finalize(self, connection): + @patch('requests.request') + def test_60_finalize(self, requests): auth = Authentication([{'id': 'docker', 'type': 'Docker', 'host': 'http://server.com:2375'}]) docker_cloud = self.get_docker_cloud() @@ -255,11 +238,7 @@ def test_60_finalize(self, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "1", docker_cloud.cloud, "", "", docker_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = docker_cloud.finalize(vm, auth) From f2e58121fd682b76960769d1b4ac3bf3ba37011b Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Dec 2016 13:41:24 +0100 Subject: [PATCH 10/28] Style changes --- IM/connectors/OCCI.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index 8cc6f9841..b3045f76f 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -71,7 +71,7 @@ def create_request_static(method, url, auth, headers, body=None): except: pass - return resp + return resp def create_request(self, method, url, auth_data, headers, body=None): url = "%s://%s:%d%s" % (self.cloud.protocol, self.cloud.server, self.cloud.port, url) @@ -837,7 +837,7 @@ def stop(self, vm, auth_data): body = ('Category: suspend;scheme="http://schemas.ogf.org/occi/infrastructure/compute/action#"' ';class="action";\n') resp = self.create_request('POST', self.cloud.path + "/compute/" + vm.id + "?action=suspend", - auth_data, headers, body) + auth_data, headers, body) if resp.status_code != 200: return (False, "Error stopping the VM: " + resp.reason + "\n" + resp.text) @@ -857,7 +857,7 @@ def start(self, vm, auth_data): body = ('Category: start;scheme="http://schemas.ogf.org/occi/infrastructure/compute/action#"' ';class="action";\n') resp = self.create_request('POST', self.cloud.path + "/compute/" + vm.id + "?action=start", - auth_data, headers, body) + auth_data, headers, body) if resp.status_code != 200: return (False, "Error starting the VM: " + resp.reason + "\n" + resp.text) From beb6328f36fd5ee51eff48471b068be9ecc988b7 Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 16 Dec 2016 12:00:20 +0100 Subject: [PATCH 11/28] Move AzureClassic to requests --- IM/connectors/AzureClassic.py | 218 ++++++++++++--------------- test/unit/connectors/AzureClassic.py | 156 ++++++++----------- 2 files changed, 159 insertions(+), 215 deletions(-) diff --git a/IM/connectors/AzureClassic.py b/IM/connectors/AzureClassic.py index 44e2ab66c..439b6279a 100644 --- a/IM/connectors/AzureClassic.py +++ b/IM/connectors/AzureClassic.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import base64 -import httplib +import requests import time import os import tempfile @@ -139,11 +139,24 @@ class AzureClassicCloudConnector(CloudConnector): } def __init__(self, cloud_info): - self.cert_file = '' - self.key_file = '' self.instance_type_list = None CloudConnector.__init__(self, cloud_info) + def create_request(self, method, url, auth_data, headers=None, body=None): + + auths = auth_data.getAuthInfo(AzureClassicCloudConnector.type, self.cloud.server) + if not auths: + self.logger.error("No correct auth data has been specified to Azure.") + return None + else: + auth = auths[0] + + url = "%s://%s:%d%s" % (self.cloud.protocol, self.AZURE_SERVER, self.AZURE_PORT, url) + cert = self.get_user_cert_data(auth) + resp = requests.request(method, url, verify=False, cert=cert, headers=headers, data=body) + + return resp + def concreteSystem(self, radl_system, auth_data): image_urls = radl_system.getValue("disk.0.image.url") if not image_urls: @@ -382,7 +395,7 @@ def get_azure_vm_create_xml(self, vm, storage_account, radl, num, auth_data): return res - def get_connection_and_subscription_id(self, auth_data): + def get_subscription_id(self, auth_data): auths = auth_data.getAuthInfo(self.type) if not auths: raise Exception("No auth data has been specified to Azure.") @@ -395,19 +408,7 @@ def get_connection_and_subscription_id(self, auth_data): raise Exception( "No correct auth data has been specified to Azure: subscription_id, public_key and private_key.") - # We check if the cert and key files exist - if os.path.isfile(self.cert_file) and os.path.isfile(self.key_file): - cert_file = self.cert_file - key_file = self.key_file - else: - cert_file, key_file = self.get_user_cert_data(auth) - self.cert_file = cert_file - self.key_file = key_file - - conn = httplib.HTTPSConnection( - self.AZURE_SERVER, self.AZURE_PORT, cert_file=cert_file, key_file=key_file) - - return conn, subscription_id + return subscription_id def get_user_cert_data(self, auth): """ @@ -442,8 +443,7 @@ def create_service(self, auth_data, region): service_name + " in region: " + region) try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) + subscription_id = self.get_subscription_id(auth_data) uri = "https://%s/%s/services/hostedservices" % ( self.AZURE_SERVER, subscription_id) service_create_xml = ''' @@ -454,18 +454,16 @@ def create_service(self, auth_data, region): %s ''' % (service_name, base64.b64encode(service_name), service_name, region) - conn.request('POST', uri, body=service_create_xml, headers={ - 'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'}) - resp = conn.getresponse() - output = resp.read() + headers = {'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'} + resp = self.create_request('POST', uri, auth_data, headers, service_create_xml) except Exception, ex: self.logger.exception("Error creating the service") return None, "Error creating the service" + str(ex) - if resp.status != 201: + if resp.status_code != 201: self.logger.error( - "Error creating the service: Error code: " + str(resp.status) + ". Msg: " + output) - return None, "Error creating the service: Error code: " + str(resp.status) + ". Msg: " + output + "Error creating the service: Error code: " + str(resp.status_code) + ". Msg: " + resp.text) + return None, "Error creating the service: Error code: " + str(resp.status_code) + ". Msg: " + resp.text return service_name, None @@ -474,23 +472,20 @@ def delete_service(self, service_name, auth_data): Delete the Azure Cloud Service with name "service_name" """ try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) - uri = "/%s/services/hostedservices/%s?comp=media" % ( - subscription_id, service_name) - conn.request('DELETE', uri, headers={'x-ms-version': '2013-08-01'}) - resp = conn.getresponse() - output = resp.read() + subscription_id = self.get_subscription_id(auth_data) + uri = "/%s/services/hostedservices/%s?comp=media" % (subscription_id, service_name) + headers = {'x-ms-version': '2013-08-01'} + resp = self.create_request('DELETE', uri, auth_data, headers) except Exception, ex: self.logger.exception("Error deleting the service") return (False, "Error deleting the service: " + str(ex)) - if resp.status != 202: + if resp.status_code != 202: self.logger.error( - "Error deleting the service: Error Code " + str(resp.status) + ". Msg: " + output) - return (False, "Error deleting the service: Error Code " + str(resp.status) + ". Msg: " + output) + "Error deleting the service: Error Code " + str(resp.status_code) + ". Msg: " + resp.text) + return (False, "Error deleting the service: Error Code " + str(resp.status_code) + ". Msg: " + resp.text) - request_id = resp.getheader('x-ms-request-id') + request_id = resp.headers['x-ms-request-id'] # Call to GET OPERATION STATUS until "Succeeded" success = self.wait_operation_status(request_id, auth_data) @@ -511,22 +506,19 @@ def wait_operation_status(self, request_id, auth_data, delay=2, timeout=90): time.sleep(delay) wait += delay try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) + subscription_id = self.get_subscription_id(auth_data) uri = "/%s/operations/%s" % (subscription_id, request_id) - conn.request('GET', uri, headers={ - 'x-ms-version': '2013-03-01'}) - resp = conn.getresponse() - output = resp.read() + headers = {'x-ms-version': '2013-03-01'} + resp = self.create_request('GET', uri, auth_data, headers) - if resp.status == 200: - output = Operation(output) + if resp.status_code == 200: + output = Operation(resp.text) status_str = output.Status # InProgress|Succeeded|Failed self.logger.debug("Operation string state: " + status_str) else: self.logger.error( - "Error waiting operation to finish: Code %d. Msg: %s." % (resp.status, output)) + "Error waiting operation to finish: Code %d. Msg: %s." % (resp.status_code, resp.text)) return False except Exception: self.logger.exception( @@ -552,8 +544,7 @@ def create_storage_account(self, storage_account, auth_data, region, timeout=120 """ self.logger.info("Creating the storage account " + storage_account) try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) + subscription_id = self.get_subscription_id(auth_data) uri = "/%s/services/storageservices" % subscription_id storage_create_xml = ''' @@ -570,20 +561,18 @@ def create_storage_account(self, storage_account, auth_data, region, timeout=120 ''' % (storage_account, storage_account, base64.b64encode(storage_account), region) - conn.request('POST', uri, body=storage_create_xml, headers={ - 'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'}) - resp = conn.getresponse() - output = resp.read() + headers = {'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'} + resp = self.create_request('POST', uri, auth_data, headers, storage_create_xml) except Exception, ex: self.logger.exception("Error creating the storage account") return None, "Error creating the storage account" + str(ex) - if resp.status != 202: + if resp.status_code != 202: self.logger.error( - "Error creating the storage account: Error code " + str(resp.status) + ". Msg: " + output) - return None, "Error code " + str(resp.status) + ". Msg: " + output + "Error creating the storage account: Error code " + str(resp.status_code) + ". Msg: " + resp.text) + return None, "Error code " + str(resp.status_code) + ". Msg: " + resp.text - request_id = resp.getheader('x-ms-request-id') + request_id = resp.headers['x-ms-request-id'] # Call to GET OPERATION STATUS until 200 (OK) success = self.wait_operation_status(request_id, auth_data) @@ -605,26 +594,24 @@ def create_storage_account(self, storage_account, auth_data, region, timeout=120 else: self.logger.error( "Error waiting the creation of the storage account") - self.delete_storage_account(storage_account, subscription_id, conn) + self.delete_storage_account(storage_account, subscription_id, auth_data) return None, "Error waiting the creation of the storage account" - def delete_storage_account(self, storage_account, subscription_id, conn): + def delete_storage_account(self, storage_account, subscription_id, auth_data): """ Delete an storage account with the name specified in "storage_account" """ try: - uri = "/%s/services/storageservices/%s" % ( - subscription_id, storage_account) - conn.request('DELETE', uri, headers={'x-ms-version': '2013-03-01'}) - resp = conn.getresponse() - output = resp.read() + uri = "/%s/services/storageservices/%s" % (subscription_id, storage_account) + headers = {'x-ms-version': '2013-03-01'} + resp = self.create_request('DELETE', uri, auth_data, headers) except Exception: self.logger.exception("Error deleting the storage account") return False - if resp.status != 200: + if resp.status_code != 200: self.logger.error( - "Error deleting the storage account: Error Code " + str(resp.status) + ". Msg: " + output) + "Error deleting the storage account: Error Code " + str(resp.status_code) + ". Msg: " + resp.text) return False return True @@ -634,23 +621,21 @@ def get_storage_account(self, storage_account, auth_data): Get the information about the Storage Account named "storage_account" or None if it does not exist """ try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) + subscription_id = self.get_subscription_id(auth_data) uri = "/%s/services/storageservices/%s" % ( subscription_id, storage_account) - conn.request('GET', uri, headers={'x-ms-version': '2013-03-01'}) - resp = conn.getresponse() - output = resp.read() - if resp.status == 200: - storage_info = StorageService(output) + headers = {'x-ms-version': '2013-03-01'} + resp = self.create_request('GET', uri, auth_data, headers) + if resp.status_code == 200: + storage_info = StorageService(resp.text) return storage_info.StorageServiceProperties - elif resp.status == 404: + elif resp.status_code == 404: self.logger.debug( "Storage " + storage_account + " does not exist") return None else: self.logger.warn( - "Error checking the storage account " + storage_account + ". Msg: " + output) + "Error checking the storage account " + storage_account + ". Msg: " + resp.text) return None except Exception: self.logger.exception("Error checking the storage account") @@ -668,7 +653,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): i = 0 while i < num_vm: try: - conn, subscription_id = self.get_connection_and_subscription_id(auth_data) + subscription_id = self.get_subscription_id(auth_data) # Create storage account storage_account_name = self.get_storage_name(subscription_id, region) @@ -715,20 +700,18 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): uri = "/%s/services/hostedservices/%s/deployments" % ( subscription_id, service_name) - conn.request('POST', uri, body=vm_create_xml, headers={ - 'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'}) - resp = conn.getresponse() - output = resp.read() + headers = {'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'} + resp = self.create_request('POST', uri, auth_data, headers, vm_create_xml) - if resp.status != 202: + if resp.status_code != 202: self.delete_service(service_name, auth_data) self.logger.error( - "Error creating the VM: Error Code " + str(resp.status) + ". Msg: " + output) + "Error creating the VM: Error Code " + str(resp.status_code) + ". Msg: " + resp.text) res.append((False, "Error creating the VM: Error Code " + - str(resp.status) + ". Msg: " + output)) + str(resp.status_code) + ". Msg: " + resp.text)) else: # Call the GET OPERATION STATUS until sea 200 (OK) - request_id = resp.getheader('x-ms-request-id') + request_id = resp.headers['x-ms-request-id'] success = self.wait_operation_status(request_id, auth_data) if success: res.append((True, vm)) @@ -804,30 +787,28 @@ def updateVMInfo(self, vm, auth_data): service_name = vm.id try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) + subscription_id = self.get_subscription_id(auth_data) uri = "/%s/services/hostedservices/%s/deployments/%s" % ( subscription_id, service_name, service_name) - conn.request('GET', uri, headers={'x-ms-version': '2014-02-01'}) - resp = conn.getresponse() - output = resp.read() + headers = {'x-ms-version': '2014-02-01'} + resp = self.create_request('GET', uri, auth_data, headers) except Exception, ex: self.logger.exception("Error getting the VM info: " + vm.id) return (False, "Error getting the VM info: " + vm.id + ". " + str(ex)) - if resp.status == 404: + if resp.status_code == 404: self.logger.warn("VM with ID: " + vm.id + ". Not found!.") vm.state = VirtualMachine.OFF return (True, vm) - if resp.status != 200: + if resp.status_code != 200: self.logger.error("Error getting the VM info: " + vm.id + - ". Error Code: " + str(resp.status) + ". Msg: " + output) + ". Error Code: " + str(resp.status_code) + ". Msg: " + resp.text) return (False, "Error getting the VM info: " + vm.id + - ". Error Code: " + str(resp.status) + ". Msg: " + output) + ". Error Code: " + str(resp.status_code) + ". Msg: " + resp.text) else: self.logger.debug("VM info: " + vm.id + " obtained.") - self.logger.debug(output) - vm_info = Deployment(output) + self.logger.debug(resp.text) + vm_info = Deployment(resp.text) vm.state = self.get_vm_state(vm_info) @@ -895,25 +876,22 @@ def call_role_operation(self, op, vm, auth_data): service_name = vm.id try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) + subscription_id = self.get_subscription_id(auth_data) uri = "/%s/services/hostedservices/%s/deployments/%s/roleinstances/%s/Operations" % ( subscription_id, service_name, service_name, self.ROLE_NAME) - conn.request('POST', uri, body=op, headers={ - 'x-ms-version': '2013-06-01', 'Content-Type': 'application/xml'}) - resp = conn.getresponse() - output = resp.read() + headers = {'x-ms-version': '2013-06-01', 'Content-Type': 'application/xml'} + resp = self.create_request('POST', uri, auth_data, headers) except Exception, ex: self.logger.exception("Error calling role operation") return (False, "Error calling role operation: " + str(ex)) - if resp.status != 202: + if resp.status_code != 202: self.logger.error( - "Error calling role operation: Error Code " + str(resp.status) + ". Msg: " + output) - return (False, "Error calling role operation: Error Code " + str(resp.status) + ". Msg: " + output) + "Error calling role operation: Error Code " + str(resp.status_code) + ". Msg: " + resp.text) + return (False, "Error calling role operation: Error Code " + str(resp.status_code) + ". Msg: " + resp.text) - request_id = resp.getheader('x-ms-request-id') + request_id = resp.headers['x-ms-request-id'] # Call to GET OPERATION STATUS until "Succeded" success = self.wait_operation_status( @@ -950,24 +928,21 @@ def get_all_instance_types(self, auth_data): return self.instance_type_list else: try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) + subscription_id = self.get_subscription_id(auth_data) uri = "/%s/rolesizes" % subscription_id - conn.request('GET', uri, headers={ - 'x-ms-version': '2013-08-01'}) - resp = conn.getresponse() - output = resp.read() + headers = {'x-ms-version': '2013-08-01'} + resp = self.create_request('GET', uri, auth_data, headers) except Exception: self.logger.exception("Error getting Role Sizes") return [] - if resp.status != 200: + if resp.status_code != 200: self.logger.error( - "Error getting Role Sizes. Error Code: " + str(resp.status) + ". Msg: " + output) + "Error getting Role Sizes. Error Code: " + str(resp.status_code) + ". Msg: " + resp.text) return [] else: self.logger.debug("Role List obtained.") - role_sizes = RoleSizes(output) + role_sizes = RoleSizes(resp.text) res = [] for role_size in role_sizes.RoleSize: if role_size.SupportedByVirtualMachines == "true": @@ -998,8 +973,7 @@ def alterVM(self, vm, radl, auth_data): return (False, "Error calling update operation: No instance type found for radl: " + str(radl)) try: - conn, subscription_id = self.get_connection_and_subscription_id( - auth_data) + subscription_id = self.get_subscription_id(auth_data) uri = "/%s/services/hostedservices/%s/deployments/%s/roles/%s" % ( subscription_id, service_name, service_name, self.ROLE_NAME) @@ -1011,20 +985,18 @@ def alterVM(self, vm, radl, auth_data): ''' % (instance_type.Name) - conn.request('PUT', uri, body=body, headers={ - 'x-ms-version': '2013-11-01', 'Content-Type': 'application/xml'}) - resp = conn.getresponse() - output = resp.read() + headers = {'x-ms-version': '2013-11-01', 'Content-Type': 'application/xml'} + resp = self.create_request('PUT', uri, auth_data, headers, body) except Exception, ex: self.logger.exception("Error calling update operation") return (False, "Error calling update operation: " + str(ex)) - if resp.status != 202: + if resp.status_code != 202: self.logger.error( - "Error update role operation: Error Code " + str(resp.status) + ". Msg: " + output) - return (False, "Error update role operation: Error Code " + str(resp.status) + ". Msg: " + output) + "Error update role operation: Error Code " + str(resp.status_code) + ". Msg: " + resp.text) + return (False, "Error update role operation: Error Code " + str(resp.status_code) + ". Msg: " + resp.text) - request_id = resp.getheader('x-ms-request-id') + request_id = resp.headers['x-ms-request-id'] # Call to GET OPERATION STATUS until 200 (OK) success = self.wait_operation_status(request_id, auth_data) diff --git a/test/unit/connectors/AzureClassic.py b/test/unit/connectors/AzureClassic.py index f5a7ce7c3..b43bd59a5 100644 --- a/test/unit/connectors/AzureClassic.py +++ b/test/unit/connectors/AzureClassic.py @@ -31,6 +31,7 @@ from IM.VirtualMachine import VirtualMachine from IM.InfrastructureInfo import InfrastructureInfo from IM.connectors.AzureClassic import AzureClassicCloudConnector +from IM.uriparse import uriparse from mock import patch, MagicMock @@ -47,7 +48,6 @@ class TestAzureClassicConnector(unittest.TestCase): @classmethod def setUpClass(cls): - cls.last_op = None, None cls.log = StringIO() ch = logging.StreamHandler(cls.log) formatter = logging.Formatter( @@ -73,8 +73,8 @@ def get_azure_cloud(): cloud = AzureClassicCloudConnector(cloud_info) return cloud - @patch('httplib.HTTPSConnection') - def test_10_concrete(self, connection): + @patch('requests.request') + def test_10_concrete(self, requests): radl_data = """ network net () system test ( @@ -94,81 +94,77 @@ def test_10_concrete(self, connection): 'public_key': 'public_key', 'private_key': 'private_key'}]) azure_cloud = self.get_azure_cloud() - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response concrete = azure_cloud.concreteSystem(radl_system, auth) self.assertEqual(len(concrete), 1) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - def get_response(self): - method, url = self.__class__.last_op + def get_response(self, method, url, verify, cert, headers, data): + resp = MagicMock() + parts = uriparse(url) + url = parts[2] + params = parts[4] resp = MagicMock() if method == "GET": if "/deployments/" in url: - resp.status = 200 - resp.read.return_value = ("Running" - "RoleSizeNameStarted" - "10.0.0.1" - "158.42.1.1" - "") + resp.status_code = 200 + resp.text = ("Running" + "RoleSizeNameStarted" + "10.0.0.1" + "158.42.1.1" + "") if "/operations/" in url: - resp.status = 200 - resp.read.return_value = ("Succeeded" - "") + resp.status_code = 200 + resp.text = ("Succeeded" + "") elif "/storageservices/" in url: - resp.status = 200 - resp.read.return_value = ("North Europe" - "") + resp.status_code = 200 + resp.text = ("North Europe" + "") elif url.endswith("/rolesizes"): - resp.status = 200 - resp.read.return_value = ("true" - "RoleSizeName" - "5121" - "2014" - "" - "" - "true" - "RoleSizeName" - "20482" - "2014" - "" - "" - "") + resp.status_code = 200 + resp.text = ("true" + "RoleSizeName" + "5121" + "2014" + "" + "" + "true" + "RoleSizeName" + "20482" + "2014" + "" + "" + "") elif method == "POST": if url.endswith("/Operations"): - resp.status = 202 - resp.getheader.return_value = "id" + resp.status_code = 202 + resp.headers = {'x-ms-request-id': 'id'} elif url.endswith("/services/hostedservices"): - resp.status = 201 - resp.read.return_value = "" + resp.status_code = 201 + resp.text = "" elif url.endswith("/deployments"): - resp.status = 202 - resp.getheader.return_value = "id" + resp.status_code = 202 + resp.headers = {'x-ms-request-id': 'id'} elif method == "DELETE": - if url.endswith("comp=media"): - resp.status = 202 - resp.getheader.return_value = "id" + if params == "comp=media": + resp.status_code = 202 + resp.headers = {'x-ms-request-id': 'id'} elif method == "PUT": if "roles" in url: - resp.status = 202 - resp.getheader.return_value = "id" + resp.status_code = 202 + resp.headers = {'x-ms-request-id': 'id'} return resp - def request(self, method, url, body=None, headers={}): - self.__class__.last_op = method, url - - @patch('httplib.HTTPSConnection') + @patch('requests.request') @patch('time.sleep') - def test_20_launch(self, sleep, connection): + def test_20_launch(self, sleep, requests): radl_data = """ network net1 (outbound = 'yes' and outports = '8080') network net2 () @@ -193,11 +189,7 @@ def test_20_launch(self, sleep, connection): 'public_key': 'public_key', 'private_key': 'private_key'}]) azure_cloud = self.get_azure_cloud() - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response res = azure_cloud.launch(InfrastructureInfo(), radl, radl, 1, auth) success, _ = res[0] @@ -205,8 +197,8 @@ def test_20_launch(self, sleep, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPSConnection') - def test_30_updateVMInfo(self, connection): + @patch('requests.request') + def test_30_updateVMInfo(self, requests): radl_data = """ network net (outbound = 'yes') system test ( @@ -231,11 +223,7 @@ def test_30_updateVMInfo(self, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "1", azure_cloud.cloud, radl, radl, azure_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, vm = azure_cloud.updateVMInfo(vm, auth) @@ -243,9 +231,9 @@ def test_30_updateVMInfo(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPSConnection') + @patch('requests.request') @patch('time.sleep') - def test_40_stop(self, sleep, connection): + def test_40_stop(self, sleep, requests): auth = Authentication([{'id': 'azure', 'type': 'AzureClassic', 'subscription_id': 'user', 'public_key': 'public_key', 'private_key': 'private_key'}]) azure_cloud = self.get_azure_cloud() @@ -254,11 +242,7 @@ def test_40_stop(self, sleep, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "1", azure_cloud.cloud, "", "", azure_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = azure_cloud.stop(vm, auth) @@ -266,9 +250,9 @@ def test_40_stop(self, sleep, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPSConnection') + @patch('requests.request') @patch('time.sleep') - def test_50_start(self, sleep, connection): + def test_50_start(self, sleep, requests): auth = Authentication([{'id': 'azure', 'type': 'AzureClassic', 'subscription_id': 'user', 'public_key': 'public_key', 'private_key': 'private_key'}]) azure_cloud = self.get_azure_cloud() @@ -277,11 +261,7 @@ def test_50_start(self, sleep, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "1", azure_cloud.cloud, "", "", azure_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = azure_cloud.start(vm, auth) @@ -289,9 +269,9 @@ def test_50_start(self, sleep, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPSConnection') + @patch('requests.request') @patch('time.sleep') - def test_55_alter(self, sleep, connection): + def test_55_alter(self, sleep, requests): radl_data = """ network net (outbound = 'yes') system test ( @@ -322,11 +302,7 @@ def test_55_alter(self, sleep, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "1", azure_cloud.cloud, radl, radl, azure_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = azure_cloud.alterVM(vm, new_radl, auth) @@ -334,9 +310,9 @@ def test_55_alter(self, sleep, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPSConnection') + @patch('requests.request') @patch('time.sleep') - def test_60_finalize(self, sleep, connection): + def test_60_finalize(self, sleep, requests): auth = Authentication([{'id': 'azure', 'type': 'AzureClassic', 'subscription_id': 'user', 'public_key': 'public_key', 'private_key': 'private_key'}]) azure_cloud = self.get_azure_cloud() @@ -346,11 +322,7 @@ def test_60_finalize(self, sleep, connection): vm = VirtualMachine(inf, "1", azure_cloud.cloud, "", "", azure_cloud) sleep.return_value = True - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = azure_cloud.finalize(vm, auth) From 5b3809fc3610954e0ae6013fd755166049a82738 Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 16 Dec 2016 12:56:49 +0100 Subject: [PATCH 12/28] Bugfix --- IM/connectors/AzureClassic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/connectors/AzureClassic.py b/IM/connectors/AzureClassic.py index 439b6279a..b90e38b4e 100644 --- a/IM/connectors/AzureClassic.py +++ b/IM/connectors/AzureClassic.py @@ -151,7 +151,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): else: auth = auths[0] - url = "%s://%s:%d%s" % (self.cloud.protocol, self.AZURE_SERVER, self.AZURE_PORT, url) + url = "https://%s:%d%s" % (self.AZURE_SERVER, self.AZURE_PORT, url) cert = self.get_user_cert_data(auth) resp = requests.request(method, url, verify=False, cert=cert, headers=headers, data=body) From e544a57739c0611caabca9c00ae2433f1ea9b5e0 Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 16 Dec 2016 13:20:56 +0100 Subject: [PATCH 13/28] Bugfixes --- IM/connectors/AzureClassic.py | 54 ++++++++++++----------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/IM/connectors/AzureClassic.py b/IM/connectors/AzureClassic.py index b90e38b4e..597358382 100644 --- a/IM/connectors/AzureClassic.py +++ b/IM/connectors/AzureClassic.py @@ -151,7 +151,8 @@ def create_request(self, method, url, auth_data, headers=None, body=None): else: auth = auths[0] - url = "https://%s:%d%s" % (self.AZURE_SERVER, self.AZURE_PORT, url) + subscription_id = self.get_subscription_id(auth_data) + url = "https://%s:%d/%s%s" % (self.AZURE_SERVER, self.AZURE_PORT, subscription_id, url) cert = self.get_user_cert_data(auth) resp = requests.request(method, url, verify=False, cert=cert, headers=headers, data=body) @@ -173,8 +174,7 @@ def concreteSystem(self, radl_system, auth_data): protocol = url[0] if protocol == "azr": res_system = radl_system.clone() - instance_type = self.get_instance_type( - res_system, auth_data) + instance_type = self.get_instance_type(res_system, auth_data) if not instance_type: self.logger.error( "Error generating the RADL of the VM, no instance type available for the requirements.") @@ -353,9 +353,7 @@ def get_azure_vm_create_xml(self, vm, storage_account, radl, num, auth_data): hostname = "AzureNode" + str(num) SourceImageName = url[1] - MediaLink = "https://%s.blob.core.windows.net/vhds/%s" % (storage_account, vm.id) - if not MediaLink.endswith('.vhd'): - MediaLink = MediaLink + '.vhd' + MediaLink = "https://%s.blob.core.windows.net/vhds/%s.vhd" % (storage_account, vm.id) instance_type = self.get_instance_type(system, auth_data) DataVirtualHardDisks = self.gen_data_disks(system, storage_account) @@ -443,9 +441,7 @@ def create_service(self, auth_data, region): service_name + " in region: " + region) try: - subscription_id = self.get_subscription_id(auth_data) - uri = "https://%s/%s/services/hostedservices" % ( - self.AZURE_SERVER, subscription_id) + uri = "/services/hostedservices" service_create_xml = ''' %s @@ -472,8 +468,7 @@ def delete_service(self, service_name, auth_data): Delete the Azure Cloud Service with name "service_name" """ try: - subscription_id = self.get_subscription_id(auth_data) - uri = "/%s/services/hostedservices/%s?comp=media" % (subscription_id, service_name) + uri = "/services/hostedservices/%s?comp=media" % service_name headers = {'x-ms-version': '2013-08-01'} resp = self.create_request('DELETE', uri, auth_data, headers) except Exception, ex: @@ -506,8 +501,7 @@ def wait_operation_status(self, request_id, auth_data, delay=2, timeout=90): time.sleep(delay) wait += delay try: - subscription_id = self.get_subscription_id(auth_data) - uri = "/%s/operations/%s" % (subscription_id, request_id) + uri = "/operations/%s" % request_id headers = {'x-ms-version': '2013-03-01'} resp = self.create_request('GET', uri, auth_data, headers) @@ -544,8 +538,7 @@ def create_storage_account(self, storage_account, auth_data, region, timeout=120 """ self.logger.info("Creating the storage account " + storage_account) try: - subscription_id = self.get_subscription_id(auth_data) - uri = "/%s/services/storageservices" % subscription_id + uri = "/services/storageservices" storage_create_xml = ''' %s @@ -594,15 +587,15 @@ def create_storage_account(self, storage_account, auth_data, region, timeout=120 else: self.logger.error( "Error waiting the creation of the storage account") - self.delete_storage_account(storage_account, subscription_id, auth_data) + self.delete_storage_account(storage_account, auth_data) return None, "Error waiting the creation of the storage account" - def delete_storage_account(self, storage_account, subscription_id, auth_data): + def delete_storage_account(self, storage_account, auth_data): """ Delete an storage account with the name specified in "storage_account" """ try: - uri = "/%s/services/storageservices/%s" % (subscription_id, storage_account) + uri = "/services/storageservices/%s" % storage_account headers = {'x-ms-version': '2013-03-01'} resp = self.create_request('DELETE', uri, auth_data, headers) except Exception: @@ -621,9 +614,7 @@ def get_storage_account(self, storage_account, auth_data): Get the information about the Storage Account named "storage_account" or None if it does not exist """ try: - subscription_id = self.get_subscription_id(auth_data) - uri = "/%s/services/storageservices/%s" % ( - subscription_id, storage_account) + uri = "/services/storageservices/%s" % storage_account headers = {'x-ms-version': '2013-03-01'} resp = self.create_request('GET', uri, auth_data, headers) if resp.status_code == 200: @@ -698,8 +689,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res.append((False, "Incorrect image or auth data")) break - uri = "/%s/services/hostedservices/%s/deployments" % ( - subscription_id, service_name) + uri = "/services/hostedservices/%s/deployments" % service_name headers = {'x-ms-version': '2013-03-01', 'Content-Type': 'application/xml'} resp = self.create_request('POST', uri, auth_data, headers, vm_create_xml) @@ -787,9 +777,7 @@ def updateVMInfo(self, vm, auth_data): service_name = vm.id try: - subscription_id = self.get_subscription_id(auth_data) - uri = "/%s/services/hostedservices/%s/deployments/%s" % ( - subscription_id, service_name, service_name) + uri = "/services/hostedservices/%s/deployments/%s" % (service_name, service_name) headers = {'x-ms-version': '2014-02-01'} resp = self.create_request('GET', uri, auth_data, headers) except Exception, ex: @@ -876,9 +864,8 @@ def call_role_operation(self, op, vm, auth_data): service_name = vm.id try: - subscription_id = self.get_subscription_id(auth_data) - uri = "/%s/services/hostedservices/%s/deployments/%s/roleinstances/%s/Operations" % ( - subscription_id, service_name, service_name, self.ROLE_NAME) + uri = "/services/hostedservices/%s/deployments/%s/roleinstances/%s/Operations" % ( + service_name, service_name, self.ROLE_NAME) headers = {'x-ms-version': '2013-06-01', 'Content-Type': 'application/xml'} resp = self.create_request('POST', uri, auth_data, headers) @@ -928,8 +915,7 @@ def get_all_instance_types(self, auth_data): return self.instance_type_list else: try: - subscription_id = self.get_subscription_id(auth_data) - uri = "/%s/rolesizes" % subscription_id + uri = "/rolesizes" headers = {'x-ms-version': '2013-08-01'} resp = self.create_request('GET', uri, auth_data, headers) except Exception: @@ -973,10 +959,8 @@ def alterVM(self, vm, radl, auth_data): return (False, "Error calling update operation: No instance type found for radl: " + str(radl)) try: - subscription_id = self.get_subscription_id(auth_data) - - uri = "/%s/services/hostedservices/%s/deployments/%s/roles/%s" % ( - subscription_id, service_name, service_name, self.ROLE_NAME) + uri = "/services/hostedservices/%s/deployments/%s/roles/%s" % ( + service_name, service_name, self.ROLE_NAME) body = ''' Date: Fri, 16 Dec 2016 13:42:14 +0100 Subject: [PATCH 14/28] Move kubernetes conn to requests --- IM/connectors/Kubernetes.py | 152 ++++++++++++------------------------ 1 file changed, 48 insertions(+), 104 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 0aa156af5..c5a54010b 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -18,7 +18,7 @@ import string import base64 import json -import httplib +import requests from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from CloudConnector import CloudConnector @@ -50,20 +50,17 @@ class KubernetesCloudConnector(CloudConnector): } """Dictionary with a map with the Kubernetes POD states to the IM states.""" - def get_http_connection(self): - """ - Get the HTTPConnection object to contact the Kubernetes API - - Returns(HTTPConnection or HTTPSConnection): HTTPConnection connection object - """ + def create_request(self, method, url, auth_data, headers=None, body=None): + auth_header = self.get_auth_header(auth_data) + if auth_header: + if headers is None: + headers = {} + headers.update(auth_header) - if self.cloud.protocol == 'https': - conn = httplib.HTTPSConnection(self.cloud.server, self.cloud.port) - elif self.cloud.protocol == 'http': - self.logger.warn("Using a unsecure connection to Kubernetes API!") - conn = httplib.HTTPConnection(self.cloud.server, self.cloud.port) + url = "%s://%s:%d%s%s" % (self.cloud.protocol, self.cloud.server, self.cloud.port, self.cloud.path, url) + resp = requests.request(method, url, verify=False, headers=headers, data=body) - return conn + return resp def get_auth_header(self, auth_data): """ @@ -98,19 +95,10 @@ def get_api_version(self, auth_data): version = self._apiVersions[0] try: - auth = self.get_auth_header(auth_data) - headers = {} - if auth: - headers.update(auth) - conn = self.get_http_connection() + resp = self.create_request('GET', "/api/", auth_data) - conn.request('GET', "/api/", headers=headers) - resp = conn.getresponse() - - output = resp.read() - - if resp.status == 200: - output = json.loads(output) + if resp.status_code == 200: + output = json.loads(resp.text) for v in self._apiVersions: if v in output["versions"]: return v @@ -166,24 +154,18 @@ def concreteSystem(self, radl_system, auth_data): def _delete_volume_claim(self, namespace, vc_name, auth_data): try: - auth = self.get_auth_header(auth_data) - headers = {} - if auth: - headers.update(auth) - conn = self.get_http_connection() apiVersion = self.get_api_version(auth_data) - conn.request('DELETE', "/api/" + apiVersion + "/namespaces/" + - namespace + "/persistentvolumeclaims/" + vc_name, headers=headers) - resp = conn.getresponse() - output = str(resp.read()) - if resp.status == 404: + uri = "/api/" + apiVersion + "/namespaces/" + namespace + "/persistentvolumeclaims/" + vc_name + resp = self.create_request('DELETE', uri, auth_data) + + if resp.status_code == 404: self.logger.warn( "Trying to remove a non existing PersistentVolumeClaim: " + vc_name) return True - elif resp.status != 200: + elif resp.status_code != 200: self.logger.error( - "Error deleting the PersistentVolumeClaim: " + output) + "Error deleting the PersistentVolumeClaim: " + resp.txt) return False else: return True @@ -205,23 +187,17 @@ def _delete_volume_claims(self, pod_data, auth_data): def _create_volume_claim(self, claim_data, auth_data): try: - auth_header = self.get_auth_header(auth_data) - conn = self.get_http_connection() apiVersion = self.get_api_version(auth_data) - conn.putrequest('POST', "/api/" + apiVersion + "/namespaces/" + - claim_data['metadata']['namespace'] + "/persistentvolumeclaims") - conn.putheader('Content-Type', 'application/json') - if auth_header: - conn.putheader(auth_header.keys()[0], auth_header.values()[0]) - + headers = {'Content-Type': 'application/json'} + uri = ("/api/" + apiVersion + "/namespaces/" + + claim_data['metadata']['namespace'] + + "/persistentvolumeclaims") body = json.dumps(claim_data) - conn.putheader('Content-Length', len(body)) - conn.endheaders(body) - resp = conn.getresponse() + resp = self.create_request('POST', uri, auth_data, headers, body) - output = str(resp.read()) - if resp.status != 201: + output = str(resp.text) + if resp.status_code != 201: self.logger.error("Error deleting the POD: " + output) return False else: @@ -342,8 +318,6 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): if public_net: outports = public_net.getOutPorts() - auth_header = self.get_auth_header(auth_data) - conn = self.get_http_connection() apiVersion = self.get_api_version(auth_data) res = [] @@ -363,30 +337,22 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): volumes = self._create_volumes( apiVersion, namespace, system, pod_name, auth_data) - # Create the pod - conn.putrequest('POST', "/api/" + apiVersion + - "/namespaces/" + namespace + "/pods") - conn.putheader('Content-Type', 'application/json') - if auth_header: - conn.putheader(auth_header.keys()[ - 0], auth_header.values()[0]) - ssh_port = (KubernetesCloudConnector._port_base_num + KubernetesCloudConnector._port_counter) % 65535 KubernetesCloudConnector._port_counter += 1 pod_data = self._generate_pod_data( apiVersion, namespace, pod_name, outports, system, ssh_port, volumes) body = json.dumps(pod_data) - conn.putheader('Content-Length', len(body)) - conn.endheaders(body) - resp = conn.getresponse() - output = resp.read() - if resp.status != 201: + headers = {'Content-Type': 'application/json'} + uri = "/api/" + apiVersion + "/namespaces/" + namespace + "/pods" + resp = self.create_request('POST', uri, auth_data, headers, body) + + if resp.status_code != 201: res.append( - (False, "Error creating the Container: " + output)) + (False, "Error creating the Container: " + resp.text)) else: - output = json.loads(output) + output = json.loads(resp.text) vm.id = output["metadata"]["namespace"] + "/" + output["metadata"]["name"] # Set SSH port in the RADL info of the VM vm.setSSHPort(ssh_port) @@ -412,23 +378,15 @@ def _get_pod(self, vm_id, auth_data): namespace = vm_id.split("/")[0] pod_name = vm_id.split("/")[1] - auth = self.get_auth_header(auth_data) - headers = {} - if auth: - headers.update(auth) - conn = self.get_http_connection() apiVersion = self.get_api_version(auth_data) - conn.request('GET', "/api/" + apiVersion + "/namespaces/" + - namespace + "/pods/" + pod_name, headers=headers) - resp = conn.getresponse() - - output = resp.read() + uri = "/api/" + apiVersion + "/namespaces/" + namespace + "/pods/" + pod_name + resp = self.create_request('GET', uri, auth_data) - if resp.status == 404 or resp.status == 200: - return (True, resp.status, output) + if resp.status_code == 404 or resp.status_code == 200: + return (True, resp.status, resp.text) else: - return (False, resp.status, output) + return (False, resp.status, resp.text) except Exception, ex: self.logger.exception( @@ -490,23 +448,16 @@ def _delete_pod(self, vm_id, auth_data): namespace = vm_id.split("/")[0] pod_name = vm_id.split("/")[1] - auth = self.get_auth_header(auth_data) - headers = {} - if auth: - headers.update(auth) - conn = self.get_http_connection() apiVersion = self.get_api_version(auth_data) + uri = "/api/" + apiVersion + "/namespaces/" + namespace + "/pods/" + pod_name + resp = self.create_request('DELETE', uri, auth_data) - conn.request('DELETE', "/api/" + apiVersion + "/namespaces/" + - namespace + "/pods/" + pod_name, headers=headers) - resp = conn.getresponse() - output = str(resp.read()) - if resp.status == 404: + if resp.status_code == 404: self.logger.warn( "Trying to remove a non existing POD id: " + pod_name) return (True, pod_name) elif resp.status != 200: - return (False, "Error deleting the POD: " + output) + return (False, "Error deleting the POD: " + resp.text) else: return (True, pod_name) except Exception: @@ -525,8 +476,6 @@ def alterVM(self, vm, radl, auth_data): # But kubernetes does not permit cpu to be updated yet system = radl.systems[0] - auth_header = self.get_auth_header(auth_data) - conn = self.get_http_connection() apiVersion = self.get_api_version(auth_data) try: @@ -556,19 +505,14 @@ def alterVM(self, vm, radl, auth_data): # Create the container namespace = vm.id.split("/")[0] pod_name = vm.id.split("/")[1] - conn.putrequest('PATCH', "/api/" + apiVersion + - "/namespaces/" + namespace + "/pods/" + pod_name) - conn.putheader('Content-Type', 'application/json-patch+json') - if auth_header: - conn.putheader(auth_header.keys()[0], auth_header.values()[0]) + + headers = {'Content-Type': 'application/json-patch+json'} + uri = "/api/" + apiVersion + "/namespaces/" + namespace + "/pods/" + pod_name body = json.dumps(pod_data) - conn.putheader('Content-Length', len(body)) - conn.endheaders(body) + resp = self.create_request('PATCH', uri, auth_data, headers, body) - resp = conn.getresponse() - output = resp.read() - if resp.status != 201: - return (False, "Error updating the Pod: " + output) + if resp.status_code != 201: + return (False, "Error updating the Pod: " + resp.text) else: if new_cpu: vm.info.systems[0].setValue('cpu.count', new_cpu) From 69f26463485684edf94aec8a93323e40f8561c93 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 19 Dec 2016 08:51:25 +0100 Subject: [PATCH 15/28] Move kubernetes conn to requests --- IM/connectors/Kubernetes.py | 2 +- test/unit/connectors/Kubernetes.py | 75 +++++++++++------------------- 2 files changed, 28 insertions(+), 49 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index c5a54010b..d3d43e646 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -456,7 +456,7 @@ def _delete_pod(self, vm_id, auth_data): self.logger.warn( "Trying to remove a non existing POD id: " + pod_name) return (True, pod_name) - elif resp.status != 200: + elif resp.status_code != 200: return (False, "Error deleting the POD: " + resp.text) else: return (True, pod_name) diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 841c68a73..6941eece1 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -31,6 +31,7 @@ from IM.VirtualMachine import VirtualMachine from IM.InfrastructureInfo import InfrastructureInfo from IM.connectors.Kubernetes import KubernetesCloudConnector +from IM.uriparse import uriparse from mock import patch, MagicMock @@ -47,7 +48,6 @@ class TestKubernetesConnector(unittest.TestCase): @classmethod def setUpClass(cls): - cls.last_op = None, None cls.log = StringIO() ch = logging.StreamHandler(cls.log) formatter = logging.Formatter( @@ -100,40 +100,37 @@ def test_10_concrete(self): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - def get_response(self): - method, url = self.__class__.last_op - + def get_response(self, method, url, verify, headers, data): resp = MagicMock() + parts = uriparse(url) + url = parts[2] if method == "GET": if url == "/api/": - resp.status = 200 - resp.read.return_value = '{"versions": "v1"}' + resp.status_code = 200 + resp.text = '{"versions": "v1"}' elif url.endswith("/pods/1"): - resp.status = 200 - resp.read.return_value = ('{"metadata": {"namespace":"namespace", "name": "name"}, "status": ' - '{"phase":"Running", "hostIP": "158.42.1.1", "podIP": "10.0.0.1"}, ' - '"spec": {"volumes": [{"persistentVolumeClaim": {"claimName" : "cname"}}]}}') + resp.status_code = 200 + resp.text = ('{"metadata": {"namespace":"namespace", "name": "name"}, "status": ' + '{"phase":"Running", "hostIP": "158.42.1.1", "podIP": "10.0.0.1"}, ' + '"spec": {"volumes": [{"persistentVolumeClaim": {"claimName" : "cname"}}]}}') elif method == "POST": if url.endswith("/pods"): - resp.status = 201 - resp.read.return_value = '{"metadata": {"namespace":"namespace", "name": "name"}}' + resp.status_code = 201 + resp.text = '{"metadata": {"namespace":"namespace", "name": "name"}}' elif method == "DELETE": if url.endswith("/pods/1"): - resp.status = 200 + resp.status_code = 200 elif "persistentvolumeclaims" in url: - resp.status = 200 + resp.status_code = 200 elif method == "PATCH": if url.endswith("/pods/1"): - resp.status = 201 + resp.status_code = 201 return resp - def request(self, method, url, body=None, headers={}): - self.__class__.last_op = method, url - - @patch('httplib.HTTPConnection') - def test_20_launch(self, connection): + @patch('requests.request') + def test_20_launch(self, requests): radl_data = """ network net1 (outbound = 'yes' and outports = '8080') network net2 () @@ -157,12 +154,7 @@ def test_20_launch(self, connection): auth = Authentication([{'id': 'fogbow', 'type': 'Kubernetes', 'host': 'http://server.com:8080'}]) kube_cloud = self.get_kube_cloud() - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.putrequest.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response res = kube_cloud.launch(InfrastructureInfo(), radl, radl, 1, auth) success, _ = res[0] @@ -170,8 +162,8 @@ def test_20_launch(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') - def test_30_updateVMInfo(self, connection): + @patch('requests.request') + def test_30_updateVMInfo(self, requests): radl_data = """ network net (outbound = 'yes') system test ( @@ -195,11 +187,7 @@ def test_30_updateVMInfo(self, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "namespace/1", kube_cloud.cloud, radl, radl, kube_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, vm = kube_cloud.updateVMInfo(vm, auth) @@ -207,8 +195,8 @@ def test_30_updateVMInfo(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') - def test_55_alter(self, connection): + @patch('requests.request') + def test_55_alter(self, requests): radl_data = """ network net () system test ( @@ -238,12 +226,7 @@ def test_55_alter(self, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "namespace/1", kube_cloud.cloud, radl, radl, kube_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.putrequest.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = kube_cloud.alterVM(vm, new_radl, auth) @@ -251,8 +234,8 @@ def test_55_alter(self, connection): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() - @patch('httplib.HTTPConnection') - def test_60_finalize(self, connection): + @patch('requests.request') + def test_60_finalize(self, requests): auth = Authentication([{'id': 'fogbow', 'type': 'Kubernetes', 'host': 'http://server.com:8080'}]) kube_cloud = self.get_kube_cloud() @@ -260,11 +243,7 @@ def test_60_finalize(self, connection): inf.get_next_vm_id.return_value = 1 vm = VirtualMachine(inf, "namespace/1", kube_cloud.cloud, "", "", kube_cloud) - conn = MagicMock() - connection.return_value = conn - - conn.request.side_effect = self.request - conn.getresponse.side_effect = self.get_response + requests.side_effect = self.get_response success, _ = kube_cloud.finalize(vm, auth) From e79ada8e51f2cd0f21a5def6415ecfd5afb88447 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 19 Dec 2016 09:33:49 +0100 Subject: [PATCH 16/28] Move docker-devel to ubuntu 16 --- docker-devel/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 3e9d262de..568160d3e 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -1,14 +1,11 @@ # Dockerfile to create a container with the IM service -FROM grycap/jenkins:ubuntu14.04-im +FROM grycap/jenkins:ubuntu16.04-im MAINTAINER Miguel Caballer LABEL version="1.5.0" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 -# Add unresolved LibCloud dependency -RUN pip install backports.ssl_match_hostname - # Install im - 'devel' branch RUN cd tmp \ && git clone -b devel https://github.com/grycap/im.git \ From 4cdd27c57c067e12f0153522f418a271316b4aa2 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 20 Dec 2016 08:19:53 +0100 Subject: [PATCH 17/28] Bugfix --- IM/connectors/GCE.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index e927ddf38..8c83a8387 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -303,7 +303,7 @@ def create_firewall(self, inf, net_name, radl, driver): if net.isPublic(): public_net = net - ports = {"tcp": ["22"]} + ports = {"tcp": ["22"], "udp": []} if public_net: outports = public_net.getOutPorts() if outports: @@ -318,7 +318,7 @@ def create_firewall(self, inf, net_name, radl, driver): ports[protocol].append(str(remote_port)) allowed = [{'IPProtocol': 'tcp', 'ports': ports['tcp']}, - {'IPProtocol': 'upd', 'ports': ports['upd']}] + {'IPProtocol': 'udp', 'ports': ports['udp']}] firewall = None try: @@ -358,6 +358,9 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): instance_type = self.get_instance_type( driver.list_sizes(region), system) + if not instance_type: + raise Exception("No compatible size found") + name = system.getValue("instance_name") if not name: name = system.getValue("disk.0.image.name") From fb9cb754b099b278324b1479c6e3af25c5d356fa Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 20 Dec 2016 08:24:17 +0100 Subject: [PATCH 18/28] Bugfix --- IM/connectors/GCE.py | 7 ++++--- test/unit/connectors/GCE.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 8c83a8387..c296e61d7 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -303,7 +303,7 @@ def create_firewall(self, inf, net_name, radl, driver): if net.isPublic(): public_net = net - ports = {"tcp": ["22"], "udp": []} + ports = {"tcp": ["22"]} if public_net: outports = public_net.getOutPorts() if outports: @@ -317,8 +317,9 @@ def create_firewall(self, inf, net_name, radl, driver): ports[protocol] = [] ports[protocol].append(str(remote_port)) - allowed = [{'IPProtocol': 'tcp', 'ports': ports['tcp']}, - {'IPProtocol': 'udp', 'ports': ports['udp']}] + allowed = [{'IPProtocol': 'tcp', 'ports': ports['tcp']}] + if 'udp' in ports: + allowed.append({'IPProtocol': 'udp', 'ports': ports['udp']}) firewall = None try: diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index 8dbe410c6..86c7a61ea 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -111,7 +111,7 @@ def test_10_concrete(self, get_driver): @patch('libcloud.compute.drivers.gce.GCENodeDriver') def test_20_launch(self, get_driver): radl_data = """ - network net1 (outbound = 'yes') + network net1 (outbound = 'yes' and outports = '8080') network net2 () system test ( cpu.arch='x86_64' and From 0682b86ac139994acd610401c3493336ee851bd6 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 20 Dec 2016 16:40:59 +0100 Subject: [PATCH 19/28] Set default zone for the driver --- IM/connectors/GCE.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index c296e61d7..6a11bdba8 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -39,7 +39,7 @@ class GCECloudConnector(CloudConnector): type = "GCE" """str with the name of the provider.""" - DEFAULT_ZONE = "us-central1" + DEFAULT_ZONE = "us-central1-a" def __init__(self, cloud_info): self.auth = None @@ -76,7 +76,7 @@ def get_driver(self, auth_data): " Check that it has more than one line.") driver = cls(auth['username'], auth[ - 'password'], project=auth['project']) + 'password'], project=auth['project'], datastore=self.DEFAULT_ZONE) self.driver = driver return driver From e5a5e1c96c557facceef805f1fdf0c4f7d711740 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 20 Dec 2016 16:41:19 +0100 Subject: [PATCH 20/28] Force to update the cache in the yum install --- contextualization/conf-ansible.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contextualization/conf-ansible.yml b/contextualization/conf-ansible.yml index 120b10483..03269cb95 100644 --- a/contextualization/conf-ansible.yml +++ b/contextualization/conf-ansible.yml @@ -42,7 +42,7 @@ when: ansible_distribution == "Ubuntu" - name: Yum install Ansible RH - yum: name=ansible,python-pip,python-jinja2,sshpass,openssh-clients,wget + yum: name=ansible,python-pip,python-jinja2,sshpass,openssh-clients,wget update_cache=yes when: ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 7 and ansible_distribution != "Fedora" ############################################ In other systems use pip ################################################# From 35d200ad9a3c887d8b5e6bf9a25468c48e387419 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 20 Dec 2016 16:58:14 +0100 Subject: [PATCH 21/28] Style changes --- IM/connectors/GCE.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 6a11bdba8..5ab34e31f 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -421,17 +421,15 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res = [] if num_vm > 1: args['number'] = num_vm - args[ - 'base_name'] = "%s-%s" % (name.lower().replace("_", "-"), int(time.time() * 100)) + args['base_name'] = "%s-%s" % (name.lower().replace("_", "-"), int(time.time() * 100)) nodes = driver.ex_create_multiple_nodes(**args) else: - args[ - 'name'] = "%s-%s" % (name.lower().replace("_", "-"), int(time.time() * 100)) + args['name'] = "%s-%s" % (name.lower().replace("_", "-"), int(time.time() * 100)) nodes = [driver.create_node(**args)] for node in nodes: - vm = VirtualMachine(inf, node.extra[ - 'name'], self.cloud, radl, requested_radl, self.cloud.getCloudConnector()) + vm = VirtualMachine(inf, node.extra['name'], self.cloud, radl, + requested_radl, self.cloud.getCloudConnector()) vm.info.systems[0].setValue('instance_id', str(vm.id)) vm.info.systems[0].setValue('instance_name', str(vm.id)) self.logger.debug("Node successfully created.") From b9e708b07911c21d2ac196c46c139a16ec50a871 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 21 Dec 2016 08:25:43 +0100 Subject: [PATCH 22/28] Add launch test with 3 VMs --- test/unit/connectors/GCE.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index 86c7a61ea..2d2b35ed2 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -118,7 +118,6 @@ def test_20_launch(self, get_driver): cpu.count=1 and memory.size=512m and net_interface.0.connection = 'net1' and - net_interface.0.ip = '10.0.0.1' and net_interface.0.dns_name = 'test' and net_interface.1.connection = 'net2' and disk.0.os.name = 'linux' and @@ -157,9 +156,23 @@ def test_20_launch(self, get_driver): node.name = "gce1name" driver.create_node.return_value = node + node2 = MagicMock() + node2.id = "gce2" + node2.name = "gce2name" + node3 = MagicMock() + node3.id = "gce3" + node3.name = "gce3name" + driver.ex_create_multiple_nodes.return_value = [node, node2, node3] + res = gce_cloud.launch(InfrastructureInfo(), radl, radl, 1, auth) success, _ = res[0] - self.assertTrue(success, msg="ERROR: launching a VM.") + self.assertTrue(success, msg="ERROR: launching a single VM.") + self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) + self.clean_log() + + res = gce_cloud.launch(InfrastructureInfo(), radl, radl, 3, auth) + success, _ = res[0] + self.assertTrue(success, msg="ERROR: launching 3 VMs.") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.clean_log() From ec1ae0498a91fa8c7c5e59dd6972aa2a7c4eb8d9 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 21 Dec 2016 09:10:00 +0100 Subject: [PATCH 23/28] Add documentation about HA --- doc/source/manual.rst | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/source/manual.rst b/doc/source/manual.rst index f3f056dc4..45b0a186f 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -519,4 +519,31 @@ You can also specify an external MySQL server to store IM data using the IM_DATA Or you can also add a volume with all the IM configuration:: - $ sudo docker run -d -p 8899:8899 -p 8800:8800 -v "/some_local_path/im.cfg:/etc/im/im.cfg" --name im grycap/im \ No newline at end of file + $ sudo docker run -d -p 8899:8899 -p 8800:8800 -v "/some_local_path/im.cfg:/etc/im/im.cfg" --name im grycap/im + + +IM in high availability mode +============================ + +The IM service can be launched in high availability (HA) mode using a set of IM instances behind a +`HAProxy `_ load balancer. Currently only the REST API can be used in HA mode. + +This is an example of the HAProxy configuration file:: + + frontend http-frontend + mode http + bind *:8800 + default_backend imbackend + + backend imbackend + mode http + balance roundrobin + stick-table type string len 32 size 30k expire 60m + stick store-response hdr(InfID) + acl inf_id path -m beg /infrastructures/ + stick on path,field(3,/) if inf_id + + server im-8801 10.0.0.1:8801 check + server im-8802 10.0.0.1:8802 check + ... + \ No newline at end of file From 78059f92cb4377aa67a558a8aacd676c732c851e Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 21 Dec 2016 09:11:32 +0100 Subject: [PATCH 24/28] update docs --- doc/source/manual.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 45b0a186f..0ccdb0447 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -525,8 +525,8 @@ Or you can also add a volume with all the IM configuration:: IM in high availability mode ============================ -The IM service can be launched in high availability (HA) mode using a set of IM instances behind a -`HAProxy `_ load balancer. Currently only the REST API can be used in HA mode. +From version 1.5.0 the IM service can be launched in high availability (HA) mode using a set of IM instances +behind a `HAProxy `_ load balancer. Currently only the REST API can be used in HA mode. This is an example of the HAProxy configuration file:: @@ -546,4 +546,3 @@ This is an example of the HAProxy configuration file:: server im-8801 10.0.0.1:8801 check server im-8802 10.0.0.1:8802 check ... - \ No newline at end of file From 2b97d3d0ecefc63613790cab52d2cfbeadae113c Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 21 Dec 2016 10:37:09 +0100 Subject: [PATCH 25/28] minor changes --- IM/connectors/Docker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/IM/connectors/Docker.py b/IM/connectors/Docker.py index 0797efaf3..4861b556c 100644 --- a/IM/connectors/Docker.py +++ b/IM/connectors/Docker.py @@ -19,6 +19,7 @@ import json import socket import requests +import random from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from IM.config import Config @@ -34,7 +35,7 @@ class DockerCloudConnector(CloudConnector): type = "Docker" - _port_base_num = 35000 + _port_base_num = random.randint(35000, 40000) """ Base number to assign SSH port on Docker server host.""" _port_counter = 0 """ Counter to assign SSH port on Docker server host.""" @@ -313,9 +314,9 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm.info.systems[0].setValue('instance_id', str(vm.id)) # Now start it - success, _ = self.start(vm, auth_data) + success, msg = self.start(vm, auth_data) if not success: - res.append((False, "Error starting the Container: " + str(output))) + res.append((False, "Error starting the Container: " + str(msg))) # Delete the container resp = self.create_request('DELETE', "/containers/" + vm.id, auth_data) continue From 48ea10c2c9854fbfe5e80f75103b6d9436c30cc7 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 22 Dec 2016 09:04:41 +0100 Subject: [PATCH 26/28] Change confusing log message --- IM/InfrastructureList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/InfrastructureList.py b/IM/InfrastructureList.py index 752d9f614..12b68a7c7 100644 --- a/IM/InfrastructureList.py +++ b/IM/InfrastructureList.py @@ -145,7 +145,7 @@ def _get_data_from_db(db_url, inf_id=None): InfrastructureList.logger.exception( "ERROR reading infrastructure from database, ignoring it!.") else: - InfrastructureList.logger.error("ERROR getting inf_list from database!.") + InfrastructureList.logger.warn("No data in database!.") db.close() return inf_list From d31f1ad16b5df205cad611bc378c2a7e96e492b1 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 2 Jan 2017 14:29:14 +0100 Subject: [PATCH 27/28] Bugfixes --- IM/connectors/AzureClassic.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/IM/connectors/AzureClassic.py b/IM/connectors/AzureClassic.py index 597358382..5f824e636 100644 --- a/IM/connectors/AzureClassic.py +++ b/IM/connectors/AzureClassic.py @@ -837,7 +837,8 @@ def setIPs(self, vm, vm_info): except: return try: - private_ips.append(role_instance.IpAddress) + if role_instance.IpAddress: + private_ips.append(role_instance.IpAddress) except: pass try: @@ -998,6 +999,10 @@ def update_system_info_from_instance(self, system, instance_type): """ Update the features of the system with the information of the instance_type """ + if not instance_type: + self.logger.warn("No instance type provided. Not updating VM info.") + return + system.addFeature(Feature("cpu.count", "=", instance_type.Cores), conflict="other", missing="other") system.addFeature(Feature("memory.size", "=", instance_type.MemoryInMb, 'M'), From 9cd958b3b635509881efcc7345a4c818dfb5f283 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 2 Jan 2017 16:33:51 +0100 Subject: [PATCH 28/28] Add azure-common as dep --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 23c991a09..8f069aaf3 100644 --- a/setup.py +++ b/setup.py @@ -58,5 +58,5 @@ install_requires=["ansible >= 1.8", "paramiko >= 1.14", "PyYAML", "suds", "boto >= 2.29", "apache-libcloud >= 0.17", "RADL", "bottle", "netaddr", "requests", "scp", "cherrypy", "MySQL-python", "pysqlite", - "azure-mgmt-storage", "azure-mgmt-compute", "azure-mgmt-network", "azure-mgmt-resource"] + "azure-common", "azure-mgmt-storage", "azure-mgmt-compute", "azure-mgmt-network", "azure-mgmt-resource"] )