From 2b633cb22a633f06dd6806a59343b4f33f51c705 Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 15 Jan 2016 10:12:19 +0100 Subject: [PATCH 01/26] Update README docs to set the new YouTube links --- README | 15 ++++++--------- README.md | 16 ++++++++-------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/README b/README index af797df30..8ba625ef5 100644 --- a/README +++ b/README @@ -14,9 +14,8 @@ infrastructure. Read the documentation and more at http://www.grycap.upv.es/im. -There is also an Infrastructure Manager youtube channel with a set of videos with demos -of the functionality of the platform: https://www.youtube.com/channel/UCF16QmMHlRNtsC-0Cb2d8fg. - +There is also an Infrastructure Manager YouTube reproduction list with a set of videos with demos +of the functionality of the platform: https://www.youtube.com/playlist?list=PLgPH186Qwh_37AMhEruhVKZSfoYpHkrUp. 1. INSTALLATION =============== @@ -41,6 +40,10 @@ However, if you install IM from sources you should install: * The SOAPpy library for Python, typically available as the 'python-soappy' or 'SOAPpy' package. * The Netaddr library for Python, typically available as the 'python-netaddr' package. + + * The boto library version 2.29 or later must be installed (http://boto.readthedocs.org/en/latest/). + + * The apache-libcloud library version 0.18 or later must be installed (http://libcloud.apache.org/). * Ansible (http://www.ansibleworks.com/) to configure nodes in the infrastructures. In particular, Ansible 1.4.2+ must be installed. @@ -67,12 +70,6 @@ However, if you install IM from sources you should install: 1.2 OPTIONAL PACKAGES --------------------- -In case of using the Amazon EC2 plugin the boto library version 2.29 or later -must be installed (http://boto.readthedocs.org/en/latest/). - -In case of using the LibCloud plugin the apache-libcloud library version 0.17 or later -must be installed (http://libcloud.apache.org/). - In case of using the SSL secured version of the XMLRPC API the SpringPython framework (http://springpython.webfactional.com/) must be installed. diff --git a/README.md b/README.md index b220301a4..660a30fbf 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ infrastructure. Read the documentation and more at http://www.grycap.upv.es/im. -There is also an Infrastructure Manager youtube channel with a set of videos with demos -of the functionality of the platform: [YouTube IM channel](https://www.youtube.com/channel/UCF16QmMHlRNtsC-0Cb2d8fg) +There is also an Infrastructure Manager YouTube reproduction list with a set of videos with demos +of the functionality of the platform: https://www.youtube.com/playlist?list=PLgPH186Qwh_37AMhEruhVKZSfoYpHkrUp. 1. INSTALLATION @@ -43,6 +43,12 @@ However, if you install IM from sources you should install: + The SOAPpy library for Python, typically available as the 'python-soappy' or 'SOAPpy' package. + The Netaddr library for Python, typically available as the 'python-netaddr' package. + + + The boto library version 2.29 or later + must be installed (http://boto.readthedocs.org/en/latest/). + + + The apache-libcloud library version 0.18 or later + must be installed (http://libcloud.apache.org/). + Ansible (http://www.ansibleworks.com/) to configure nodes in the infrastructures. In particular, Ansible 1.4.2+ must be installed. @@ -71,12 +77,6 @@ pipelining = True 1.2 OPTIONAL PACKAGES --------------------- -In case of using the Amazon EC2 plugin the boto library version 2.29 or later -must be installed (http://boto.readthedocs.org/en/latest/). - -In case of using the LibCloud plugin the apache-libcloud library version 0.17 or later -must be installed (http://libcloud.apache.org/). - In case of using the SSL secured version of the XMLRPC API the SpringPython framework (http://springpython.webfactional.com/) must be installed. From 46ff97dff5f1847f9cb92d45e4097a18358172c2 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Jan 2016 14:35:01 +0100 Subject: [PATCH 02/26] Add support for new RADL JSON format --- IM/InfrastructureManager.py | 29 +++-- IM/REST.py | 83 +++++++++++-- IM/ServiceRequests.py | 4 +- IM/VirtualMachine.py | 2 +- IM/__init__.py | 2 +- IM/radl/radl.py | 5 + IM/radl/radl_json.py | 229 ++++++++++++++++++++++++++++++++++++ changelog | 3 + doc/source/REST.rst | 81 ++++++++----- test/TestRADL.py | 42 +++++++ test/TestREST_JSON.py | 188 +++++++++++++++++++++++++++++ test/test_simple.json | 24 ++++ 12 files changed, 636 insertions(+), 56 deletions(-) create mode 100644 IM/radl/radl_json.py create mode 100755 test/TestREST_JSON.py create mode 100644 test/test_simple.json diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index e9329e7e6..da05ec81c 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -31,7 +31,7 @@ import InfrastructureInfo from IM.radl import radl_parse -from IM.radl.radl import Feature +from IM.radl.radl import Feature, RADL from IM.recipe import Recipe from IM.db import DataBase @@ -259,7 +259,10 @@ def Reconfigure(inf_id, radl_data, auth, vm_list = None): """ InfrastructureManager.logger.info("Reconfiguring the inf: " + str(inf_id)) - radl = radl_parse.parse_radl(radl_data) + if isinstance(radl_data, RADL): + radl = radl_data + else: + radl = radl_parse.parse_radl(radl_data) InfrastructureManager.logger.debug(radl) sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) @@ -356,8 +359,11 @@ def AddResource(inf_id, radl_data, auth, context = True, failed_clouds = []): InfrastructureManager.logger.info("Adding resources to inf: " + str(inf_id)) - radl = radl_parse.parse_radl(radl_data) - radl.check() + if isinstance(radl_data, RADL): + radl = radl_data + else: + radl = radl_parse.parse_radl(radl_data) + radl.check() InfrastructureManager.logger.debug(radl) @@ -604,13 +610,7 @@ def GetVMProperty(inf_id, vm_id, property_name, auth): Return: a str with the property value """ - radl_data = InfrastructureManager.GetVMInfo(inf_id, vm_id, auth) - - try: - radl = radl_parse.parse_radl(radl_data) - except Exception, ex: - InfrastructureManager.logger.exception("Error parsing the RADL: " + radl_data) - raise ex + radl = InfrastructureManager.GetVMInfo(inf_id, vm_id, auth) res = None if radl.systems: @@ -689,7 +689,10 @@ def AlterVM(inf_id, vm_id, radl_data, auth): InfrastructureManager.logger.info("VM does not exist or Access Error") raise Exception("VM does not exist or Access Error") - radl = radl_parse.parse_radl(radl_data) + if isinstance(radl_data, RADL): + radl = radl_data + else: + radl = radl_parse.parse_radl(radl_data) exception = None try: @@ -706,7 +709,7 @@ def AlterVM(inf_id, vm_id, radl_data, auth): vm.update_status(auth) InfrastructureManager.save_data(inf_id) - return str(vm.info) + return vm.info @staticmethod def GetInfrastructureRADL(inf_id, auth): diff --git a/IM/REST.py b/IM/REST.py index 0fba58e6f..7ab3f86a8 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -21,6 +21,8 @@ import bottle import json from config import Config +from radl.radl_json import parse_radl as parse_radl_json, dump_radl as dump_radl_json +from radl.radl_parse import parse_radl AUTH_LINE_SEPARATOR = '\\n' @@ -105,6 +107,7 @@ def RESTDestroyInfrastructure(id=None): try: InfrastructureManager.DestroyInfrastructure(id, auth) + bottle.response.content_type = "text/plain" return "" except DeletedInfrastructureException, ex: bottle.abort(404, "Error Destroying Inf: " + str(ex)) @@ -209,7 +212,16 @@ def RESTCreateInfrastructure(): bottle.abort(401, "No authentication data provided") try: + content_type = bottle.request.headers.get('Content-Type') radl_data = bottle.request.body.read() + + if content_type: + if content_type == "application/json": + radl_data = parse_radl_json(radl_data) + elif content_type != "text/plain": + bottle.abort(415, "Unsupported Media Type %s" % content_type) + return False + inf_id = InfrastructureManager.CreateInfrastructure(radl_data, auth) bottle.response.content_type = "text/uri-list" @@ -230,8 +242,24 @@ def RESTGetVMInfo(infid=None, vmid=None): bottle.abort(401, "No authentication data provided") try: - info = InfrastructureManager.GetVMInfo(infid, vmid, auth) - bottle.response.content_type = "text/plain" + accept = bottle.request.headers.get('Accept') + + radl = InfrastructureManager.GetVMInfo(infid, vmid, auth) + + if accept: + if accept == "application/json": + bottle.response.content_type = accept + info = dump_radl_json(radl, enter="", indent="") + elif accept == "text/plain": + info = str(radl) + bottle.response.content_type = accept + else: + bottle.abort(404, "Unsupported Accept Media Type: %s" % accept) + return False + else: + info = str(radl) + bottle.response.content_type = "text/plain" + return info except DeletedInfrastructureException, ex: bottle.abort(404, "Error Getting VM. info: " + str(ex)) @@ -298,8 +326,17 @@ def RESTAddResource(id=None): context = False else: bottle.abort(400, "Incorrect value in context parameter") - + + content_type = bottle.request.headers.get('Content-Type') radl_data = bottle.request.body.read() + + if content_type: + if content_type == "application/json": + radl_data = parse_radl_json(radl_data) + elif content_type != "text/plain": + bottle.abort(415, "Unsupported Media Type %s" % content_type) + return False + vm_ids = InfrastructureManager.AddResource(id, radl_data, auth, context) res = "" @@ -340,6 +377,7 @@ def RESTRemoveResource(infid=None, vmid=None): bottle.abort(400, "Incorrect value in context parameter") InfrastructureManager.RemoveResource(infid, vmid, auth, context) + bottle.response.content_type = "text/plain" return "" except DeletedInfrastructureException, ex: bottle.abort(404, "Error Removing resources: " + str(ex)) @@ -366,10 +404,33 @@ def RESTAlterVM(infid=None, vmid=None): bottle.abort(401, "No authentication data provided") try: + content_type = bottle.request.headers.get('Content-Type') + accept = bottle.request.headers.get('Accept') radl_data = bottle.request.body.read() - bottle.response.content_type = "text/plain" - return InfrastructureManager.AlterVM(infid, vmid, radl_data, auth) + if content_type: + if content_type == "application/json": + radl_data = parse_radl_json(radl_data) + elif content_type != "text/plain": + bottle.abort(415, "Unsupported Media Type %s" % content_type) + return False + + vm_info = InfrastructureManager.AlterVM(infid, vmid, radl_data, auth) + + if accept: + if accept == "application/json": + bottle.response.content_type = accept + res = dump_radl_json(vm_info, enter="", indent="") + elif accept == "text/plain": + res = str(vm_info) + bottle.response.content_type = accept + else: + bottle.abort(404, "Unsupported Accept Media Type: %s" % accept) + return False + else: + bottle.response.content_type = "text/plain" + + return res except DeletedInfrastructureException, ex: bottle.abort(404, "Error modifying resources: " + str(ex)) return False @@ -403,8 +464,16 @@ def RESTReconfigureInfrastructure(id=None): except: bottle.abort(400, "Incorrect vm_list format.") - if 'radl' in bottle.request.forms.keys(): - radl_data = bottle.request.forms.get('radl') + content_type = bottle.request.headers.get('Content-Type') + radl_data = bottle.request.body.read() + + if radl_data: + if content_type: + if content_type == "application/json": + radl_data = parse_radl_json(radl_data) + elif content_type != "text/plain": + bottle.abort(415, "Unsupported Media Type %s" % content_type) + return False else: radl_data = "" return InfrastructureManager.Reconfigure(id, radl_data, auth, vm_list) diff --git a/IM/ServiceRequests.py b/IM/ServiceRequests.py index c399bc26a..0a7fd98a4 100644 --- a/IM/ServiceRequests.py +++ b/IM/ServiceRequests.py @@ -152,7 +152,7 @@ class Request_GetVMInfo(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Getting VM Info." (inf_id, vm_id, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.GetVMInfo(inf_id, vm_id, Authentication(auth_data)) + return str(InfrastructureManager.InfrastructureManager.GetVMInfo(inf_id, vm_id, Authentication(auth_data))) class Request_GetVMProperty(IMBaseRequest): """ @@ -170,7 +170,7 @@ class Request_AlterVM(IMBaseRequest): def _call_function(self): self._error_mesage = "Error Changing VM Info." (inf_id, vm_id, radl, auth_data) = self.arguments - return InfrastructureManager.InfrastructureManager.AlterVM(inf_id, vm_id, radl, Authentication(auth_data)) + return str(InfrastructureManager.InfrastructureManager.AlterVM(inf_id, vm_id, radl, Authentication(auth_data))) class Request_DestroyInfrastructure(IMBaseRequest): """ diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index 893407fc6..d4dc6df0b 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -730,7 +730,7 @@ def get_vm_info(self): res = RADL() res.networks = self.info.networks res.systems = self.info.systems - return str(res) + return res def get_ssh_ansible_master(self): ansible_host = None diff --git a/IM/__init__.py b/IM/__init__.py index 9e9ca5724..132412d54 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -16,6 +16,6 @@ __all__ = ['auth','bottle','CloudManager','config','ConfManager','db','ganglia','HTTPHeaderTransport','ImageManager','InfrastructureInfo','InfrastructureManager','parsetab','radl','recipe','request','REST', 'ServiceRequests','SSH','timedcall','uriparse','VMRC','xmlobject'] -__version__ = '1.4.1' +__version__ = '1.4.2' __author__ = 'Miguel Caballer' diff --git a/IM/radl/radl.py b/IM/radl/radl.py index b64019f2b..b36704178 100644 --- a/IM/radl/radl.py +++ b/IM/radl/radl.py @@ -1322,6 +1322,11 @@ def __init__(self, name, features, line=None): def __str__(self): return "ansible %s (%s)" % (self.id, Features.__str__(self)) + def getId(self): + """Return the id of the aspect.""" + + return self.id + def check(self, radl): """Check the features in this network.""" diff --git a/IM/radl/radl_json.py b/IM/radl/radl_json.py new file mode 100644 index 000000000..d81564744 --- /dev/null +++ b/IM/radl/radl_json.py @@ -0,0 +1,229 @@ +# IM - Infrastructure Manager +# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +import sys + +try: + import json +except ImportError: + import simplejson as json +try: + import yaml +except ImportError: + yaml = None + +from radl import Feature, Features, Aspect, RADL, configure, contextualize, contextualize_item, deploy, SoftFeatures, RADLParseException +import radl + +def encode_simple(d): + """Encode strings in basic python objects.""" + if isinstance(d, unicode): return d.encode() + if isinstance(d, list): return map(encode_simple, d) + if isinstance(d, dict): return dict([ (encode_simple(k), encode_simple(v)) for k, v in d.items() ]) + return d + +def parse_radl(data): + """ + Parse a RADL document in JSON. + + Args.: + - data(str or list): document to parse. + + Return(RADL): RADL object. + """ + if not isinstance(data, list): + if os.path.isfile(data): + f = open(data) + data = "".join(f.readlines()) + f.close() + data = json.loads(data) + data = encode_simple(data) + res = RADL() + for aspect in [ p_aspect(a) for a in data ]: + res.add(aspect) + return res + +def p_aspect(a): + assert "class" in a + if a["class"] == "configure": + return p_configure(a) + elif a["class"] == "contextualize": + return p_contextualize(a) + elif a["class"] == "deploy": + return p_deploy(a) + else: + return p_cfeatures(a) + +def p_configure(a): + assert a["class"] == "configure" + if a.get("reference", False): + return configure(a["id"], reference=True) + recipes = a["recipes"] + if isinstance(recipes, str) and yaml: + try: + yaml.safe_load(recipes) + except Exception, e: + raise RADLParseException("Error parsing YAML: %s" % str(e)) + return configure(a["id"], recipes) + +def p_contextualize(a): + assert a["class"] == "contextualize" + return contextualize([ p_contextualize_item(i) for i in a.get("items", []) ], + max_time=a.get("max_time", 0)) + +def p_contextualize_item(a): + return contextualize_item(a["system"], a["configure"], a.get("step", 0), a.get("ctxt_tool", None)) + +def p_deploy(a): + assert a["class"] == "deploy" + return deploy(a["system"], a["vm_number"], a.get("cloud", None)) + +def p_cfeatures(a): + assert a["class"] and a["id"] + cls = getattr(radl, a["class"]) + if a.get("reference", False): + return cls(a["id"], reference=True) + return cls(a["id"], p_features(a)) + +def p_features(a): + assert isinstance(a, dict) + def val(k, v): + if k == "softs": + return [ SoftFeatures(i.get("weight", 0), p_features(i.get("items", {}))) for i in v ] + elif k.endswith("_min") and isinstance(v, (int, float)): + return [ Feature(k[0:-4], ">=", v) ] + elif k.endswith("_max") and isinstance(v, (int, float)): + return [ Feature(k[0:-4], "<=", v) ] + elif isinstance(v, list): + return [ Feature(k, "contains", p_feature(i)) for i in v ] + else: + return [ Feature(k, "=", p_feature(v)) ] + return [ i for k, v in a.items() if k != "class" and k != "id" for i in val(k, v) ] + +def p_feature(a): + if isinstance(a, (int, float, str)): + return a + elif isinstance(a, unicode): + return str(a) + elif isinstance(a, dict) and "class" in a: + return p_cfeatures(a) + elif isinstance(a, dict): + return Features(p_features(a)) + assert False + +def dump_radl(radl, enter="\n", indent=" "): + """Dump a RADL document.""" + + indent = len(indent) if enter else None + sort_keys = indent is not None + separators = (",", ":" if indent is None else ": ") + return json.dumps(radlToSimple(radl), indent=indent, sort_keys=sort_keys, separators=separators) + +def radlToSimple(radl): + """ + Return a list of maps whose values are only other maps or lists. + """ + + aspects = radl.ansible_hosts + radl.networks + radl.systems + radl.configures + radl.deploys + if radl.contextualize.items is not None: + aspects.append(radl.contextualize) + return [ aspectToSimple(a) for a in aspects ] + +def aspectToSimple(a): + if isinstance(a, Features): + return cfeaturesToSimple(a) + elif isinstance(a, configure): + return configureToSimple(a) + elif isinstance(a, contextualize): + return contextualizeToSimple(a) + elif isinstance(a, deploy): + return deployToSimple(a) + assert False + +def configureToSimple(a): + assert isinstance(a, configure) + if a.reference or not a.recipes: + return { "class": "configure", "id": a.name, "reference": True } + else: + return { "class": "configure", "id": a.name, "recipes": a.recipes } + +def contextualizeToSimple(a): + assert isinstance(a, contextualize) + r = {"class": "contextualize"} + if a.max_time: r["max_time"] = a.max_time + if a.items: + r["items"] = [ contextualizeItemToSimple(i) for i in a.items.values() ] + return r + +def contextualizeItemToSimple(a): + assert isinstance(a, contextualize_item) + r = {"system": a.system, "configure": a.configure} + if a.num: r["step"] = a.num + if a.ctxt_tool: r["ctxt_tool"] = a.ctxt_tool + return r + +def deployToSimple(a): + assert isinstance(a, deploy) + r = {"class": "deploy", "system": a.id, "vm_number": a.vm_number} + if a.cloud_id: r["cloud"] = a.cloud_id + return r + +def cfeaturesToSimple(a): + assert isinstance(a, Features) + r = { "class": a.__class__.__name__, "id": a.getId() } + if a.reference: + r["reference"] = True + return r + r.update(featuresToSimple(a)) + return r + +def featuresToSimple(a): + assert isinstance(a, Features) + r = {} + for k, v in a.props.items(): + if k == SoftFeatures.SOFT: + r["softs"] = [ {"weight": i.soft, "items": featuresToSimple(i)} + for i in a.props[SoftFeatures.SOFT] ] + elif isinstance(v, tuple): + if v[0] and v[1] and v[0].value == v[1].value: + r[k] = v[0].value + elif v[0]: + r[k + "_min"] = v[0].value + elif v[1]: + r[k + "_max"] = v[1].value + elif isinstance(v, (set, list)): + r[k] = [ featureToSimple(i.value) for i in v ] + elif isinstance(v, dict): + r[k] = [ featureToSimple(i.value) for i in v.values() ] + else: + r[k] = featureToSimple(v.value) + return r + +def featureToSimple(a): + if isinstance(a, (int, float, str)): + return a + elif isinstance(a, unicode): + return str(a) + elif isinstance(a, Aspect): + return referenceToSimple(a) + elif isinstance(a, Features): + return featuresToSimple(a) + assert False + +def referenceToSimple(a): + assert isinstance(a, Aspect) + return { "class": a.__class__.__name__, "id": a.getId(), + "reference": True } diff --git a/changelog b/changelog index 67c126d25..96f795f9e 100644 --- a/changelog +++ b/changelog @@ -166,3 +166,6 @@ IM 1.4.1 * Add support for Ansible v2.X * Add supoort for using an external ansible master node * Bugfix in incorrects links inside containers + +IM 1.4.2 + * Add support for new RADL JSON format diff --git a/doc/source/REST.rst b/doc/source/REST.rst index 5a8e751e7..9443a675a 100644 --- a/doc/source/REST.rst +++ b/doc/source/REST.rst @@ -53,7 +53,7 @@ Next tables summaries the resources and the HTTP methods available. +-------------+--------------------------------------------+---------------------------------------------+ GET ``http://imserver.com/infrastructures`` - :Content-type: text/uri-list + :Response Content-type: text/uri-list :ok response: 200 OK :fail response: 401, 400 @@ -62,23 +62,24 @@ GET ``http://imserver.com/infrastructures`` POST ``http://imserver.com/infrastructures`` :body: ``RADL document`` - :Content-type: text/uri-list + :body Content-type: text/plain or application/json + :Response Content-type: text/uri-list :ok response: 200 OK - :fail response: 401, 400 + :fail response: 401, 400, 415 Create and configure an infrastructure with the requirements specified in - the RADL document of the body contents. If success, it is returned the - URI of the new infrastructure. + the RADL document of the body contents (in plain RADL or in JSON formats). + If success, it is returned the URI of the new infrastructure. GET ``http://imserver.com/infrastructures/`` - :Content-type: text/uri-list + :Response Content-type: text/uri-list :ok response: 200 OK :fail response: 401, 404, 400 Return a list of URIs referencing the virtual machines associated to the infrastructure with ID ``infId``. GET ``http://imserver.com/infrastructures//`` - :Content-type: application/json + :Response Content-type: text/plain or application/json :ok response: 200 OK :fail response: 401, 404, 400, 403 @@ -92,67 +93,75 @@ GET ``http://imserver.com/infrastructures//`` POST ``http://imserver.com/infrastructures/`` :body: ``RADL document`` + :body Content-type: text/plain or application/json :input fields: ``context`` (optional) - :Content-type: text/uri-list + :Response Content-type: text/uri-list :ok response: 200 OK - :fail response: 401, 404, 400 + :fail response: 401, 404, 400, 415 - Add the resources specified in the body contents to the infrastructure with ID - ``infId``. The RADL restrictions are the same as in + Add the resources specified in the body contents (in plain RADL or in JSON formats) + to the infrastructure with ID ``infId``. The RADL restrictions are the same as in :ref:`RPC-XML AddResource `. If success, it is returned a list of URIs of the new virtual machines. The ``context`` parameter is optional and is a flag to specify if the contextualization step will be launched just after the VM addition. Accetable values: yes, no, true, false, 1 or 0. If not specified the flag is set to True. PUT ``http://imserver.com/infrastructures//stop`` - :Content-type: text/uri-list + :Response Content-type: text/plain :ok response: 200 OK :fail response: 401, 404, 400 Perform the ``stop`` action in all the virtual machines in the - the infrastructure with ID ``infID``: + the infrastructure with ID ``infID``. If the operation has been performed + successfully the return value is an empty string. PUT ``http://imserver.com/infrastructures//start`` - :Content-type: text/uri-list + :Response Content-type: text/plain :ok response: 200 OK :fail response: 401, 404, 400 Perform the ``start`` action in all the virtual machines in the - the infrastructure with ID ``infID``: + the infrastructure with ID ``infID``. If the operation has been performed + successfully the return value is an empty string. PUT ``http://imserver.com/infrastructures//reconfigure`` - :input fields: ``radl`` (compulsory), ``vm_list`` (optional) - :Content-type: text/uri-list + :body: ``RADL document`` + :body Content-type: text/plain or application/json + :input fields: ``vm_list`` (optional) + :Response Content-type: text/plain :ok response: 200 OK - :fail response: 401, 404, 400 + :fail response: 401, 404, 400, 415 Perform the ``reconfigure`` action in all the virtual machines in the the infrastructure with ID ``infID``. It updates the configuration - of the infrastructure as indicated in ``radl``. The RADL restrictions - are the same as in :ref:`RPC-XML Reconfigure `. If no + of the infrastructure as indicated in the body contents (in plain RADL or in JSON formats). + The RADL restrictions are the same as in :ref:`RPC-XML Reconfigure `. If no RADL are specified, the contextualization process is stated again. - The last ``vm_list`` parameter is optional - and is a coma separated list of IDs of the VMs to reconfigure. If not - specified all the VMs will be reconfigured. + The ``vm_list`` parameter is optional and is a coma separated list of + IDs of the VMs to reconfigure. If not specified all the VMs will be reconfigured. + If the operation has been performed successfully the return value is an empty string. DELETE ``http://imserver.com/infrastructures/`` + :Response Content-type: text/plain :ok response: 200 OK :fail response: 401, 404, 400 Undeploy the virtual machines associated to the infrastructure with ID - ``infId``. + ``infId``. If the operation has been performed successfully + the return value is an empty string. GET ``http://imserver.com/infrastructures//vms/`` - :Content-type: text/plain + :Response Content-type: text/plain or application/json :ok response: 200 OK :fail response: 401, 404, 400 Return information about the virtual machine with ID ``vmId`` associated to - the infrastructure with ID ``infId``. The returned string is in RADL format. + the infrastructure with ID ``infId``. The returned string is in RADL format, + either in plain RADL or in JSON formats. See more the details of the output in :ref:`GetVMInfo `. GET ``http://imserver.com/infrastructures//vms//`` - :Content-type: text/plain + :Response Content-type: text/plain :ok response: 200 OK :fail response: 401, 404, 400 @@ -161,35 +170,43 @@ GET ``http://imserver.com/infrastructures//vms//`` PUT ``http://imserver.com/infrastructures//vms/`` :body: ``RADL document`` + :body Content-type: text/plain or application/json + :Response Content-type: text/plain or application/json :ok response: 200 OK - :fail response: 401, 404, 400 + :fail response: 401, 404, 400, 415 Change the features of the virtual machine with ID ``vmId`` in the infrastructure with with ID ``infId``, specified by the RADL document specified - in the body contents. + in the body contents (in plain RADL or in JSON formats). If the operation has + been performed successfully the return value the return value is an RADL document + with the VM properties modified (also in plain RADL or in JSON formats). DELETE ``http://imserver.com/infrastructures//vms/`` :input fields: ``context`` (optional) + :Response Content-type: text/plain :ok response: 200 OK :fail response: 401, 404, 400 Undeploy the virtual machine with ID ``vmId`` associated to the infrastructure with ID ``infId``. The ``context`` parameter is optional and is a flag to specify if the contextualization step will be launched just after the VM - addition. Accetable values: yes, no, true, false, 1 or 0. If not specified the flag is set to True. + addition. Accetable values: yes, no, true, false, 1 or 0. If not specified the flag is set to True. + If the operation has been performed successfully the return value is an empty string. PUT ``http://imserver.com/infrastructures//vms//start`` - :Content-type: text/plain + :Response Content-type: text/plain :ok response: 200 OK :fail response: 401, 404, 400 Perform the ``start`` action in the virtual machine with ID ``vmId`` associated to the infrastructure with ID ``infId``. + If the operation has been performed successfully the return value is an empty string. PUT ``http://imserver.com/infrastructures//vms//stop`` - :Content-type: text/plain + :Response Content-type: text/plain :ok response: 200 OK :fail response: 401, 404, 400 Perform the ``stop`` action in the virtual machine with ID ``vmId`` associated to the infrastructure with ID ``infId``. + If the operation has been performed successfully the return value is an empty string. diff --git a/test/TestRADL.py b/test/TestRADL.py index 240974c40..86f050abd 100755 --- a/test/TestRADL.py +++ b/test/TestRADL.py @@ -26,6 +26,7 @@ from IM.radl.radl_parse import parse_radl from IM.radl.radl import RADL, Features, Feature, RADLParseException, system +from IM.radl.radl_json import parse_radl as parse_radl_json, dump_radl as dump_radl_json import unittest class TestRADL(unittest.TestCase): @@ -49,6 +50,13 @@ def test_basic(self): self.assertIsInstance(s, system) self.assertEqual(len(s.features), 17) self.assertEqual(s.getValue("disk.0.os.name"), "linux") + + radl_json = dump_radl_json(r) + r = parse_radl_json(radl_json) + s = r.get_system_by_name("cursoaws") + self.assertIsInstance(s, system) + self.assertEqual(len(s.features), 17) + self.assertEqual(s.getValue("disk.0.os.name"), "linux") def test_basic0(self): @@ -57,11 +65,22 @@ def test_basic0(self): s = r.get_system_by_name("main") self.assertEqual(s.getValue("cpu.arch"), "x86_64") self.assertEqual(s.getValue("net_interface.0.connection"), "publica") + + radl_json = dump_radl_json(r) + r = parse_radl_json(radl_json) + self.radl_check(r, [2, 2, 0, 0, 0]) + s = r.get_system_by_name("main") + self.assertEqual(s.getValue("cpu.arch"), "x86_64") + self.assertEqual(s.getValue("net_interface.0.connection"), "publica") def test_references(self): r = parse_radl(TESTS_PATH + "/test_radl_ref.radl") self.radl_check(r, [2, 2, 0, 2, 2]) + + radl_json = dump_radl_json(r) + r = parse_radl_json(radl_json) + self.radl_check(r, [2, 2, 0, 2, 2]) def test_logic0(self): @@ -135,6 +154,15 @@ def test_concrete(self): self.assertIsInstance(concrete_s, system) self.assertEqual(score, 201) + radl_json = dump_radl_json(r) + r = parse_radl_json(radl_json) + self.radl_check(r) + s = r.get_system_by_name("main") + self.assertIsInstance(s, system) + concrete_s, score = s.concrete() + self.assertIsInstance(concrete_s, system) + self.assertEqual(score, 201) + def test_outports(self): @@ -194,6 +222,11 @@ def test_empty_contextualize(self): r.check() self.assertEqual(r.contextualize.items, {}) + radl_json = dump_radl_json(r) + r = parse_radl_json(radl_json) + r.check() + self.assertEqual(r.contextualize.items, {}) + radl = """ system test ( cpu.count>=1 @@ -204,6 +237,11 @@ def test_empty_contextualize(self): r = parse_radl(radl) r.check() self.assertEqual(r.contextualize.items, None) + + radl_json = dump_radl_json(r) + r = parse_radl_json(radl_json) + r.check() + self.assertEqual(r.contextualize.items, None) def test_ansible_host(self): @@ -217,6 +255,10 @@ def test_ansible_host(self): ) """ r = parse_radl(radl) self.radl_check(r) + + radl_json = dump_radl_json(r) + r = parse_radl_json(radl_json) + self.radl_check(r) radl = """ ansible ansible_master (host = 'host' and credentials.username = 'user' and credentials.password = 'pass') diff --git a/test/TestREST_JSON.py b/test/TestREST_JSON.py new file mode 100755 index 000000000..18478f116 --- /dev/null +++ b/test/TestREST_JSON.py @@ -0,0 +1,188 @@ +#! /usr/bin/env python +# +# IM - Infrastructure Manager +# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import unittest +import os +import httplib +import time +import sys +import json + +sys.path.append("..") +sys.path.append(".") + +from IM.VirtualMachine import VirtualMachine +from IM.uriparse import uriparse +from IM.radl.radl_json import parse_radl as parse_radl_json + +PID = None +RADL_ADD = '[{"class":"network","reference":true,"id":"publica"},{"class":"system","reference":true,"id":"front"},{"vm_number":1,"class":"deploy","system":"front"}]' +TESTS_PATH = os.path.dirname(os.path.realpath(__file__)) +RADL_FILE = TESTS_PATH + '/test_simple.json' +AUTH_FILE = TESTS_PATH + '/auth.dat' + +HOSTNAME = "localhost" +TEST_PORT = 8800 + +class TestIM(unittest.TestCase): + + server = None + auth_data = None + inf_id = 0 + + @classmethod + def setUpClass(cls): + cls.server = httplib.HTTPConnection(HOSTNAME, TEST_PORT) + f = open(AUTH_FILE) + cls.auth_data = "" + for line in f.readlines(): + cls.auth_data += line.strip() + "\\n" + f.close() + cls.inf_id = "0" + + @classmethod + def tearDownClass(cls): + # Assure that the infrastructure is destroyed + try: + cls.server.request('DELETE', "/infrastructures/" + cls.inf_id, headers = {'Authorization' : cls.auth_data}) + cls.server.getresponse() + except Exception: + pass + + def wait_inf_state(self, state, timeout, incorrect_states = [], vm_ids = None): + """ + Wait for an infrastructure to have a specific state + """ + if not vm_ids: + self.server.request('GET', "/infrastructures/" + self.inf_id, headers = {'AUTHORIZATION' : self.auth_data}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR getting infrastructure info:" + output) + + vm_ids = output.split("\n") + else: + pass + + err_states = [VirtualMachine.FAILED, VirtualMachine.OFF, VirtualMachine.UNCONFIGURED] + err_states.extend(incorrect_states) + + wait = 0 + all_ok = False + while not all_ok and wait < timeout: + all_ok = True + for vm_id in vm_ids: + vm_uri = uriparse(vm_id) + self.server.request('GET', vm_uri[2] + "/state", headers = {'AUTHORIZATION' : self.auth_data}) + resp = self.server.getresponse() + vm_state = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR getting VM info:" + vm_state) + + self.assertFalse(vm_state in err_states, msg="ERROR waiting for a state. '%s' state was expected and '%s' was obtained in the VM %s" % (state, vm_state, vm_uri)) + + if vm_state in err_states: + return False + elif vm_state != state: + all_ok = False + + if not all_ok: + wait += 5 + time.sleep(5) + + return all_ok + + def test_20_create(self): + f = open(RADL_FILE) + radl = "" + for line in f.readlines(): + radl += line + f.close() + + self.server.request('POST', "/infrastructures", body = radl, headers = {'AUTHORIZATION' : self.auth_data, 'Content-Type':'application/json'}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR creating the infrastructure:" + output) + + self.__class__.inf_id = str(os.path.basename(output)) + + all_configured = self.wait_inf_state(VirtualMachine.CONFIGURED, 600) + self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") + + def test_30_get_vm_info(self): + self.server.request('GET', "/infrastructures/" + self.inf_id, headers = {'AUTHORIZATION' : self.auth_data}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR getting the infrastructure info:" + output) + vm_ids = output.split("\n") + + vm_uri = uriparse(vm_ids[0]) + self.server.request('GET', vm_uri[2], headers = {'AUTHORIZATION' : self.auth_data, 'Accept':'application/json'}) + resp = self.server.getresponse() + ct = resp.getheader('Content-type') + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR getting VM info:" + output) + self.assertEqual(ct, "application/json", msg="ERROR getting VM info: Incorrect Content-type: %s" % ct) + parse_radl_json(output) + + def test_40_addresource(self): + self.server.request('POST', "/infrastructures/" + self.inf_id, body = RADL_ADD, headers = {'AUTHORIZATION' : self.auth_data, 'Content-Type':'application/json'}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR adding resources:" + output) + + self.server.request('GET', "/infrastructures/" + self.inf_id, headers = {'AUTHORIZATION' : self.auth_data}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR getting the infrastructure info:" + output) + vm_ids = output.split("\n") + self.assertEqual(len(vm_ids), 2, msg="ERROR getting infrastructure info: Incorrect number of VMs(" + str(len(vm_ids)) + "). It must be 2") + all_configured = self.wait_inf_state(VirtualMachine.CONFIGURED, 600) + self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") + + def test_55_reconfigure(self): + new_config = """ +[ +{ +"class": "configure", +"id": "front", +"recipes": "---\\n - tasks:\\n - debug: msg=RECONTEXTUALIZAMOS!\\n\\n" +} +] + """ + + self.server.request('PUT', "/infrastructures/" + self.inf_id + "/reconfigure", body = new_config, headers = {'AUTHORIZATION' : self.auth_data, 'Content-Type':'application/json'}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR reconfiguring:" + output) + + all_configured = self.wait_inf_state(VirtualMachine.CONFIGURED, 300) + self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") + + self.server.request('GET', "/infrastructures/" + self.inf_id + "/contmsg", headers = {'AUTHORIZATION' : self.auth_data}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR getting the infrastructure info:" + output) + self.assertNotEqual(output.find("RECONTEXTUALIZAMOS"), -1, msg="Incorrect contextualization message: " + output) + + def test_95_destroy(self): + self.server.request('DELETE', "/infrastructures/" + self.inf_id, headers = {'Authorization' : self.auth_data}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR destroying the infrastructure:" + output) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_simple.json b/test/test_simple.json new file mode 100644 index 000000000..e52cf6f6b --- /dev/null +++ b/test/test_simple.json @@ -0,0 +1,24 @@ +[ + { + "class": "network", + "id": "publica", + "outbound": "yes" + }, + { + "class": "system", + "cpu.arch": "x86_64", + "cpu.count_min": 1, + "disk.0.image.url": "one://ramses.i3m.upv.es/95", + "disk.0.os.credentials.password": "yoyoyo", + "disk.0.os.credentials.username": "ubuntu", + "disk.0.os.name": "linux", + "id": "front", + "memory.size_min": 536870912, + "net_interface.0.connection": "publica" + }, + { + "class": "deploy", + "system": "front", + "vm_number": 1 + } +] \ No newline at end of file From 8f389e14eb187833f2a575643a242f971a23f043 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Jan 2016 15:40:43 +0100 Subject: [PATCH 03/26] Bugfixes --- IM/InfrastructureManager.py | 2 +- IM/radl/__init__.py | 2 +- etc/im.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index da05ec81c..3e042fd04 100755 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -363,9 +363,9 @@ def AddResource(inf_id, radl_data, auth, context = True, failed_clouds = []): radl = radl_data else: radl = radl_parse.parse_radl(radl_data) - radl.check() InfrastructureManager.logger.debug(radl) + radl.check() sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) diff --git a/IM/radl/__init__.py b/IM/radl/__init__.py index dc055b705..ea76271ff 100644 --- a/IM/radl/__init__.py +++ b/IM/radl/__init__.py @@ -15,4 +15,4 @@ # along with this program. If not, see . -__all__ = ['radl_lex','radl_parse','radl'] +__all__ = ['radl_lex','radl_parse','radl_json','radl'] diff --git a/etc/im.cfg b/etc/im.cfg index f7fe593c0..798f2f698 100644 --- a/etc/im.cfg +++ b/etc/im.cfg @@ -60,7 +60,7 @@ DEFAULT_VM_NAME = vnode-#N# DEFAULT_DOMAIN = localdomain # REST API Info -ACTIVATE_REST = False +ACTIVATE_REST = True REST_PORT = 8800 REST_ADDRESS = 0.0.0.0 From 2c51e58c8a8e12d11083ae2ca9969f0e8cb5cb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20P=C3=A9rez?= Date: Tue, 19 Jan 2016 16:44:15 +0100 Subject: [PATCH 04/26] Update Dockerfile Added REST Port to the exposed ports of the container --- docker-devel/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index f42dce53f..02f20ad70 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Miguel Caballer LABEL version="1.4.1" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" -EXPOSE 8899 +EXPOSE 8899 8800 RUN apt-get update && apt-get install -y \ gcc \ From bcf66401fc5945a59db494432ea244f8e2e03dc7 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 Jan 2016 09:39:39 +0100 Subject: [PATCH 05/26] Minor bugfixes and docker files update --- connectors/FogBow.py | 6 ++++++ connectors/OpenNebula.py | 3 +++ connectors/OpenStack.py | 6 ++++++ docker-devel/Dockerfile | 2 +- docker-devel/ansible.cfg | 4 ++-- docker/Dockerfile | 2 +- docker/ansible.cfg | 4 ++-- 7 files changed, 21 insertions(+), 6 deletions(-) diff --git a/connectors/FogBow.py b/connectors/FogBow.py index d8b0459df..3bb7c6742 100644 --- a/connectors/FogBow.py +++ b/connectors/FogBow.py @@ -54,6 +54,12 @@ class FogBowCloudConnector(CloudConnector): } """Dictionary with a map with the FogBow Request states to the IM states.""" + def __init__(self, cloud_info): + # check if the user has specified the http protocol in the host and remove it + pos = cloud_info.server.find('://') + cloud_info.server = cloud_info.server[pos+3:] + CloudConnector.__init__(self, cloud_info) + def get_auth_headers(self, auth_data): """ Generate the auth header needed to contact with the FogBow server. diff --git a/connectors/OpenNebula.py b/connectors/OpenNebula.py index 386003850..b4b5a8eea 100644 --- a/connectors/OpenNebula.py +++ b/connectors/OpenNebula.py @@ -103,6 +103,9 @@ class OpenNebulaCloudConnector(CloudConnector): """str with the name of the provider.""" def __init__(self, cloud_info): + # check if the user has specified the http protocol in the host and remove it + pos = cloud_info.server.find('://') + cloud_info.server = cloud_info.server[pos+3:] CloudConnector.__init__(self, cloud_info) self.server_url = "http://%s:%d/RPC2" % (self.cloud.server, self.cloud.port) diff --git a/connectors/OpenStack.py b/connectors/OpenStack.py index 14fcc5410..b1dab8df8 100644 --- a/connectors/OpenStack.py +++ b/connectors/OpenStack.py @@ -32,6 +32,12 @@ class OpenStackCloudConnector(LibCloudCloudConnector): type = "OpenStack" """str with the name of the provider.""" + def __init__(self, cloud_info): + # check if the user has specified the http protocol in the host and remove it + pos = cloud_info.server.find('://') + cloud_info.server = cloud_info.server[pos+3:] + LibCloudCloudConnector.__init__(self, cloud_info) + def get_driver(self, auth_data): """ Get the driver from the auth data diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index f42dce53f..c2f7afbf5 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM ubuntu:14.04 MAINTAINER Miguel Caballer -LABEL version="1.4.1" +LABEL version="1.4.2" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 diff --git a/docker-devel/ansible.cfg b/docker-devel/ansible.cfg index 3cfba7837..2ae9bcc53 100644 --- a/docker-devel/ansible.cfg +++ b/docker-devel/ansible.cfg @@ -1,8 +1,8 @@ [defaults] transport = smart host_key_checking = False -sudo_user = root -sudo_exe = sudo +become_user = root +become_method = sudo [paramiko_connection] diff --git a/docker/Dockerfile b/docker/Dockerfile index a1febc08f..855a6d3e0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM ubuntu:14.04 MAINTAINER Miguel Caballer -LABEL version="1.4.1" +LABEL version="1.4.2" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 RUN apt-get update && apt-get install -y gcc python-dev python-pip python-soappy openssh-client sshpass diff --git a/docker/ansible.cfg b/docker/ansible.cfg index 3cfba7837..2ae9bcc53 100644 --- a/docker/ansible.cfg +++ b/docker/ansible.cfg @@ -1,8 +1,8 @@ [defaults] transport = smart host_key_checking = False -sudo_user = root -sudo_exe = sudo +become_user = root +become_method = sudo [paramiko_connection] From 666cfa02b02afd1b3035c854e99e865764a841f5 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 Jan 2016 10:01:24 +0100 Subject: [PATCH 06/26] Change in Auth Header in new version of FogBow --- connectors/FogBow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/connectors/FogBow.py b/connectors/FogBow.py index 3bb7c6742..ba449564a 100644 --- a/connectors/FogBow.py +++ b/connectors/FogBow.py @@ -71,8 +71,7 @@ def get_auth_headers(self, auth_data): plugin = IdentityPlugin.getIdentityPlugin(token_type) token = plugin.create_token(auth[0]).replace("\n", "").replace("\r", "") - auth_headers = {'X-Federation-Auth-Token' : token} - #auth_headers = {'X-Auth-Token' : token, 'X-Local-Auth-Token' : token, 'Authorization' : token} + auth_headers = {'X-Auth-Token' : token} return auth_headers else: From f3e2641ca5f365a21374caf0dec82cacce508ea5 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 Jan 2016 10:02:48 +0100 Subject: [PATCH 07/26] Update changelog --- changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog b/changelog index 96f795f9e..f8dee4086 100644 --- a/changelog +++ b/changelog @@ -169,3 +169,5 @@ IM 1.4.1 IM 1.4.2 * Add support for new RADL JSON format + * Change in Auth Header in new version of FogBow + * Code improvements in OpenStack, OpenNebula and FogBow connectors From 7cf23b7252ef5901aff7718984cd7bcec35b8c88 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 Jan 2016 10:19:55 +0100 Subject: [PATCH 08/26] Code improvements --- connectors/FogBow.py | 88 +++++++++++++++++++++++----------------- connectors/OpenNebula.py | 14 +++---- 2 files changed, 57 insertions(+), 45 deletions(-) diff --git a/connectors/FogBow.py b/connectors/FogBow.py index ba449564a..f33c75503 100644 --- a/connectors/FogBow.py +++ b/connectors/FogBow.py @@ -68,16 +68,16 @@ def get_auth_headers(self, auth_data): if auth and 'token_type' in auth[0]: token_type = auth[0]['token_type'] - plugin = IdentityPlugin.getIdentityPlugin(token_type) - token = plugin.create_token(auth[0]).replace("\n", "").replace("\r", "") - - auth_headers = {'X-Auth-Token' : token} - - return auth_headers else: - raise Exception("Incorrect auth data") - self.logger.error("Incorrect auth data") + # If not token_type supplied, we assume that is VOMS one + token_type = 'VOMS' + + plugin = IdentityPlugin.getIdentityPlugin(token_type) + token = plugin.create_token(auth[0]).replace("\n", "").replace("\r", "") + auth_headers = {'X-Auth-Token' : token} + + return auth_headers def concreteSystem(self, radl_system, auth_data): image_urls = radl_system.getValue("disk.0.image.url") @@ -394,19 +394,28 @@ class OpenNebulaIdentityPlugin(IdentityPlugin): @staticmethod def create_token(params): - return params['username'] + ":" + params['password'] + if 'username' in params and 'password' in params: + return params['username'] + ":" + params['password'] + else: + raise Exception("Incorrect auth data, username and password must be specified") class X509IdentityPlugin(IdentityPlugin): @staticmethod def create_token(params): - return params['proxy'] + if 'proxy' in params: + return params['proxy'] + else: + raise Exception("Incorrect auth data, proxy must be specified") class VOMSIdentityPlugin(IdentityPlugin): @staticmethod def create_token(params): - return params['proxy'] + if 'proxy' in params: + return params['proxy'] + else: + raise Exception("Incorrect auth data, no proxy has been specified") class KeyStoneIdentityPlugin(IdentityPlugin): """ @@ -418,32 +427,35 @@ def create_token(params): """ Contact the specified keystone server to return the token """ - try: - keystone_uri = params['auth_url'] - uri = uriparse(keystone_uri) - server = uri[1].split(":")[0] - port = int(uri[1].split(":")[1]) - - conn = httplib.HTTPSConnection(server, port) - conn.putrequest('POST', "/v2.0/tokens") - conn.putheader('Accept', 'application/json') - conn.putheader('Content-Type', 'application/json') - conn.putheader('Connection', 'close') - - body = '{"auth":{"passwordCredentials":{"username": "' + params['username'] + '","password": "' + params['password'] + '"},"tenantName": "' + params['tenant'] + '"}}' - - conn.putheader('Content-Length', len(body)) - conn.endheaders(body) + if 'username' in params and 'password' in params and 'auth_url' in params and 'tenant' in params: + try: + keystone_uri = params['auth_url'] + uri = uriparse(keystone_uri) + server = uri[1].split(":")[0] + port = int(uri[1].split(":")[1]) - resp = conn.getresponse() - - # format: -> "{\"access\": {\"token\": {\"issued_at\": \"2014-12-29T17:10:49.609894\", \"expires\": \"2014-12-30T17:10:49Z\", \"id\": \"c861ab413e844d12a61d09b23dc4fb9c\"}, \"serviceCatalog\": [], \"user\": {\"username\": \"/DC=es/DC=irisgrid/O=upv/CN=miguel-caballer\", \"roles_links\": [], \"id\": \"475ce4978fb042e49ce0391de9bab49b\", \"roles\": [], \"name\": \"/DC=es/DC=irisgrid/O=upv/CN=miguel-caballer\"}, \"metadata\": {\"is_admin\": 0, \"roles\": []}}}" - output = json.loads(resp.read()) - token_id = output['access']['token']['id'] - - if conn.cert_file and os.path.isfile(conn.cert_file): - os.unlink(conn.cert_file) + conn = httplib.HTTPSConnection(server, port) + conn.putrequest('POST', "/v2.0/tokens") + conn.putheader('Accept', 'application/json') + conn.putheader('Content-Type', 'application/json') + conn.putheader('Connection', 'close') + + body = '{"auth":{"passwordCredentials":{"username": "' + params['username'] + '","password": "' + params['password'] + '"},"tenantName": "' + params['tenant'] + '"}}' + + conn.putheader('Content-Length', len(body)) + conn.endheaders(body) - return token_id - except: - return None \ No newline at end of file + resp = conn.getresponse() + + # format: -> "{\"access\": {\"token\": {\"issued_at\": \"2014-12-29T17:10:49.609894\", \"expires\": \"2014-12-30T17:10:49Z\", \"id\": \"c861ab413e844d12a61d09b23dc4fb9c\"}, \"serviceCatalog\": [], \"user\": {\"username\": \"/DC=es/DC=irisgrid/O=upv/CN=miguel-caballer\", \"roles_links\": [], \"id\": \"475ce4978fb042e49ce0391de9bab49b\", \"roles\": [], \"name\": \"/DC=es/DC=irisgrid/O=upv/CN=miguel-caballer\"}, \"metadata\": {\"is_admin\": 0, \"roles\": []}}}" + output = json.loads(resp.read()) + token_id = output['access']['token']['id'] + + if conn.cert_file and os.path.isfile(conn.cert_file): + os.unlink(conn.cert_file) + + return token_id + except: + return None + else: + raise Exception("Incorrect auth data, auth_url, username, password and tenant must be specified") \ No newline at end of file diff --git a/connectors/OpenNebula.py b/connectors/OpenNebula.py index b4b5a8eea..39b214990 100644 --- a/connectors/OpenNebula.py +++ b/connectors/OpenNebula.py @@ -220,7 +220,7 @@ def updateVMInfo(self, vm, auth_data): session_id = self.getSessionID(auth_data) if session_id == None: - return (False, "Incorrect auth data") + return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") func_res = server.one.vm.info(session_id, int(vm.id)) if len(func_res) == 2: @@ -275,7 +275,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): server = xmlrpclib.ServerProxy(self.server_url,allow_none=True) session_id = self.getSessionID(auth_data) if session_id == None: - return [(False, "Incorrect auth data")] + return [(False, "Incorrect auth data, username and password must be specified for OpenNebula provider.")] system = radl.systems[0] # Currently ONE plugin prioritizes user-password credentials @@ -308,7 +308,7 @@ def finalize(self, vm, auth_data): server = xmlrpclib.ServerProxy(self.server_url,allow_none=True) session_id = self.getSessionID(auth_data) if session_id == None: - return (False, "Incorrect auth data") + return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") func_res = server.one.vm.action(session_id, 'finalize', int(vm.id)) if len(func_res) == 1: @@ -327,7 +327,7 @@ def stop(self, vm, auth_data): server = xmlrpclib.ServerProxy(self.server_url,allow_none=True) session_id = self.getSessionID(auth_data) if session_id == None: - return (False, "Incorrect auth data") + return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") func_res = server.one.vm.action(session_id, 'suspend', int(vm.id)) if len(func_res) == 1: @@ -346,7 +346,7 @@ def start(self, vm, auth_data): server = xmlrpclib.ServerProxy(self.server_url,allow_none=True) session_id = self.getSessionID(auth_data) if session_id == None: - return (False, "Incorrect auth data") + return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") func_res = server.one.vm.action(session_id, 'resume', int(vm.id)) if len(func_res) == 1: @@ -732,7 +732,7 @@ def poweroff(self, vm, auth_data, timeout = 60): server = xmlrpclib.ServerProxy(self.server_url,allow_none=True) session_id = self.getSessionID(auth_data) if session_id == None: - return (False, "Incorrect auth data") + return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") func_res = server.one.vm.action(session_id, 'poweroff', int(vm.id)) if len(func_res) == 1: @@ -774,7 +774,7 @@ def alterVM(self, vm, radl, auth_data): server = xmlrpclib.ServerProxy(self.server_url,allow_none=True) session_id = self.getSessionID(auth_data) if session_id == None: - return (False, "Incorrect auth data") + return (False, "Incorrect auth data, username and password must be specified for OpenNebula provider.") if self.checkResize(): if not radl.systems: From 84051a6cf77efb84e4a6592c0c4377547355d4b9 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 Jan 2016 10:29:46 +0100 Subject: [PATCH 09/26] Update docker devel file --- docker-devel/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 557d463f0..22149d755 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -19,12 +19,6 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Install tosca-parser -RUN cd tmp \ - && git clone https://github.com/indigo-dc/tosca-parser.git \ - && cd tosca-parser \ - && python setup.py install - # Install im - 'devel' branch RUN cd tmp \ && git clone -b devel https://github.com/grycap/im.git \ From 9fba382d36e5a7b113231e278bb21e697c6547b6 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 Jan 2016 15:43:37 +0100 Subject: [PATCH 10/26] Bugfixes --- connectors/FogBow.py | 3 ++- connectors/OpenNebula.py | 3 ++- connectors/OpenStack.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/connectors/FogBow.py b/connectors/FogBow.py index f33c75503..f0898665b 100644 --- a/connectors/FogBow.py +++ b/connectors/FogBow.py @@ -57,7 +57,8 @@ class FogBowCloudConnector(CloudConnector): def __init__(self, cloud_info): # check if the user has specified the http protocol in the host and remove it pos = cloud_info.server.find('://') - cloud_info.server = cloud_info.server[pos+3:] + if pos != -1: + cloud_info.server = cloud_info.server[pos+3:] CloudConnector.__init__(self, cloud_info) def get_auth_headers(self, auth_data): diff --git a/connectors/OpenNebula.py b/connectors/OpenNebula.py index 39b214990..378b2d673 100644 --- a/connectors/OpenNebula.py +++ b/connectors/OpenNebula.py @@ -105,7 +105,8 @@ class OpenNebulaCloudConnector(CloudConnector): def __init__(self, cloud_info): # check if the user has specified the http protocol in the host and remove it pos = cloud_info.server.find('://') - cloud_info.server = cloud_info.server[pos+3:] + if pos != -1: + cloud_info.server = cloud_info.server[pos+3:] CloudConnector.__init__(self, cloud_info) self.server_url = "http://%s:%d/RPC2" % (self.cloud.server, self.cloud.port) diff --git a/connectors/OpenStack.py b/connectors/OpenStack.py index b1dab8df8..80e58d632 100644 --- a/connectors/OpenStack.py +++ b/connectors/OpenStack.py @@ -35,7 +35,8 @@ class OpenStackCloudConnector(LibCloudCloudConnector): def __init__(self, cloud_info): # check if the user has specified the http protocol in the host and remove it pos = cloud_info.server.find('://') - cloud_info.server = cloud_info.server[pos+3:] + if pos != -1: + cloud_info.server = cloud_info.server[pos+3:] LibCloudCloudConnector.__init__(self, cloud_info) def get_driver(self, auth_data): From 5c61d2af72ce78c054c74ee53bb9d608ebdf7f31 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 Jan 2016 16:52:29 +0100 Subject: [PATCH 11/26] Make the tests fast --- test/TestREST.py | 4 ++-- test/test_simple.radl | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/TestREST.py b/test/TestREST.py index ea06f8259..12eb3d89d 100755 --- a/test/TestREST.py +++ b/test/TestREST.py @@ -166,14 +166,14 @@ def test_32_get_vm_contmsg(self): resp = self.server.getresponse() output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR getting VM contmsg:" + output) - self.assertGreater(len(output), 100, msg="Incorrect VM contextualization message: " + output) + self.assertEqual(len(output), 0, msg="Incorrect VM contextualization message: " + output) def test_33_get_contmsg(self): self.server.request('GET', "/infrastructures/" + self.inf_id + "/contmsg", headers = {'AUTHORIZATION' : self.auth_data}) resp = self.server.getresponse() output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR getting the infrastructure info:" + output) - self.assertGreater(len(output), 100, msg="Incorrect contextualization message: " + output) + self.assertGreater(len(output), 30, msg="Incorrect contextualization message: " + output) def test_34_get_radl(self): self.server.request('GET', "/infrastructures/" + self.inf_id + "/radl", headers = {'AUTHORIZATION' : self.auth_data}) diff --git a/test/test_simple.radl b/test/test_simple.radl index 2a5aef9e7..4b17b5f24 100644 --- a/test/test_simple.radl +++ b/test/test_simple.radl @@ -11,4 +11,6 @@ disk.0.os.credentials.password = 'yoyoyo' and disk.0.os.name = 'linux' ) +contextualize () + deploy front 1 \ No newline at end of file From 3d99d4a959aae0226bb6ab0cbb74c1e2b34e8479 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 20 Jan 2016 16:52:39 +0100 Subject: [PATCH 12/26] Bugfixes --- connectors/FogBow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/connectors/FogBow.py b/connectors/FogBow.py index f0898665b..f0480a1c4 100644 --- a/connectors/FogBow.py +++ b/connectors/FogBow.py @@ -207,6 +207,10 @@ def updateVMInfo(self, vm, auth_data): if len(parts) > 1: vm.setSSHPort(int(parts[1])) + ssh_user = self.get_occi_attribute_value(output, 'org.fogbowcloud.request.ssh-username') + if ssh_user: + vm.info.systems[0].selValue('disk.0.os.credentials.username',ssh_user) + return (True, vm) except Exception, ex: From b296c67d9c0634af7d671b6c5071a0387854039d Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 Jan 2016 11:43:00 +0100 Subject: [PATCH 13/26] Add support to requirements in FogBow --- changelog | 2 +- connectors/FogBow.py | 64 +++++++++++++++++--------------------------- 2 files changed, 25 insertions(+), 41 deletions(-) diff --git a/changelog b/changelog index f8dee4086..cf6a7542f 100644 --- a/changelog +++ b/changelog @@ -169,5 +169,5 @@ IM 1.4.1 IM 1.4.2 * Add support for new RADL JSON format - * Change in Auth Header in new version of FogBow + * Change in Auth Header in new version of FogBow and support for requirements * Code improvements in OpenStack, OpenNebula and FogBow connectors diff --git a/connectors/FogBow.py b/connectors/FogBow.py index f0480a1c4..ef265e6b2 100644 --- a/connectors/FogBow.py +++ b/connectors/FogBow.py @@ -15,12 +15,9 @@ # along with this program. If not, see . import json -import subprocess -import shutil import os import sys import httplib -import tempfile from IM.uriparse import uriparse from IM.VirtualMachine import VirtualMachine from CloudConnector import CloudConnector @@ -34,7 +31,7 @@ class FogBowCloudConnector(CloudConnector): type = "FogBow" """str with the name of the provider.""" - INSTANCE_TYPE = 'small' + INSTANCE_TYPE = 'fogbow_small' """str with the name of the default instance type to launch.""" VM_STATE_MAP = { @@ -94,9 +91,6 @@ def concreteSystem(self, radl_system, auth_data): protocol = url[0] if protocol in ['fbw']: res_system = radl_system.clone() - - if not res_system.hasFeature('instance_type'): - res_system.addFeature(Feature("instance_type", "=", self.INSTANCE_TYPE), conflict="me", missing="other") res_system.addFeature(Feature("disk.0.image.url", "=", str_url), conflict="other", missing="other") @@ -209,7 +203,7 @@ def updateVMInfo(self, vm, auth_data): ssh_user = self.get_occi_attribute_value(output, 'org.fogbowcloud.request.ssh-username') if ssh_user: - vm.info.systems[0].selValue('disk.0.os.credentials.username',ssh_user) + vm.info.systems[0].addFeature(Feature("disk.0.os.credentials.username", "=", ssh_user), conflict="other", missing="other") return (True, vm) @@ -217,35 +211,6 @@ def updateVMInfo(self, vm, auth_data): self.logger.exception("Error connecting with FogBow Manager") return (False, "Error connecting with FogBow Manager: " + str(ex)) - def keygen(self): - """ - Generates a keypair using the ssh-keygen command and returns a tuple (public, private) - """ - tmp_dir = tempfile.mkdtemp() - pk_file = tmp_dir + "/occi-key" - command = 'ssh-keygen -t rsa -b 2048 -q -N "" -f ' + pk_file - p=subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - (out, err) = p.communicate() - if p.returncode!=0: - shutil.rmtree(tmp_dir, ignore_errors=True) - self.logger.error("Error executing ssh-keygen: " + out + err) - return (None, None) - else: - public = None - private = None - try: - with open(pk_file) as f: private = f.read() - except: - self.logger.exception("Error reading private_key file.") - - try: - with open(pk_file + ".pub") as f: public = f.read() - except: - self.logger.exception("Error reading public_key file.") - - shutil.rmtree(tmp_dir, ignore_errors=True) - return (public, private) - def launch(self, inf, radl, requested_radl, num_vm, auth_data): system = radl.systems[0] auth_headers = self.get_auth_headers(auth_data) @@ -283,16 +248,35 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): conn.putheader(k, v) conn.putheader('Category', 'fogbow_request; scheme="http://schemas.fogbowcloud.org/request#"; class="kind"') - + conn.putheader('X-OCCI-Attribute', 'org.fogbowcloud.request.instance-count=1') conn.putheader('X-OCCI-Attribute', 'org.fogbowcloud.request.type="one-time"') - conn.putheader('Category', 'fogbow_' + system.getValue('instance_type') + '; scheme="http://schemas.fogbowcloud.org/template/resource#"; class="mixin"') + requirements = "" + if system.getValue('instance_type'): + conn.putheader('Category', system.getValue('instance_type') + '; scheme="http://schemas.fogbowcloud.org/template/resource#"; class="mixin"') + else: + cpu = system.getValue('cpu.count') + memory = system.getFeature('memory.size').getValue('M') + if cpu: + requirements += "Glue2vCPU >= %d" % cpu + if memory: + if requirements: + requirements += " && " + requirements += "Glue2RAM >= %d" % memory + conn.putheader('Category', os_tpl + '; scheme="http://schemas.fogbowcloud.org/template/os#"; class="mixin"') conn.putheader('Category', 'fogbow_public_key; scheme="http://schemas.fogbowcloud/credentials#"; class="mixin"') - conn.putheader('X-OCCI-Attribute', 'org.fogbowcloud.credentials.publickey.data="' + public_key.strip() + '"') + if system.getValue('availability_zone'): + if requirements: + requirements += ' && ' + requirements += 'Glue2CloudComputeManagerID == "%s"' % system.getValue('availability_zone') + + if requirements: + conn.putheader('X-OCCI-Attribute', 'org.fogbowcloud.request.requirements=' + requirements) + conn.endheaders() resp = conn.getresponse() From ab0ad5eebf318a0174d7fd2938eb4da4a5d4cc70 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 Jan 2016 12:44:56 +0100 Subject: [PATCH 14/26] Simplify REST test --- test/TestREST.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/TestREST.py b/test/TestREST.py index 12eb3d89d..d70681bdc 100755 --- a/test/TestREST.py +++ b/test/TestREST.py @@ -225,14 +225,8 @@ def test_45_getstate(self): for vm_id, vm_state in vm_states.iteritems(): self.assertEqual(vm_state, "configured", msg="Unexpected vm state: " + vm_state + " in VM ID " + str(vm_id) + ". It must be 'configured'.") - def test_46_addresource_noconfig(self): - self.server.request('POST', "/infrastructures/" + self.inf_id + "?context=0", body = RADL_ADD, headers = {'AUTHORIZATION' : self.auth_data}) - resp = self.server.getresponse() - output = str(resp.read()) - self.assertEqual(resp.status, 200, msg="ERROR adding resources:" + output) - - def test_47_removeresource_noconfig(self): - self.server.request('GET', "/infrastructures/" + self.inf_id + "?context=0", headers = {'AUTHORIZATION' : self.auth_data}) + def test_46_removeresource(self): + self.server.request('GET', "/infrastructures/" + self.inf_id, headers = {'AUTHORIZATION' : self.auth_data}) resp = self.server.getresponse() output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR getting the infrastructure info:" + output) @@ -249,10 +243,19 @@ def test_47_removeresource_noconfig(self): output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR getting the infrastructure info:" + output) vm_ids = output.split("\n") - self.assertEqual(len(vm_ids), 2, msg="ERROR getting infrastructure info: Incorrect number of VMs(" + str(len(vm_ids)) + "). It must be 1") + self.assertEqual(len(vm_ids), 1, msg="ERROR getting infrastructure info: Incorrect number of VMs(" + str(len(vm_ids)) + "). It must be 1") - def test_50_removeresource(self): - self.server.request('GET', "/infrastructures/" + self.inf_id, headers = {'AUTHORIZATION' : self.auth_data}) + all_configured = self.wait_inf_state(VirtualMachine.CONFIGURED, 300) + self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") + + def test_47_addresource_noconfig(self): + self.server.request('POST', "/infrastructures/" + self.inf_id + "?context=0", body = RADL_ADD, headers = {'AUTHORIZATION' : self.auth_data}) + resp = self.server.getresponse() + output = str(resp.read()) + self.assertEqual(resp.status, 200, msg="ERROR adding resources:" + output) + + def test_50_removeresource_noconfig(self): + self.server.request('GET', "/infrastructures/" + self.inf_id + "?context=0", headers = {'AUTHORIZATION' : self.auth_data}) resp = self.server.getresponse() output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR getting the infrastructure info:" + output) @@ -271,9 +274,6 @@ def test_50_removeresource(self): vm_ids = output.split("\n") self.assertEqual(len(vm_ids), 1, msg="ERROR getting infrastructure info: Incorrect number of VMs(" + str(len(vm_ids)) + "). It must be 1") - all_configured = self.wait_inf_state(VirtualMachine.CONFIGURED, 300) - self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") - def test_55_reconfigure(self): self.server.request('PUT', "/infrastructures/" + self.inf_id + "/reconfigure", headers = {'AUTHORIZATION' : self.auth_data}) resp = self.server.getresponse() From ef0eed934f8c60ad423ef507db1e5ffa98e0e9f9 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 Jan 2016 13:10:21 +0100 Subject: [PATCH 15/26] Bugfix in getvmproperty in json --- IM/REST.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/IM/REST.py b/IM/REST.py index 7ab3f86a8..1d9f42173 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -290,7 +290,21 @@ def RESTGetVMProperty(infid=None, vmid=None, prop=None): info = InfrastructureManager.GetVMContMsg(infid, vmid, auth) else: info = InfrastructureManager.GetVMProperty(infid, vmid, prop, auth) - bottle.response.content_type = "text/plain" + + accept = bottle.request.headers.get('Accept') + if accept: + if accept == "application/json": + bottle.response.content_type = accept + if isinstance(info, str) or isinstance(info, unicode): + info = '"' + info + '"' + elif accept == "text/plain": + bottle.response.content_type = accept + else: + bottle.abort(404, "Unsupported Accept Media Type: %s" % accept) + return False + else: + bottle.response.content_type = "text/plain" + return str(info) except DeletedInfrastructureException, ex: bottle.abort(404, "Error Getting VM. property: " + str(ex)) From 36c50474e5c4112ee34e99777e920c8978af9859 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 Jan 2016 13:49:44 +0100 Subject: [PATCH 16/26] Add JSON RADL in docs --- doc/source/radl.rst | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/doc/source/radl.rst b/doc/source/radl.rst index 6f5136e75..2c0518047 100644 --- a/doc/source/radl.rst +++ b/doc/source/radl.rst @@ -607,3 +607,64 @@ The next RADL deploys a single node that will be configured using Cloud-Init ins It depends on the Cloud provider to process correctly the cloud-init recipes of the configure section. In some cases (EGI FedCloud) it uses the cloud-init language (see `Cloud-Init documentation `_). In other cases as Amazon EC2 or OpenStack it must be a script to be executed in the instance. + +JSON Version +------------ + +There is a JSON version of the RADL language. It has the same semantics that the original RADL but +using JSON syntax to describe the objects. This is a complete example of the JSON format:: + + [ + { + "class": "ansible", + "id": "ansible_jost", + "credentials.username": "user", + "credentials.password": "pass", + "host": "server" + }, + { + "class": "network", + "id": "publica", + "outbound": "yes" + }, + { + "class": "system", + "cpu.arch": "x86_64", + "cpu.count_min": 1, + "disk.0.os.name": "linux", + "id": "front", + "memory.size_min": 536870912, + "net_interface.0.connection": "publica" + }, + { + "class": "configure", + "id": "front", + "recipes": "\\n---\\n- roles:\\n- { role: 'micafer.hadoop', hadoop_master: 'hadoopmaster', hadoop_type_of_node: 'master' }" + }, + { + "class": "deploy", + "system": "front", + "vm_number": 1, + "cloud": "cloud_id" + }, + { + "class": "contextualize", + "items": [ + { + "configure": "front", + "system": "front", + "ctxt_tool": "Ansible" + } + ] + } + ] + +The RADL JSON document is described as a list of objects. Each main object has a field named ``class`` that +described the type of RADL object (ansible, network, system, configure, contextualize or deploy). In case of +ansible, network, system and configure, the must also have and ``id`` field. Then the other fields correspond +to the features described in the RADL object. A particularity of the JSON format is that it does not uses +the comparators (``<=`` or ``>=``) so it is expressed using the ``_min`` and ``_max`` suffixes as show in the +example in ``cpu.count_min`` and ``memory.size_min``. Also the JSON format does not use units in the amount of +memory or disk size, so all these quantities are expresed in bytes. + +Currently this format is only supported in the REST API (not in the native XML-RPC one). \ No newline at end of file From 0c15ced658fc2d1bb945ca36a06d6281ef17a63b Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 21 Jan 2016 16:03:18 +0100 Subject: [PATCH 17/26] Minor bugfix --- connectors/FogBow.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/connectors/FogBow.py b/connectors/FogBow.py index ef265e6b2..f40c68118 100644 --- a/connectors/FogBow.py +++ b/connectors/FogBow.py @@ -63,8 +63,10 @@ def get_auth_headers(self, auth_data): Generate the auth header needed to contact with the FogBow server. """ auth = auth_data.getAuthInfo(FogBowCloudConnector.type) + if not auth: + raise Exception("No correct auth data has been specified to FogBow.") - if auth and 'token_type' in auth[0]: + if 'token_type' in auth[0]: token_type = auth[0]['token_type'] else: # If not token_type supplied, we assume that is VOMS one @@ -164,12 +166,15 @@ def updateVMInfo(self, vm, auth_data): elif resp.status != 200: return (False, resp.reason + "\n" + output) else: + providing_member = self.get_occi_attribute_value(output,'org.fogbowcloud.request.providing-member') + if providing_member == "null": + providing_member = None instance_id = self.get_occi_attribute_value(output,'org.fogbowcloud.request.instance-id') if instance_id == "null": instance_id = None if not instance_id: - vm.state = self.VM_REQ_STATE_MAP.get(self.get_occi_attribute_value(output, 'org.fogbowcloud.request.state'), VirtualMachine.UNKNOWN) + vm.state = VirtualMachine.PENDING return (True, vm) else: # Now get the instance info @@ -205,6 +210,9 @@ def updateVMInfo(self, vm, auth_data): if ssh_user: vm.info.systems[0].addFeature(Feature("disk.0.os.credentials.username", "=", ssh_user), conflict="other", missing="other") + vm.info.systems[0].setValue('instance_id', instance_id) + vm.info.systems[0].setValue('availability_zone', providing_member) + return (True, vm) except Exception, ex: From c05b773e74ecd9ae4d9bb1906554c4c1631d490f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20P=C3=A9rez?= Date: Tue, 26 Jan 2016 10:51:10 +0100 Subject: [PATCH 18/26] Update Dockerfile Added python test libraries (mock, nose) in docker devel --- docker-devel/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 22149d755..97b6a5b3e 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -13,6 +13,8 @@ RUN apt-get update && apt-get install -y \ python-soappy \ python-dateutil \ python-pbr \ + python-mock \ + python-nose \ openssh-client \ sshpass \ git \ From 70c40e90f9b106776ea677a2be714b733c23d453 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 27 Jan 2016 12:11:24 +0100 Subject: [PATCH 19/26] Bugfix in ansible_launcher with host = None --- IM/ansible/ansible_launcher.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/IM/ansible/ansible_launcher.py b/IM/ansible/ansible_launcher.py index a7ee01ece..e9512f4db 100755 --- a/IM/ansible/ansible_launcher.py +++ b/IM/ansible/ansible_launcher.py @@ -140,7 +140,8 @@ def launch_playbook_v2(self): inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=options.inventory) variable_manager.set_inventory(inventory) - inventory.subset(self.host) + if self.host: + inventory.subset(self.host) # let inventory know which playbooks are using so it can know the basedirs inventory.set_playbook_basedir(os.path.dirname(self.playbook_file)) @@ -217,8 +218,9 @@ def launch_playbook_v1(self): inventory = ansible.inventory.Inventory(self.inventory_file) else: inventory = ansible.inventory.Inventory(options.inventory) - - inventory.subset(self.host) + + if self.host: + inventory.subset(self.host) # let inventory know which playbooks are using so it can know the basedirs inventory.set_playbook_basedir(os.path.dirname(self.playbook_file)) From 791d531e388a99bcf87d04b8b23a0d90acb84c96 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 27 Jan 2016 12:11:52 +0100 Subject: [PATCH 20/26] Bugfix in ansible_launcher with host = None --- changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog b/changelog index cf6a7542f..1f21dbdad 100644 --- a/changelog +++ b/changelog @@ -171,3 +171,4 @@ IM 1.4.2 * Add support for new RADL JSON format * Change in Auth Header in new version of FogBow and support for requirements * Code improvements in OpenStack, OpenNebula and FogBow connectors + * Bugfix in ansible_launcher with host = None From 2d7821853a7a3bf5cb08796799e8283cf529464a Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 27 Jan 2016 15:46:51 +0100 Subject: [PATCH 21/26] Added workaround to problems in ansible_launcher with HOSTS_PATTERNS_CACHE --- IM/ansible/ansible_launcher.py | 8 ++++++-- changelog | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/IM/ansible/ansible_launcher.py b/IM/ansible/ansible_launcher.py index e9512f4db..c938b5f06 100755 --- a/IM/ansible/ansible_launcher.py +++ b/IM/ansible/ansible_launcher.py @@ -37,7 +37,7 @@ from ansible.cli import CLI from ansible.parsing.dataloader import DataLoader from ansible.vars import VariableManager - from ansible.inventory import Inventory + import ansible.inventory from ansible_executor_v2 import IMPlaybookExecutor @@ -137,7 +137,11 @@ def launch_playbook_v2(self): options.forks = self.threads loader = DataLoader() - inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=options.inventory) + # Add this to avoid the Ansible bug: no host vars as host is not in inventory + # Hope to remove this if this problem is solved in Ansible + ansible.inventory.HOSTS_PATTERNS_CACHE = {} + + inventory = ansible.inventory.Inventory(loader=loader, variable_manager=variable_manager, host_list=options.inventory) variable_manager.set_inventory(inventory) if self.host: diff --git a/changelog b/changelog index 1f21dbdad..060bd29a9 100644 --- a/changelog +++ b/changelog @@ -171,4 +171,4 @@ IM 1.4.2 * Add support for new RADL JSON format * Change in Auth Header in new version of FogBow and support for requirements * Code improvements in OpenStack, OpenNebula and FogBow connectors - * Bugfix in ansible_launcher with host = None + * Added workaround to problems in ansible_launcher with HOSTS_PATTERNS_CACHE From f11fc40c5291e45ac27c8a1c71444ec1280be01c Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 27 Jan 2016 16:33:31 +0100 Subject: [PATCH 22/26] Bugfixes in REST API --- IM/REST.py | 42 +++++++++++++++++++++++++++------- IM/ansible/ansible_launcher.py | 2 +- changelog | 1 + 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/IM/REST.py b/IM/REST.py index 1d9f42173..81be84da9 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -22,7 +22,7 @@ import json from config import Config from radl.radl_json import parse_radl as parse_radl_json, dump_radl as dump_radl_json -from radl.radl_parse import parse_radl +from bottle import HTTPError AUTH_LINE_SEPARATOR = '\\n' @@ -97,6 +97,16 @@ def run(host, port): def stop(): bottle_server.shutdown() +def get_media_type(header): + """ + Function to get only the header media type + """ + accept = bottle.request.headers.get(header) + pos = accept.find(";") + if pos != 1: + accept = accept[:pos] + return accept.strip() + @app.route('/infrastructures/:id', method='DELETE') def RESTDestroyInfrastructure(id=None): try: @@ -169,6 +179,8 @@ def RESTGetInfrastructureProperty(id=None, prop=None): else: bottle.abort(403, "Incorrect infrastructure property") return str(res) + except HTTPError, ex: + raise ex except DeletedInfrastructureException, ex: bottle.abort(404, "Error Getting Inf. info: " + str(ex)) return False @@ -212,7 +224,7 @@ def RESTCreateInfrastructure(): bottle.abort(401, "No authentication data provided") try: - content_type = bottle.request.headers.get('Content-Type') + content_type = get_media_type('Content-Type') radl_data = bottle.request.body.read() if content_type: @@ -226,6 +238,8 @@ def RESTCreateInfrastructure(): bottle.response.content_type = "text/uri-list" return "http://" + bottle.request.environ['HTTP_HOST'] + "/infrastructures/" + str(inf_id) + except HTTPError, ex: + raise ex except UnauthorizedUserException, ex: bottle.abort(401, "Error Getting Inf. info: " + str(ex)) return False @@ -242,7 +256,7 @@ def RESTGetVMInfo(infid=None, vmid=None): bottle.abort(401, "No authentication data provided") try: - accept = bottle.request.headers.get('Accept') + accept = get_media_type('Accept') radl = InfrastructureManager.GetVMInfo(infid, vmid, auth) @@ -261,6 +275,8 @@ def RESTGetVMInfo(infid=None, vmid=None): bottle.response.content_type = "text/plain" return info + except HTTPError, ex: + raise ex except DeletedInfrastructureException, ex: bottle.abort(404, "Error Getting VM. info: " + str(ex)) return False @@ -291,7 +307,7 @@ def RESTGetVMProperty(infid=None, vmid=None, prop=None): else: info = InfrastructureManager.GetVMProperty(infid, vmid, prop, auth) - accept = bottle.request.headers.get('Accept') + accept = get_media_type('Accept') if accept: if accept == "application/json": bottle.response.content_type = accept @@ -306,6 +322,8 @@ def RESTGetVMProperty(infid=None, vmid=None, prop=None): bottle.response.content_type = "text/plain" return str(info) + except HTTPError, ex: + raise ex except DeletedInfrastructureException, ex: bottle.abort(404, "Error Getting VM. property: " + str(ex)) return False @@ -341,7 +359,7 @@ def RESTAddResource(id=None): else: bottle.abort(400, "Incorrect value in context parameter") - content_type = bottle.request.headers.get('Content-Type') + content_type = get_media_type('Content-Type') radl_data = bottle.request.body.read() if content_type: @@ -361,6 +379,8 @@ def RESTAddResource(id=None): bottle.response.content_type = "text/uri-list" return res + except HTTPError, ex: + raise ex except DeletedInfrastructureException, ex: bottle.abort(404, "Error Adding resources: " + str(ex)) return False @@ -393,6 +413,8 @@ def RESTRemoveResource(infid=None, vmid=None): InfrastructureManager.RemoveResource(infid, vmid, auth, context) bottle.response.content_type = "text/plain" return "" + except HTTPError, ex: + raise ex except DeletedInfrastructureException, ex: bottle.abort(404, "Error Removing resources: " + str(ex)) return False @@ -418,8 +440,8 @@ def RESTAlterVM(infid=None, vmid=None): bottle.abort(401, "No authentication data provided") try: - content_type = bottle.request.headers.get('Content-Type') - accept = bottle.request.headers.get('Accept') + content_type = get_media_type('Content-Type') + accept = get_media_type('Accept') radl_data = bottle.request.body.read() if content_type: @@ -445,6 +467,8 @@ def RESTAlterVM(infid=None, vmid=None): bottle.response.content_type = "text/plain" return res + except HTTPError, ex: + raise ex except DeletedInfrastructureException, ex: bottle.abort(404, "Error modifying resources: " + str(ex)) return False @@ -478,7 +502,7 @@ def RESTReconfigureInfrastructure(id=None): except: bottle.abort(400, "Incorrect vm_list format.") - content_type = bottle.request.headers.get('Content-Type') + content_type = get_media_type('Content-Type') radl_data = bottle.request.body.read() if radl_data: @@ -491,6 +515,8 @@ def RESTReconfigureInfrastructure(id=None): else: radl_data = "" return InfrastructureManager.Reconfigure(id, radl_data, auth, vm_list) + except HTTPError, ex: + raise ex except DeletedInfrastructureException, ex: bottle.abort(404, "Error reconfiguring infrastructure: " + str(ex)) return False diff --git a/IM/ansible/ansible_launcher.py b/IM/ansible/ansible_launcher.py index c938b5f06..328b34581 100755 --- a/IM/ansible/ansible_launcher.py +++ b/IM/ansible/ansible_launcher.py @@ -138,7 +138,7 @@ def launch_playbook_v2(self): loader = DataLoader() # Add this to avoid the Ansible bug: no host vars as host is not in inventory - # Hope to remove this if this problem is solved in Ansible + # In version 2.0.1 it must be fixed ansible.inventory.HOSTS_PATTERNS_CACHE = {} inventory = ansible.inventory.Inventory(loader=loader, variable_manager=variable_manager, host_list=options.inventory) diff --git a/changelog b/changelog index 060bd29a9..c6f4838a2 100644 --- a/changelog +++ b/changelog @@ -172,3 +172,4 @@ IM 1.4.2 * Change in Auth Header in new version of FogBow and support for requirements * Code improvements in OpenStack, OpenNebula and FogBow connectors * Added workaround to problems in ansible_launcher with HOSTS_PATTERNS_CACHE + * Bugfixes in REST API From 4ccf76a831e3e2bf01a0148b76aa37ca3dee591d Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 28 Jan 2016 08:54:23 +0100 Subject: [PATCH 23/26] Bugfixes in REST API --- IM/REST.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/REST.py b/IM/REST.py index 81be84da9..4ebe02a02 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -103,7 +103,7 @@ def get_media_type(header): """ accept = bottle.request.headers.get(header) pos = accept.find(";") - if pos != 1: + if pos != -1: accept = accept[:pos] return accept.strip() From 9f2e72cc7ac9bd3f4e4bcca6878d4b628a63d6a5 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 28 Jan 2016 09:24:28 +0100 Subject: [PATCH 24/26] Bugfixes in REST API --- IM/REST.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/IM/REST.py b/IM/REST.py index 4ebe02a02..549c2e940 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -102,10 +102,13 @@ def get_media_type(header): Function to get only the header media type """ accept = bottle.request.headers.get(header) - pos = accept.find(";") - if pos != -1: - accept = accept[:pos] - return accept.strip() + if accept: + pos = accept.find(";") + if pos != -1: + accept = accept[:pos] + return accept.strip() + else: + return accept @app.route('/infrastructures/:id', method='DELETE') def RESTDestroyInfrastructure(id=None): From c64f2afd02e342044764bc12bd4d3fe86e8dd441 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 28 Jan 2016 10:02:44 +0100 Subject: [PATCH 25/26] Bugfixes in Test REST --- test/TestREST.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/TestREST.py b/test/TestREST.py index d70681bdc..894406416 100755 --- a/test/TestREST.py +++ b/test/TestREST.py @@ -293,10 +293,12 @@ def test_57_reconfigure_list(self): self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") def test_60_stop(self): + time.sleep(10) self.server.request('PUT', "/infrastructures/" + self.inf_id + "/stop", headers = {"Content-type": "application/x-www-form-urlencoded", 'AUTHORIZATION' : self.auth_data}) resp = self.server.getresponse() output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR stopping the infrastructure:" + output) + time.sleep(10) all_stopped = self.wait_inf_state(VirtualMachine.STOPPED, 120, [VirtualMachine.RUNNING]) self.assertTrue(all_stopped, msg="ERROR waiting the infrastructure to be stopped (timeout).") @@ -308,15 +310,18 @@ def test_70_start(self): resp = self.server.getresponse() output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR starting the infrastructure:" + output) + time.sleep(10) all_configured = self.wait_inf_state(VirtualMachine.CONFIGURED, 120, [VirtualMachine.RUNNING]) self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be started (timeout).") def test_80_stop_vm(self): + time.sleep(10) self.server.request('PUT', "/infrastructures/" + self.inf_id + "/vms/0/stop", headers = {"Content-type": "application/x-www-form-urlencoded", 'AUTHORIZATION' : self.auth_data}) resp = self.server.getresponse() output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR stopping the vm:" + output) + time.sleep(10) all_stopped = self.wait_inf_state(VirtualMachine.STOPPED, 120, [VirtualMachine.RUNNING], ["/infrastructures/" + self.inf_id + "/vms/0"]) self.assertTrue(all_stopped, msg="ERROR waiting the infrastructure to be stopped (timeout).") @@ -328,6 +333,7 @@ def test_90_start_vm(self): resp = self.server.getresponse() output = str(resp.read()) self.assertEqual(resp.status, 200, msg="ERROR starting the vm:" + output) + time.sleep(10) all_configured = self.wait_inf_state(VirtualMachine.CONFIGURED, 120, [VirtualMachine.RUNNING], ["/infrastructures/" + self.inf_id + "/vms/0"]) self.assertTrue(all_configured, msg="ERROR waiting the vm to be started (timeout).") From f2f415a1c87d4e35f417380b1e492860db3ece17 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 28 Jan 2016 12:06:39 +0100 Subject: [PATCH 26/26] Update README --- README | 7 ++++++- README.md | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README b/README index 8ba625ef5..6cfffebdb 100644 --- a/README +++ b/README @@ -52,9 +52,14 @@ However, if you install IM from sources you should install: [defaults] transport = smart host_key_checking = False + # For old versions 1.X sudo_user = root sudo_exe = sudo + # For new versions 2.X + become_user = root + become_method = sudo + [paramiko_connection] record_host_keys=False @@ -181,4 +186,4 @@ default configuration. Information about this image can be found here: https://r How to launch the IM service using docker: - $ sudo docker run -d -p 8899:8899 --name im grycap/im \ No newline at end of file + $ sudo docker run -d -p 8899:8899 -p 8800:8800 --name im grycap/im \ No newline at end of file diff --git a/README.md b/README.md index 660a30fbf..7710799fe 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,14 @@ However, if you install IM from sources you should install: [defaults] transport = smart host_key_checking = False +# For old versions 1.X sudo_user = root sudo_exe = sudo +# For new versions 2.X +become_user = root +become_method = sudo + [paramiko_connection] record_host_keys=False @@ -204,5 +209,5 @@ default configuration. Information about this image can be found here: https://r How to launch the IM service using docker: ```sh -sudo docker run -d -p 8899:8899 --name im grycap/im +sudo docker run -d -p 8899:8899 -p 8800:8800 --name im grycap/im ```