Merge pull request #473 from grycap/indigo3

Indigo3

IM/InfrastructureInfo.py

@@ -25,13 +25,15 @@
 from radl.radl import RADL, Feature, deploy, system, contextualize_item
 from radl.radl_parse import parse_radl
 from radl.radl_json import radlToSimple
+from IM.openid.JWT import JWT
 from IM.config import Config
 try:
     from Queue import PriorityQueue
 except ImportError:
     from queue import PriorityQueue
 from IM.VirtualMachine import VirtualMachine
 from IM.auth import Authentication
+from IM.tosca.Tosca import Tosca
 
 
 class IncorrectVMException(Exception):
@@ -91,6 +93,8 @@ def __init__(self):
         """Flag to specify that the configuration threads of this inf has finished successfully or with errors."""
         self.conf_threads = []
         """ List of configuration threads."""
+        self.extra_info = {}
+        """ Extra information about the Infrastructure."""
         self.last_access = datetime.now()
         """ Time of the last access to this Inf. """
         self.snapshots = []
@@ -116,6 +120,8 @@ def serialize(self):
             odict['auth'] = odict['auth'].serialize()
         if odict['radl']:
             odict['radl'] = str(odict['radl'])
+        if odict['extra_info'] and "TOSCA" in odict['extra_info']:
+            odict['extra_info'] = {'TOSCA': odict['extra_info']['TOSCA'].serialize()}
         return json.dumps(odict)
 
     @staticmethod
@@ -130,6 +136,12 @@ def deserialize(str_data):
             dic['auth'] = Authentication.deserialize(dic['auth'])
         if dic['radl']:
             dic['radl'] = parse_radl(dic['radl'])
+        if 'extra_info' in dic and dic['extra_info'] and "TOSCA" in dic['extra_info']:
+            try:
+                dic['extra_info']['TOSCA'] = Tosca.deserialize(dic['extra_info']['TOSCA'])
+            except:
+                del dic['extra_info']['TOSCA']
+                InfrastructureInfo.logger.exception("Error deserializing TOSCA document")
         newinf.__dict__.update(dic)
         newinf.cloud_connector = None
         # Set the ConfManager object and the lock to the data loaded
@@ -264,14 +276,14 @@ def update_radl(self, radl, deployed_vms):
         """
 
         with self._lock:
-            # Add new systems and networks only
+            # Add new networks only
             for s in radl.systems + radl.networks + radl.ansible_hosts:
                 if not self.radl.add(s.clone(), "ignore"):
                     InfrastructureInfo.logger.warn(
                         "Ignoring the redefinition of %s %s" % (type(s), s.getId()))
 
             # Add or update configures
-            for s in radl.configures:
+            for s in radl.configures + radl.systems:
                 self.radl.add(s.clone(), "replace")
                 InfrastructureInfo.logger.warn(
                     "(Re)definition of %s %s" % (type(s), s.getId()))
@@ -571,6 +583,15 @@ def is_authorized(self, auth):
                 if self_im_auth[elem] != other_im_auth[elem]:
                     return False
 
+            if 'token' in self_im_auth:
+                if 'token' not in other_im_auth:
+                    return False
+                decoded_token = JWT().get_info(other_im_auth['token'])
+                password = str(decoded_token['iss']) + str(decoded_token['sub'])
+                # check that the token provided is associated with the current owner of the inf.
+                if self_im_auth['password'] != password:
+                    return False
+
             return True
         else:
             return False

IM/InfrastructureManager.py

@@ -34,6 +34,12 @@
 from radl.radl import Feature, RADL
 from radl.radl_json import dump_radl as dump_radl_json
 
+from IM.config import Config
+from IM.VirtualMachine import VirtualMachine
+from IM.openid.JWT import JWT
+from IM.openid.OpenIDClient import OpenIDClient
+
+
 if Config.MAX_SIMULTANEOUS_LAUNCHES > 1:
     from multiprocessing.pool import ThreadPool
 
@@ -438,6 +444,13 @@ def AddResource(inf_id, radl_data, auth, context=True, failed_clouds=None):
                 raise Exception(
                     "No correct VMRC auth data provided nor image URL")
 
+            if Config.SINGLE_SITE:
+                image_id = os.path.basename(s.getValue("disk.0.image.url"))
+                url_prefix = Config.SINGLE_SITE_IMAGE_URL_PREFIX
+                if not url_prefix.endswith("/"):
+                    url_prefix = url_prefix + "/"
+                s.setValue("disk.0.image.url", url_prefix + image_id)
+
             # Remove the requested apps from the system
             s_without_apps = radl.get_system_by_name(system_id).clone()
             s_without_apps.delValue("disk.0.applications")
@@ -1192,6 +1205,78 @@ def check_im_user(auth):
         else:
             return True
 
+    @staticmethod
+    def check_oidc_token(im_auth):
+        token = im_auth["token"]
+        success = False
+        try:
+            # decode the token to get the info
+            decoded_token = JWT().get_info(token)
+        except Exception as ex:
+            InfrastructureManager.logger.exception("Error trying decode OIDC auth token: %s" % str(ex))
+            raise Exception("Error trying to decode OIDC auth token: %s" % str(ex))
+
+        # First check if the issuer is in valid
+        if decoded_token['iss'] not in Config.OIDC_ISSUERS:
+            InfrastructureManager.logger.error("Incorrect OIDC issuer: %s" % decoded_token['iss'])
+            raise InvaliddUserException("Invalid InfrastructureManager credentials. Issuer not accepted.")
+
+        # Now check the audience
+        if Config.OIDC_AUDIENCE:
+            if 'aud' in decoded_token and decoded_token['aud']:
+                found = False
+                for aud in decoded_token['aud'].split(","):
+                    if aud == Config.OIDC_AUDIENCE:
+                        found = True
+                        break
+                if found:
+                    InfrastructureManager.logger.debug("Audience %s successfully checked." % Config.OIDC_AUDIENCE)
+                else:
+                    InfrastructureManager.logger.error("Audience %s not found in access token." % Config.OIDC_AUDIENCE)
+                    raise InvaliddUserException("Invalid InfrastructureManager credentials. Audience not accepted.")
+            else:
+                InfrastructureManager.logger.error("Audience %s not found in access token." % Config.OIDC_AUDIENCE)
+                raise InvaliddUserException("Invalid InfrastructureManager credentials. Audience not accepted.")
+
+        if Config.OIDC_SCOPES and Config.OIDC_CLIENT_ID and Config.OIDC_CLIENT_SECRET:
+            success, res = OpenIDClient.get_token_introspection(token,
+                                                                Config.OIDC_CLIENT_ID,
+                                                                Config.OIDC_CLIENT_SECRET)
+            if not success:
+                raise InvaliddUserException("Invalid InfrastructureManager credentials. "
+                                            "Invalid token or Client credentials.")
+            else:
+                if not res["scope"]:
+                    raise InvaliddUserException("Invalid InfrastructureManager credentials. "
+                                                "No scope obtained from introspection.")
+                else:
+                    scopes = res["scope"].split(" ")
+                    if not all([elem in scopes for elem in Config.OIDC_SCOPES]):
+                        raise InvaliddUserException("Invalid InfrastructureManager credentials. Scopes %s "
+                                                    "not in introspection scopes: %s" % (" ".join(Config.OIDC_SCOPES),
+                                                                                         res["scope"]))
+
+        # Now check if the token is not expired
+        expired, msg = OpenIDClient.is_access_token_expired(token)
+        if expired:
+            InfrastructureManager.logger.error("OIDC auth %s." % msg)
+            raise InvaliddUserException("Invalid InfrastructureManager credentials. OIDC auth %s." % msg)
+
+        try:
+            # Now try to get user info
+            success, userinfo = OpenIDClient.get_user_info_request(token)
+            if success:
+                # convert to username to use it in the rest of the IM
+                im_auth['username'] = str(userinfo.get("preferred_username"))
+                im_auth['password'] = str(decoded_token['iss']) + str(userinfo.get("sub"))
+        except Exception as ex:
+            InfrastructureManager.logger.exception("Error trying to validate OIDC auth token: %s" % str(ex))
+            raise Exception("Error trying to validate OIDC auth token: %s" % str(ex))
+
+        if not success:
+            InfrastructureManager.logger.error("Incorrect OIDC auth token: %s" % userinfo)
+            raise InvaliddUserException("Invalid InfrastructureManager credentials. %s." % userinfo)
+
     @staticmethod
     def check_auth_data(auth):
         # First check if it is configured to check the users from a list
@@ -1200,10 +1285,26 @@ def check_auth_data(auth):
         if not im_auth:
             raise IncorrectVMCrecentialsException("No credentials provided for the InfrastructureManager.")
 
-        # if not assume the basic user/password auth data
+        # First check if an OIDC token is included
+        if "token" in im_auth[0]:
+            InfrastructureManager.check_oidc_token(im_auth[0])
+
+        # Now check if the user is in authorized
         if not InfrastructureManager.check_im_user(im_auth):
             raise InvaliddUserException()
 
+        if Config.SINGLE_SITE:
+            vmrc_auth = auth.getAuthInfo("VMRC")
+            single_site_auth = auth.getAuthInfo(Config.SINGLE_SITE_TYPE)
+
+            single_site_auth[0]["host"] = Config.SINGLE_SITE_AUTH_HOST
+
+            auth_list = []
+            auth_list.extend(im_auth)
+            auth_list.extend(vmrc_auth)
+            auth_list.extend(single_site_auth)
+            auth = Authentication(auth_list)
+
         # We have to check if TTS is needed for other auth item
         return auth
 

IM/REST.py

@@ -17,6 +17,7 @@
 import logging
 import threading
 import json
+import base64
 import bottle
 
 from IM.InfrastructureInfo import IncorrectVMException, DeletedVMException
@@ -27,6 +28,7 @@
 from IM.config import Config
 from radl.radl_json import parse_radl as parse_radl_json, dump_radl as dump_radl_json, featuresToSimple, radlToSimple
 from radl.radl import RADL, Features, Feature
+from IM.tosca.Tosca import Tosca
 
 logger = logging.getLogger('InfrastructureManager')
 
@@ -168,8 +170,36 @@ def get_auth_header():
     Get the Authentication object from the AUTHORIZATION header
     replacing the new line chars.
     """
-    auth_data = bottle.request.headers[
-        'AUTHORIZATION'].replace(AUTH_NEW_LINE_SEPARATOR, "\n")
+    auth_header = bottle.request.headers['AUTHORIZATION']
+    if Config.SINGLE_SITE:
+        if auth_header.startswith("Basic "):
+            auth_data = base64.b64decode(auth_header[6:])
+            user_pass = auth_data.split(":")
+            im_auth = {"type": "InfrastructureManager",
+                       "username": user_pass[0],
+                       "password": user_pass[1]}
+            single_site_auth = {"type": Config.SINGLE_SITE_TYPE,
+                                "host": Config.SINGLE_SITE_AUTH_HOST,
+                                "username": user_pass[0],
+                                "password": user_pass[1]}
+            return Authentication([im_auth, single_site_auth])
+        elif auth_header.startswith("Bearer "):
+            token = auth_header[7:].strip()
+            im_auth = {"type": "InfrastructureManager",
+                       "username": "user",
+                       "token": token}
+            if Config.SINGLE_SITE_TYPE == "OpenStack":
+                single_site_auth = {"type": Config.SINGLE_SITE_TYPE,
+                                    "host": Config.SINGLE_SITE_AUTH_HOST,
+                                    "username": "indigo-dc",
+                                    "tenant": "oidc",
+                                    "password": token}
+            else:
+                single_site_auth = {"type": Config.SINGLE_SITE_TYPE,
+                                    "host": Config.SINGLE_SITE_AUTH_HOST,
+                                    "token": token}
+            return Authentication([im_auth, single_site_auth])
+    auth_data = auth_header.replace(AUTH_NEW_LINE_SEPARATOR, "\n")
     auth_data = auth_data.split(AUTH_LINE_SEPARATOR)
     return Authentication(Authentication.read_auth_data(auth_data))
 
@@ -326,6 +356,19 @@ def RESTGetInfrastructureProperty(infid=None, prop=None):
             bottle.response.content_type = "application/json"
             res = InfrastructureManager.GetInfrastructureState(infid, auth)
             return format_output(res, default_type="application/json", field_name="state")
+        elif prop == "outputs":
+            accept = get_media_type('Accept')
+            if accept and "application/json" not in accept and "*/*" not in accept and "application/*" not in accept:
+                return return_error(415, "Unsupported Accept Media Types: %s" % accept)
+            bottle.response.content_type = "application/json"
+            auth = InfrastructureManager.check_auth_data(auth)
+            sel_inf = InfrastructureManager.get_infrastructure(infid, auth)
+            if "TOSCA" in sel_inf.extra_info:
+                res = sel_inf.extra_info["TOSCA"].get_outputs(sel_inf)
+            else:
+                bottle.abort(
+                    403, "'outputs' infrastructure property is not valid in this infrastructure")
+            return format_output(res, default_type="application/json", field_name="outputs")
         else:
             return return_error(404, "Incorrect infrastructure property")
 
@@ -377,17 +420,26 @@ def RESTCreateInfrastructure():
     try:
         content_type = get_media_type('Content-Type')
         radl_data = bottle.request.body.read().decode("utf-8")
+        tosca_data = None
 
         if content_type:
             if "application/json" in content_type:
                 radl_data = parse_radl_json(radl_data)
+            elif "text/yaml" in content_type:
+                tosca_data = Tosca(radl_data)
+                _, radl_data = tosca_data.to_radl()
             elif "text/plain" in content_type or "*/*" in content_type or "text/*" in content_type:
                 content_type = "text/plain"
             else:
                 return return_error(415, "Unsupported Media Type %s" % content_type)
 
         inf_id = InfrastructureManager.CreateInfrastructure(radl_data, auth)
 
+        # Store the TOSCA document
+        if tosca_data:
+            sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth)
+            sel_inf.extra_info['TOSCA'] = tosca_data
+
         bottle.response.headers['InfID'] = inf_id
         bottle.response.content_type = "text/uri-list"
         protocol = "http://"
@@ -483,17 +535,35 @@ def RESTAddResource(infid=None):
 
         content_type = get_media_type('Content-Type')
         radl_data = bottle.request.body.read().decode("utf-8")
+        tosca_data = None
+        remove_list = []
 
         if content_type:
             if "application/json" in content_type:
                 radl_data = parse_radl_json(radl_data)
+            elif "text/yaml" in content_type:
+                tosca_data = Tosca(radl_data)
+                auth = InfrastructureManager.check_auth_data(auth)
+                sel_inf = InfrastructureManager.get_infrastructure(infid, auth)
+                # merge the current TOSCA with the new one
+                if isinstance(sel_inf.extra_info['TOSCA'], Tosca):
+                    tosca_data = sel_inf.extra_info['TOSCA'].merge(tosca_data)
+                remove_list, radl_data = tosca_data.to_radl(sel_inf)
             elif "text/plain" in content_type or "*/*" in content_type or "text/*" in content_type:
                 content_type = "text/plain"
             else:
                 return return_error(415, "Unsupported Media Type %s" % content_type)
 
+        if remove_list:
+            InfrastructureManager.RemoveResource(infid, remove_list, auth, context)
+
         vm_ids = InfrastructureManager.AddResource(infid, radl_data, auth, context)
 
+        # Replace the TOSCA document
+        if tosca_data:
+            sel_inf = InfrastructureManager.get_infrastructure(infid, auth)
+            sel_inf.extra_info['TOSCA'] = tosca_data
+
         protocol = "http://"
         if Config.REST_SSL:
             protocol = "https://"

IM/config.py

@@ -89,8 +89,17 @@ class Config:
     CONFMAMAGER_CHECK_STATE_INTERVAL = 5
     UPDATE_CTXT_LOG_INTERVAL = 20
     ANSIBLE_INSTALL_TIMEOUT = 500
+    SINGLE_SITE = False
+    SINGLE_SITE_TYPE = ''
+    SINGLE_SITE_AUTH_HOST = ''
+    SINGLE_SITE_IMAGE_URL_PREFIX = ''
+    OIDC_ISSUERS = ["https://iam-test.indigo-datacloud.eu/"]
+    OIDC_AUDIENCE = None
     INF_CACHE_TIME = None
     VMINFO_JSON = False
+    OIDC_CLIENT_ID = None
+    OIDC_CLIENT_SECRET = None
+    OIDC_SCOPES = []
     VM_NUM_USE_CTXT_DIST = 30
 
 config = ConfigParser()
@@ -105,11 +114,23 @@ class Config:
 if 'IM_DATA_DB' in os.environ:
     Config.DATA_DB = os.environ['IM_DATA_DB']
 
+if 'IM_SINGLE_SITE_ONE_HOST' in os.environ:
+    Config.SINGLE_SITE = True
+    Config.SINGLE_SITE_TYPE = 'OpenNebula'
+    Config.SINGLE_SITE_AUTH_HOST = 'http://%s:2633' % os.environ['IM_SINGLE_SITE_ONE_HOST']
+    Config.SINGLE_SITE_IMAGE_URL_PREFIX = 'one://%s/' % os.environ['IM_SINGLE_SITE_ONE_HOST']
+
 
 class ConfigOpenNebula:
     TEMPLATE_CONTEXT = ''
     TEMPLATE_OTHER = 'GRAPHICS = [type="vnc",listen="0.0.0.0"]'
     IMAGE_UNAME = ''
+    TTS_URL = 'https://localhost:8443'
 
 if config.has_section("OpenNebula"):
     parse_options(config, 'OpenNebula', ConfigOpenNebula)
+
+
+# In this case set assume that the TTS server is in the same server
+if 'IM_SINGLE_SITE_ONE_HOST' in os.environ:
+    ConfigOpenNebula.TTS_URL = 'https://%s:8443' % os.environ['IM_SINGLE_SITE_ONE_HOST']